diff options
Diffstat (limited to 'config-model/src')
332 files changed, 4690 insertions, 3172 deletions
diff --git a/config-model/src/main/Makefile b/config-model/src/main/Makefile index 85f0c888446..7b16bedb4c7 100644 --- a/config-model/src/main/Makefile +++ b/config-model/src/main/Makefile @@ -5,12 +5,12 @@ outputdir=../../target/generated-sources/trang/resources/schema trangjar=../../target/trang.jar -all: ${outputdir} ${outputdir}/services.rng ${outputdir}/hosts.rng ${outputdir}/container-include.rng ${outputdir}/services.xsd ${outputdir}/hosts.xsd ${outputdir}/container-include.xsd ${outputdir}/deployment.xsd +all: ${outputdir} ${outputdir}/services.rng ${outputdir}/hosts.rng ${outputdir}/container-include.rng ${outputdir}/services.xsd ${outputdir}/hosts.xsd ${outputdir}/container-include.xsd ${outputdir}/deployment.xsd ${outputdir}/validation-overrides.xsd ${outputdir}: mkdir -p ${outputdir} -${outputdir}/services.rng: ${srcdir}/services.rnc ${srcdir}/common.rnc ${srcdir}/admin.rnc ${srcdir}/clients.rnc ${srcdir}/docproc.rnc ${srcdir}/routing.rnc ${srcdir}/clients-v2.rnc ${srcdir}/content.rnc ${srcdir}/genericmodule.rnc ${srcdir}/legacygenericcluster.rnc ${srcdir}/genericcluster.rnc ${srcdir}/legacygenericmodule.rnc ${srcdir}/containercluster.rnc +${outputdir}/services.rng: ${srcdir}/services.rnc ${srcdir}/common.rnc ${srcdir}/admin.rnc ${srcdir}/clients.rnc ${srcdir}/docproc.rnc ${srcdir}/routing.rnc ${srcdir}/clients-v2.rnc ${srcdir}/content.rnc ${srcdir}/genericmodule.rnc ${srcdir}/genericcluster.rnc ${srcdir}/legacygenericmodule.rnc ${srcdir}/containercluster.rnc java -jar $(trangjar) -I rnc -O rng ${srcdir}/services.rnc ${outputdir}/services.rng ${outputdir}/services.xsd: ${outputdir}/services.rng @@ -39,7 +39,7 @@ ${outputdir}/deployment.xsd: ${outputdir}/deployment.rng ${outputdir}/validation-overrides.rng: ${srcdir}/validation-overrides.rnc java -jar $(trangjar) -I rnc -O rng ${srcdir}/validation-overrides.rnc ${outputdir}/validation-overrides.rng -${outputdir}/deployment.xsd: ${outputdir}/validation-overrides.rng +${outputdir}/validation-overrides.xsd: ${outputdir}/validation-overrides.rng java -jar $(trangjar) -I rng -O xsd ${outputdir}/validation-overrides.rng ${outputdir}/validation-overrides.xsd clean: diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 8d192414871..0e0f992952a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -138,12 +138,9 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter continue; } if (tagName.equals("config")) { - // TODO: disallow on Vespa 8 + // Top level config, mainly to be used by the Vespa team. continue; } - if (tagName.equals("cluster")) { - throw new IllegalArgumentException("<" + tagName + "> on top-level is not allowed anymore"); - } if ((tagName.equals("clients")) && deployState.isHosted()) throw new IllegalArgumentException("<" + tagName + "> is not allowed when running Vespa in a hosted environment"); @@ -153,7 +150,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter Collection<ConfigModelBuilder> builders = configModelRegistry.resolve(xmlId); if (builders.isEmpty()) - throw new RuntimeException("Could not resolve tag <" + tagName + " version=\"" + tagVersion + "\"> to a config model component"); + throw new IllegalArgumentException("Could not resolve tag <" + tagName + " version=\"" + tagVersion + "\"> to a config model component"); for (ConfigModelBuilder builder : builders) { if ( ! model2Element.containsKey(builder)) { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index dac62ce7e1b..300a77d4a6b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -28,10 +28,10 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; +import com.yahoo.searchdefinition.Application; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -40,7 +40,6 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.container.search.QueryProfilesBuilder; import com.yahoo.vespa.model.container.search.SemanticRuleBuilder; import com.yahoo.vespa.model.container.search.SemanticRules; -import com.yahoo.vespa.model.search.NamedSchema; import java.io.File; import java.io.FileNotFoundException; @@ -69,7 +68,7 @@ public class DeployState implements ConfigDefinitionStore { private final DeployLogger logger; private final FileRegistry fileRegistry; private final DocumentModel documentModel; - private final List<NamedSchema> schemas; + private final List<Schema> schemas; private final ApplicationPackage applicationPackage; private final Optional<ConfigDefinitionRepo> configDefinitionRepo; private final Optional<ApplicationPackage> permanentApplicationPackage; @@ -103,8 +102,7 @@ public class DeployState implements ConfigDefinitionStore { return new Builder().applicationPackage(applicationPackage).build(); } - private DeployState(ApplicationPackage applicationPackage, - SearchDocumentModel searchDocumentModel, + private DeployState(Application application, RankProfileRegistry rankProfileRegistry, FileRegistry fileRegistry, ExecutorService executor, @@ -130,15 +128,15 @@ public class DeployState implements ConfigDefinitionStore { this.fileRegistry = fileRegistry; this.executor = executor; this.rankProfileRegistry = rankProfileRegistry; - this.applicationPackage = applicationPackage; + this.applicationPackage = application.applicationPackage(); this.properties = properties; this.vespaVersion = vespaVersion; this.previousModel = previousModel; this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; this.provisioner = hostProvisioner.orElse(getDefaultModelHostProvisioner(applicationPackage)); this.provisioned = provisioned; - this.schemas = searchDocumentModel.getSchemas(); - this.documentModel = searchDocumentModel.getDocumentModel(); + this.schemas = List.copyOf(application.schemas().values()); + this.documentModel = application.documentModel(); this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; this.endpoints = Set.copyOf(endpoints); @@ -161,7 +159,7 @@ public class DeployState implements ConfigDefinitionStore { return hostsReader == null ? new SingleNodeProvisioner() : new HostsXmlProvisioner(hostsReader); } catch (IOException e) { - throw new IllegalStateException("Could not read hosts.xml", e); + throw new RuntimeException("Could not read hosts.xml", e); } } @@ -217,7 +215,7 @@ public class DeployState implements ConfigDefinitionStore { ImportedMlModels importedModels = new ImportedMlModels(importFrom, executor, modelImporters); for (var entry : importedModels.getSkippedModels().entrySet()) { deployLogger.logApplicationPackage(Level.WARNING, "Skipping import of model " + entry.getKey() + " as an exception " + - "occurred during import. Error: " + entry.getValue()); + "occurred during import. Error: " + entry.getValue()); } return importedModels; } @@ -236,9 +234,7 @@ public class DeployState implements ConfigDefinitionStore { return applicationPackage; } - public List<NamedSchema> getSchemas() { - return schemas; - } + public List<Schema> getSchemas() { return schemas; } public DocumentModel getDocumentModel() { return documentModel; @@ -444,9 +440,10 @@ public class DeployState implements ConfigDefinitionStore { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage, logger); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); - SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, queryProfiles, validationParameters); - return new DeployState(applicationPackage, - searchDocumentModel, + Application application = new ApplicationBuilder(applicationPackage, fileRegistry, logger, properties, + rankProfileRegistry, queryProfiles.getRegistry()) + .build(! validationParameters.ignoreValidationErrors()); + return new DeployState(application, rankProfileRegistry, fileRegistry, executor, @@ -470,46 +467,6 @@ public class DeployState implements ConfigDefinitionStore { reindexing); } - private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, - ValidationParameters validationParameters) { - Collection<NamedReader> readers = applicationPackage.getSchemas(); - Map<String, String> names = new LinkedHashMap<>(); - SchemaBuilder builder = new SchemaBuilder(applicationPackage, fileRegistry, logger, properties, rankProfileRegistry, queryProfiles.getRegistry()); - for (NamedReader reader : readers) { - try { - String readerName = reader.getName(); - String topLevelName = builder.importReader(reader, readerName); - String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); - names.put(topLevelName, sdName); - if ( ! sdName.equals(topLevelName)) { - throw new IllegalArgumentException("Schema file name ('" + sdName + "') and name of " + - "top level element ('" + topLevelName + - "') are not equal for file '" + readerName + "'"); - } - } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); - } catch (IOException e) { - throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); - } finally { - closeIgnoreException(reader.getReader()); - } - } - builder.build(! validationParameters.ignoreValidationErrors()); - return SearchDocumentModel.fromBuilderAndNames(builder, names); - } - - private static String stripSuffix(String nodeName, String postfix) { - assert (nodeName.endsWith(postfix)); - return nodeName.substring(0, nodeName.length() - postfix.length()); - } - - @SuppressWarnings("EmptyCatchBlock") - private static void closeIgnoreException(Reader reader) { - try { - reader.close(); - } catch(Exception e) {} - } } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java deleted file mode 100644 index aaba9a2b98c..00000000000 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.model.deploy; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; -import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.model.search.NamedSchema; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Internal helper class to retrieve document model and search definitions. - * - * @author Ulf Lilleengen - */ -public class SearchDocumentModel { - - private final DocumentModel documentModel; - private final List<NamedSchema> schemas; - - public SearchDocumentModel(DocumentModel documentModel, List<NamedSchema> schemas) { - this.documentModel = documentModel; - this.schemas = schemas; - - } - - public DocumentModel getDocumentModel() { - return documentModel; - } - - public List<NamedSchema> getSchemas() { - return schemas; - } - - public static SearchDocumentModel fromBuilderAndNames(SchemaBuilder builder, Map<String, String> names) { - List<NamedSchema> ret = new ArrayList<>(); - for (Schema schema : builder.getSchemaList()) { - ret.add(new NamedSchema(names.get(schema.getName()), schema)); - } - return new SearchDocumentModel(builder.getModel(), ret); - } - - public static SearchDocumentModel fromBuilder(SchemaBuilder builder) { - List<NamedSchema> ret = new ArrayList<>(); - for (Schema schema : builder.getSchemaList()) { - ret.add(new NamedSchema(schema.getName(), schema)); - } - return new SearchDocumentModel(builder.getModel(), ret); - } - -} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 28933a37ffa..9ae1a75b858 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.deploy; -import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; @@ -52,29 +51,30 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Quota quota = Quota.unlimited(); private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean enableFeedBlockInDistributor = true; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int maxConcurrentMergesPerNode = 16; private int maxMergeQueueSize = 100; - private boolean ignoreMergeQueueLimit = true; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = Collections.emptyList(); - private double resourceLimitDisk = 0.8; + private double resourceLimitDisk = 0.75; private double resourceLimitMemory = 0.8; private double minNodeRatioPerGroup = 0.0; private boolean containerDumpHeapOnShutdownTimeout = false; private double containerShutdownTimeout = 50.0; - private int distributorMergeBusyWait = 1; private int maxUnCommittedMemory = 123456; - private boolean distributorEnhancedMaintenanceScheduling = true; - private boolean asyncApplyBucketDiff = true; private boolean unorderedMergeChaining = true; private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; private String persistenceAsyncThrottling = "UNLIMITED"; + private String mergeThrottlingPolicy = "STATIC"; + private double persistenceThrottlingWsDecrementFactor = 1.2; + private double persistenceThrottlingWsBackoff = 0.95; + private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; + private boolean useV8GeoPositions = false; + private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @@ -107,7 +107,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Quota quota() { return quota; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @@ -115,22 +114,23 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } - @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } - @Override public int metricsproxyNumThreads() { return 1; } @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } - @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } - @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } + @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } + @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } + @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } + @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { @@ -199,11 +199,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setIgnoreMergeQueueLimit(boolean ignoreMergeQueueLimit) { - this.ignoreMergeQueueLimit = ignoreMergeQueueLimit; - return this; - } - public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -230,7 +225,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea } public TestProperties setConfigServerSpecs(List<Spec> configServerSpecs) { - this.configServerSpecs = ImmutableList.copyOf(configServerSpecs); + this.configServerSpecs = List.copyOf(configServerSpecs); return this; } @@ -259,11 +254,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties enableFeedBlockInDistributor(boolean enabled) { - enableFeedBlockInDistributor = enabled; - return this; - } - public TestProperties maxActivationInhibitedOutOfSyncGroups(int nGroups) { maxActivationInhibitedOutOfSyncGroups = nGroups; return this; @@ -304,21 +294,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setDistributorMergeBusyWait(int value) { - distributorMergeBusyWait = value; - return this; - } - - public TestProperties distributorEnhancedMaintenanceScheduling(boolean enhancedScheduling) { - distributorEnhancedMaintenanceScheduling = enhancedScheduling; - return this; - } - - public TestProperties setAsyncApplyBucketDiff(boolean value) { - asyncApplyBucketDiff = value; - return this; - } - public TestProperties setUnorderedMergeChaining(boolean unordered) { unorderedMergeChaining = unordered; return this; @@ -344,6 +319,36 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMergeThrottlingPolicy(String policy) { + this.mergeThrottlingPolicy = policy; + return this; + } + + public TestProperties setPersistenceThrottlingWsDecrementFactor(double factor) { + this.persistenceThrottlingWsDecrementFactor = factor; + return this; + } + + public TestProperties setPersistenceThrottlingWsBackoff(double backoff) { + this.persistenceThrottlingWsBackoff = backoff; + return this; + } + + public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) { + this.inhibitDefaultMergesWhenGlobalMergesPending = value; + return this; + } + + public TestProperties setUseV8GeoPositions(boolean value) { + this.useV8GeoPositions = value; + return this; + } + + public TestProperties setEnvironmentVariables(List<String> value) { + this.environmentVariables = value; + return this; + } + public TestProperties setAvoidRenamingSummaryFeatures(boolean value) { this.avoidRenamingSummaryFeatures = value; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 47a5fb24a43..f64d71ee364 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -160,7 +160,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce * because config IDs must be registered through setConfigId(). */ public final String getConfigId() { - if (configId == null) throw new RuntimeException("The system topology must be frozen first."); + if (configId == null) throw new IllegalStateException("The system topology must be frozen first."); return configId; } @@ -234,6 +234,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce return didApply; } + @SuppressWarnings("removal") // TODO Vespa 8: remove private void applyUserConfig(ConfigInstance.Builder builder, ConfigPayloadBuilder payloadBuilder) { ConfigInstance.Builder override; if (builder instanceof GenericConfig.GenericConfigBuilder) { diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index c678618311e..c9a03dad65e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -72,7 +72,7 @@ public class Hosts { for (Element hostE : XML.getChildren(doc.getDocumentElement(), "host")) { String name = hostE.getAttribute("name"); if (name.equals("")) { - throw new RuntimeException("Missing 'name' attribute for host."); + throw new IllegalArgumentException("Missing 'name' attribute for host."); } if ("localhost".equals(name)) { name = HostName.getLocalhost(); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 5822b1bca05..824bf248b5c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -19,7 +19,7 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.config.application.api.ApplicationPackage; @@ -33,6 +33,7 @@ import java.io.InputStream; import java.io.Reader; import java.io.StringReader; import java.io.UncheckedIOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -95,8 +96,8 @@ public class MockApplicationPackage implements ApplicationPackage { /** Returns the root of this application package relative to the current dir */ protected File root() { return root; } + @SuppressWarnings("deprecation") // not redundant @Override - @SuppressWarnings("deprecation") // NOT redundant public String getApplicationName() { return "mock application"; } @@ -118,23 +119,26 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> getSchemas() { ArrayList<NamedReader> readers = new ArrayList<>(); - SchemaBuilder schemaBuilder = new SchemaBuilder(this, - new MockFileRegistry(), - new BaseDeployLogger(), - new TestProperties(), - new RankProfileRegistry(), - queryProfileRegistry); - for (String sd : schemas) { - try { - String name = schemaBuilder.importString(sd); - readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd))); - } catch (ParseException e) { - throw new RuntimeException(e); - } - } + for (String sd : schemas) + readers.add(new NamedReader(extractSdName(sd) + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd))); return readers; } + /** To avoid either double parsing or supplying a name explicitly */ + private String extractSdName(String sd) { + String s = sd.split("\n")[0]; + if (s.startsWith("schema")) + s = s.substring("schema".length()).trim(); + else if (s.startsWith("search")) + s = s.substring("search".length()).trim(); + else + throw new IllegalArgumentException("Expected the first line of a schema but got '" + sd + "'"); + int end = s.indexOf(' '); + if (end < 0) + end = s.indexOf('}'); + return s.substring(0, end).trim(); + } + @Override public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { return Collections.emptyMap(); @@ -142,7 +146,21 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> getFiles(Path dir, String fileSuffix, boolean recurse) { - return new ArrayList<>(); + try { + if (dir.elements().contains(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getName())) return List.of(); // No legacy paths + File dirFile = new File(root, dir.getName()); + if ( ! dirFile.exists()) return List.of(); + if (recurse) throw new RuntimeException("Recurse not implemented"); + List<NamedReader> readers = new ArrayList<>(); + for (var i = Files.list(dirFile.toPath()).filter(p -> p.getFileName().toString().endsWith(fileSuffix)).iterator(); i.hasNext(); ) { + var file = i.next(); + readers.add(new NamedReader(file.toString(), IOUtils.createReader(file.toString()))); + } + return readers; + } + catch (IOException e) { + throw new RuntimeException(e); + } } @Override diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java index 8848759b415..9f4eeeb44c9 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java @@ -27,8 +27,8 @@ public class DataTypeRepo implements DataTypeCollection { public DataTypeRepo add(DataType type) { if (typeByName.containsKey(type.getName()) || typeById.containsKey(type.getId())) { - throw new IllegalStateException("Data type '" + type.getName() + "', id '" + - type.getId() + "' is already registered."); + throw new IllegalArgumentException("Data type '" + type.getName() + "', id '" + + type.getId() + "' is already registered."); } typeByName.put(type.getName(), type); typeById.put(type.getId(), type); @@ -43,9 +43,7 @@ public class DataTypeRepo implements DataTypeCollection { } public DataTypeRepo replace(DataType type) { - if (!typeByName.containsKey(type.getName()) || - !typeById.containsKey(type.getId())) - { + if (!typeByName.containsKey(type.getName()) || !typeById.containsKey(type.getId())) { throw new IllegalStateException("Data type '" + type.getName() + "' is not registered."); } var oldByName = typeByName.remove(type.getName()); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java index 42572ae763a..885db34510b 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java @@ -28,10 +28,10 @@ public class DocumentTypeRepo implements DocumentTypeCollection { public DocumentTypeRepo add(NewDocumentType type) { if (typeByName.containsKey(type.getFullName())) { - throw new IllegalStateException("Document type " + type.toString() + " is already registered"); + throw new IllegalArgumentException("Document type " + type + " is already registered"); } if (typeById.containsKey(type.getFullName().getId())) { - throw new IllegalStateException("Document type " + type.toString() + " is already registered"); + throw new IllegalArgumentException("Document type " + type + " is already registered"); } typeByName.put(type.getFullName(), type); typeById.put(type.getFullName().getId(), type); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java index fb76c945344..2dda670f07c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java @@ -3,10 +3,20 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.searchdefinition.derived.SearchOrderer; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.searchdefinition.processing.Processor; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; /** * A collection of objects representing the content of an application package. @@ -18,28 +28,79 @@ import java.util.Map; public class Application { private final ApplicationPackage applicationPackage; - private final Map<String, Schema> schemas = new LinkedHashMap<>(); + private final Map<String, Schema> schemas; + private final DocumentModel documentModel; - public Application(ApplicationPackage applicationPackage) { + public Application(ApplicationPackage applicationPackage, + List<Schema> schemas, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles, + ModelContext.Properties properties, + boolean documentsOnly, + boolean validate, + Set<Class<? extends Processor>> processorsToSkip, + DeployLogger logger) { this.applicationPackage = applicationPackage; - } - public ApplicationPackage applicationPackage() { return applicationPackage; } + Map<String, Schema> schemaMap = new LinkedHashMap<>(); + for (Schema schema : schemas) { + if (schemaMap.containsKey(schema.getName())) + throw new IllegalArgumentException("Duplicate schema '" + schema.getName() + "' in " + this); + schemaMap.put(schema.getName(), schema); + } + this.schemas = Collections.unmodifiableMap(schemaMap); + + schemas.forEach(schema -> schema.setOwner(this)); + if (validate) + schemas.forEach(schema -> schema.validate(logger)); + + new TemporarySDTypeResolver(schemas, logger).process(); + + List<SDDocumentType> sdocs = new ArrayList<>(); + sdocs.add(SDDocumentType.VESPA_DOCUMENT); + for (Schema schema : schemas) { + if (schema.hasDocument()) { + sdocs.add(schema.getDocument()); + } + } + + var orderer = new SDDocumentTypeOrderer(sdocs, logger); + orderer.process(); + for (SDDocumentType sdoc : orderer.getOrdered()) { + new FieldOperationApplierForStructs().process(sdoc); + new FieldOperationApplier().process(sdoc); + } + var resolver = new DocumentReferenceResolver(schemas); + sdocs.forEach(resolver::resolveReferences); + sdocs.forEach(resolver::resolveInheritedReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(schemas); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); - public void add(Schema schema) { - if (schemas.containsKey(schema.getName())) - throw new IllegalArgumentException("Duplicate schema '" + schema.getName() + "' in " + this); - schemas.put(schema.getName(), schema); + if (validate) + new DocumentGraphValidator().validateDocumentGraph(sdocs); + + List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas); + for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { + new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? + new Processing(properties).process(schema, + logger, + rankProfileRegistry, + queryProfiles, + validate, + documentsOnly, + processorsToSkip); + } + + this.documentModel = new DocumentModelBuilder().build(schemasSomewhatOrdered); } + public ApplicationPackage applicationPackage() { return applicationPackage; } + /** Returns an unmodifiable list of the schemas of this application */ - public Map<String, Schema> schemas() { return Collections.unmodifiableMap(schemas); } + public Map<String, Schema> schemas() { return schemas; } - /** Validates this. Must be called after all content is added to it. */ - public void validate(DeployLogger logger) { - schemas.values().forEach(schema -> schema.validate(logger)); - } + public DocumentModel documentModel() { return documentModel; } @Override public String toString() { return "application " + applicationPackage.getApplicationId(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SchemaBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java index 51466a5dbfa..533546b4d39 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SchemaBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java @@ -12,15 +12,13 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DocumentTypeManager; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; -import com.yahoo.searchdefinition.derived.SearchOrderer; -import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.parser.SDParser; import com.yahoo.searchdefinition.parser.SimpleCharStream; import com.yahoo.searchdefinition.parser.TokenMgrException; -import com.yahoo.searchdefinition.processing.Processing; import com.yahoo.searchdefinition.processing.Processor; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -28,28 +26,26 @@ import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; +import java.io.Reader; import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; /** - * Helper class for building {@link Schema}s. The pattern for using this is to 1) Import - * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank - * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built - * search objects using the {@link #getSchema(String)} method. + * Application builder. Usage: + * 1) Add all schemas, using the addXXX() methods, + * 2) provide the available rank types and rank expressions, using the setRankXXX() methods, + * 3) invoke the {@link #build} method */ -// NOTE: Since this was created we have added Application, and much of the content in this should migrate there. -public class SchemaBuilder { +public class ApplicationBuilder { - private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); - private final DocumentModel model = new DocumentModel(); - private final Application application; + private final ApplicationPackage applicationPackage; + private final List<Schema> schemas = new ArrayList<>(); + private final DocumentTypeManager documentTypeManager = new DocumentTypeManager(); private final RankProfileRegistry rankProfileRegistry; private final QueryProfileRegistry queryProfileRegistry; private final FileRegistry fileRegistry; @@ -58,221 +54,224 @@ public class SchemaBuilder { /** True to build the document aspect only, skipping instantiation of rank profiles */ private final boolean documentsOnly; - private boolean isBuilt = false; + private Application application; private final Set<Class<? extends Processor>> processorsToSkip = new HashSet<>(); /** For testing only */ - public SchemaBuilder() { + public ApplicationBuilder() { this(new RankProfileRegistry(), new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(DeployLogger deployLogger) { + public ApplicationBuilder(DeployLogger deployLogger) { this(MockApplicationPackage.createEmpty(), deployLogger); } /** For testing only */ - public SchemaBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { this(MockApplicationPackage.createEmpty(), deployLogger, rankProfileRegistry); } /** Used for generating documents for typed access to document fields in Java */ - public SchemaBuilder(boolean documentsOnly) { + public ApplicationBuilder(boolean documentsOnly) { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); } /** For testing only */ - public SchemaBuilder(ApplicationPackage app, DeployLogger deployLogger) { + public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger) { this(app, new MockFileRegistry(), deployLogger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { this(app, new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry) { this(rankProfileRegistry, new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { this(rankProfileRegistry, queryProfileRegistry, new TestProperties()); } - public SchemaBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); } - public SchemaBuilder(ApplicationPackage app, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { + public ApplicationBuilder(ApplicationPackage app, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { this(app, fileRegistry, deployLogger, properties, rankProfileRegistry, queryProfileRegistry, false); } - private SchemaBuilder(ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry, - boolean documentsOnly) { - this.application = new Application(applicationPackage); + private ApplicationBuilder(ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { + this.applicationPackage = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; this.fileRegistry = fileRegistry; this.deployLogger = deployLogger; this.properties = properties; this.documentsOnly = documentsOnly; + for (NamedReader reader : applicationPackage.getSchemas()) + addSchema(reader); } /** - * Import search definition. + * Adds a schema to this application. * * @param fileName the name of the file to import * @return the name of the imported object * @throws IOException thrown if the file can not be read for some reason * @throws ParseException thrown if the file does not contain a valid search definition */ - public String importFile(String fileName) throws IOException, ParseException { + public Schema addSchemaFile(String fileName) throws IOException, ParseException { File file = new File(fileName); - return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent()); - } - - private String importFile(Path file) throws IOException, ParseException { - return importFile(file.toString()); + return addSchema(IOUtils.readFile(file)); } /** - * Reads and parses the search definition string provided by the given reader. Once all search definitions have been - * imported, call {@link #build()}. + * Reads and parses the schema string provided by the given reader. Once all schemas have been + * imported, call {@link #build}. * - * @param reader the reader whose content to import - * @param searchDefDir the path to use when resolving file references - * @return the name of the imported object - * @throws ParseException thrown if the file does not contain a valid search definition + * @param reader the reader whose content to import */ - public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException { - return importString(IOUtils.readAll(reader), searchDefDir); + public void addSchema(NamedReader reader) { + try { + String schemaName = addSchema(IOUtils.readAll(reader)).getName(); + String schemaFileName = stripSuffix(reader.getName(), ApplicationPackage.SD_NAME_SUFFIX); + if ( ! schemaFileName.equals(schemaName)) { + throw new IllegalArgumentException("The file containing schema '" + schemaName + "' must be named '" + + schemaName + ApplicationPackage.SD_NAME_SUFFIX + "', not " + reader.getName()); + } + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); + } finally { + closeIgnoreException(reader.getReader()); + } + } + + private static String stripSuffix(String readerName, String suffix) { + if ( ! readerName.endsWith(suffix)) + throw new IllegalArgumentException("Schema '" + readerName + "' does not end with " + suffix); + return readerName.substring(0, readerName.length() - suffix.length()); } /** - * Import search definition. + * Adds a schema to this * - * @param str the string to parse - * @return the name of the imported object - * @throws ParseException thrown if the file does not contain a valid search definition + * @param schemaString the content of the schema */ - public String importString(String str) throws ParseException { - return importString(str, null); - } - - private String importString(String str, String searchDefDir) throws ParseException { - SimpleCharStream stream = new SimpleCharStream(str); - try { - return importRawSchema(new SDParser(stream, fileRegistry, deployLogger, properties, application, - rankProfileRegistry, documentsOnly) - .schema(docTypeMgr, searchDefDir)); - } catch (TokenMgrException e) { - throw new ParseException("Unknown symbol: " + e.getMessage()); - } catch (ParseException pe) { - throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); - } + public Schema addSchema(String schemaString) throws ParseException { + return add(createSchema(schemaString)); } /** - * Registers the given schema to the application to be built during {@link #build()}. A + * Registers the given schema to the application to be built during {@link #build}. A * {@link Schema} object is considered to be "raw" if it has not already been processed. This is the case for most * programmatically constructed schemas used in unit tests. * * @param schema the object to import - * @return the name of the imported object * @throws IllegalArgumentException if the given search object has already been processed */ - public String importRawSchema(Schema schema) { + public Schema add(Schema schema) { if (schema.getName() == null) throw new IllegalArgumentException("Schema has no name"); - String rawName = schema.getName(); - application.add(schema); - return rawName; + schemas.add(schema); + return schema; } - /** - * Processes and finalizes the schemas of this. - * Only for testing. - * - * @throws IllegalStateException Thrown if this method has already been called. - */ - public void build() { - build(true); + private Schema createSchema(String schemaString) throws ParseException { + Schema schema = parseSchema(schemaString); + addRankProfileFiles(schema); + return schema; } - /** - * Processes and finalizes the schemas of this. - * - * @throws IllegalStateException thrown if this method has already been called - */ - public void build(boolean validate) { - if (isBuilt) throw new IllegalStateException("Application already built"); + private Schema parseSchema(String schemaString) throws ParseException { + SimpleCharStream stream = new SimpleCharStream(schemaString); + try { + return parserOf(stream).schema(documentTypeManager); + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); + } + } + + private void addRankProfileFiles(Schema schema) { + if (applicationPackage == null) return; - new TemporarySDTypeResolver(application.schemas().values(), deployLogger).process(); + Path legacyRankProfilePath = ApplicationPackage.SEARCH_DEFINITIONS_DIR.append(schema.getName()); + for (NamedReader reader : applicationPackage.getFiles(legacyRankProfilePath, ".profile")) + parseRankProfile(reader, schema); - if (validate) - application.validate(deployLogger); + Path rankProfilePath = ApplicationPackage.SCHEMAS_DIR.append(schema.getName()); + for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile")) + parseRankProfile(reader, schema); + } - List<SDDocumentType> sdocs = new ArrayList<>(); - sdocs.add(SDDocumentType.VESPA_DOCUMENT); - for (Schema schema : application.schemas().values()) { - if (schema.hasDocument()) { - sdocs.add(schema.getDocument()); + /** Parses the rank profile of the given reader and adds it to the rank profile registry for this schema. */ + private void parseRankProfile(NamedReader reader, Schema schema) { + try { + SimpleCharStream stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); + try { + parserOf(stream).rankProfile(schema); + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); } } - - var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); - orderer.process(); - for (SDDocumentType sdoc : orderer.getOrdered()) { - new FieldOperationApplierForStructs().process(sdoc); - new FieldOperationApplier().process(sdoc); + catch (IOException e) { + throw new IllegalArgumentException("Could not read rank profile " + reader.getName(), e); } - - var resolver = new DocumentReferenceResolver(application.schemas().values()); - sdocs.forEach(resolver::resolveReferences); - sdocs.forEach(resolver::resolveInheritedReferences); - var importedFieldsEnumerator = new ImportedFieldsEnumerator(application.schemas().values()); - sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); - - if (validate) - new DocumentGraphValidator().validateDocumentGraph(sdocs); - - var builder = new DocumentModelBuilder(model); - List<Schema> schemasSomewhatOrdered = new ArrayList<>(application.schemas().values()); - for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { - new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? - process(schema, new QueryProfiles(queryProfileRegistry, deployLogger), validate); + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse rank profile " + reader.getName(), e); } - builder.addToModel(schemasSomewhatOrdered); - isBuilt = true; } - /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ - public Set<Class<? extends Processor>> processorsToSkip() { return processorsToSkip; } + private SDParser parserOf(SimpleCharStream stream) { + return new SDParser(stream, applicationPackage, fileRegistry, deployLogger, properties, + rankProfileRegistry, documentsOnly); + } /** - * Processes and returns the given {@link Schema} object. This method has been factored out of the {@link - * #build()} method so that subclasses can choose not to build anything. + * Processes and finalizes the schemas of this. + * + * @throws IllegalStateException thrown if this method has already been called */ - private void process(Schema schema, QueryProfiles queryProfiles, boolean validate) { - new Processing(properties).process(schema, deployLogger, - rankProfileRegistry, queryProfiles, - validate, documentsOnly, - processorsToSkip); + public Application build(boolean validate) { + if (application != null) throw new IllegalStateException("Application already built"); + + application = new Application(applicationPackage, + schemas, + rankProfileRegistry, + new QueryProfiles(queryProfileRegistry, deployLogger), + properties, + documentsOnly, + validate, + processorsToSkip, + deployLogger); + return application; } + /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ + public Set<Class<? extends Processor>> processorsToSkip() { return processorsToSkip; } + /** * Convenience method to call {@link #getSchema(String)} when there is only a single {@link Schema} object * built. This method will never return null. @@ -281,7 +280,7 @@ public class SchemaBuilder { * @throws IllegalStateException if there is not exactly one search. */ public Schema getSchema() { - if ( ! isBuilt) throw new IllegalStateException("Application not built."); + if (application == null) throw new IllegalStateException("Application not built"); if (application.schemas().size() != 1) throw new IllegalStateException("This call only works if we have 1 schema. Schemas: " + application.schemas().values()); @@ -289,9 +288,7 @@ public class SchemaBuilder { return application.schemas().values().stream().findAny().get(); } - public DocumentModel getModel() { - return model; - } + public DocumentModel getModel() { return application.documentModel(); } /** * Returns the built {@link Schema} object that has the given name. If the name is unknown, this method will simply @@ -300,10 +297,10 @@ public class SchemaBuilder { * @param name the name of the schema to return, * or null to return the only one or throw an exception if there are multiple to choose from * @return the built object, or null if none with this name - * @throws IllegalStateException if {@link #build()} has not been called. + * @throws IllegalStateException if {@link #build} has not been called. */ public Schema getSchema(String name) { - if ( ! isBuilt) throw new IllegalStateException("Application not built."); + if (application == null) throw new IllegalStateException("Application not built"); if (name == null) return getSchema(); return application.schemas().get(name); } @@ -323,24 +320,24 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a string. * * @param sd the string to build from - * @return the built {@link SchemaBuilder} object + * @return the built {@link ApplicationBuilder} object * @throws ParseException thrown if there is a problem parsing the string */ - public static SchemaBuilder createFromString(String sd) throws ParseException { + public static ApplicationBuilder createFromString(String sd) throws ParseException { return createFromString(sd, new BaseDeployLogger()); } - public static SchemaBuilder createFromString(String sd, DeployLogger logger) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(logger); - builder.importString(sd); + public static ApplicationBuilder createFromString(String sd, DeployLogger logger) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(sd); builder.build(true); return builder; } - public static SchemaBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(logger); + public static ApplicationBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(logger); for (var schema : schemas) - builder.importString(schema); + builder.addSchema(schema); builder.build(true); return builder; } @@ -349,26 +346,26 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. * * @param fileName the file to build from - * @return the built {@link SchemaBuilder} object + * @return the built {@link ApplicationBuilder} object * @throws IOException if there was a problem reading the file. * @throws ParseException if there was a problem parsing the file content. */ - public static SchemaBuilder createFromFile(String fileName) throws IOException, ParseException { + public static ApplicationBuilder createFromFile(String fileName) throws IOException, ParseException { return createFromFile(fileName, new BaseDeployLogger()); } /** * Convenience factory methdd to create a SearchBuilder from multiple SD files. Only for testing. */ - public static SchemaBuilder createFromFiles(Collection<String> fileNames) throws IOException, ParseException { + public static ApplicationBuilder createFromFiles(Collection<String> fileNames) throws IOException, ParseException { return createFromFiles(fileNames, new BaseDeployLogger()); } - public static SchemaBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { + public static ApplicationBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { return createFromFile(fileName, logger, new RankProfileRegistry(), new QueryProfileRegistry()); } - private static SchemaBuilder createFromFiles(Collection<String> fileNames, DeployLogger logger) throws IOException, ParseException { + private static ApplicationBuilder createFromFiles(Collection<String> fileNames, DeployLogger logger) throws IOException, ParseException { return createFromFiles(fileNames, new MockFileRegistry(), logger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); } @@ -378,14 +375,14 @@ public class SchemaBuilder { * @param fileName the file to build from. * @param deployLogger logger for deploy messages. * @param rankProfileRegistry registry for rank profiles. - * @return the built {@link SchemaBuilder} object. + * @return the built {@link ApplicationBuilder} object. * @throws IOException if there was a problem reading the file. * @throws ParseException if there was a problem parsing the file content. */ - private static SchemaBuilder createFromFile(String fileName, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) + private static ApplicationBuilder createFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { return createFromFiles(Collections.singletonList(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, queryprofileRegistry); @@ -394,62 +391,62 @@ public class SchemaBuilder { /** * Convenience factory methdd to create a SearchBuilder from multiple SD files.. */ - private static SchemaBuilder createFromFiles(Collection<String> fileNames, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) + private static ApplicationBuilder createFromFiles(Collection<String> fileNames, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { - SchemaBuilder builder = new SchemaBuilder(MockApplicationPackage.createEmpty(), - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryprofileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + queryprofileRegistry); for (String fileName : fileNames) { - builder.importFile(fileName); + builder.addSchemaFile(fileName); } builder.build(true); return builder; } - public static SchemaBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { + public static ApplicationBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { return createFromDirectory(dir, fileRegistry, logger, properties, new RankProfileRegistry()); } - public static SchemaBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { + public static ApplicationBuilder createFromDirectory(String dir, + FileRegistry fileRegistry, + DeployLogger logger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { return createFromDirectory(dir, fileRegistry, logger, properties, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir)); } - private static SchemaBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + private static ApplicationBuilder createFromDirectory(String dir, + FileRegistry fileRegistry, + DeployLogger logger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { return createFromDirectory(dir, MockApplicationPackage.fromSearchDefinitionAndRootDirectory(dir), fileRegistry, logger, properties, rankProfileRegistry, queryProfileRegistry); } - private static SchemaBuilder createFromDirectory(String dir, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - SchemaBuilder builder = new SchemaBuilder(applicationPackage, - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryProfileRegistry); - for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { - builder.importFile(i.next()); + private static ApplicationBuilder createFromDirectory(String dir, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + ApplicationBuilder builder = new ApplicationBuilder(applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + queryProfileRegistry); + for (var i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { + builder.addSchemaFile(i.next().toString()); } builder.build(true); return builder; @@ -513,15 +510,15 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a raw object. * * @param rawSchema the raw object to build from - * @return the built {@link SchemaBuilder} object - * @see #importRawSchema(Schema) + * @return the built {@link ApplicationBuilder} object + * @see #add(Schema) */ - public static SchemaBuilder createFromRawSchema(Schema rawSchema, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry, queryProfileRegistry); - builder.importRawSchema(rawSchema); - builder.build(); + public static ApplicationBuilder createFromRawSchema(Schema rawSchema, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry); + builder.add(rawSchema); + builder.build(true); return builder; } @@ -530,7 +527,7 @@ public class SchemaBuilder { * * @param rawSchema the raw object to build from * @return the built {@link Schema} object - * @see #importRawSchema(Schema) + * @see #add(Schema) */ public static Schema buildFromRawSchema(Schema rawSchema, RankProfileRegistry rankProfileRegistry, @@ -550,4 +547,10 @@ public class SchemaBuilder { public DeployLogger getDeployLogger() { return deployLogger; } + @SuppressWarnings("EmptyCatchBlock") + private static void closeIgnoreException(Reader reader) { + try { + reader.close(); + } catch(Exception e) {} + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java index 56a739ced8b..812726a609e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java @@ -4,52 +4,48 @@ package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.document.ImmutableSDField; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; /** * The rank profile containing default settings. This is derived from the fields * whenever this is accessed. * - * @author bratseth + * @author bratseth */ public class DefaultRankProfile extends RankProfile { /** * Creates a new rank profile * - * @param rankProfileRegistry The {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing and looking up rank profiles. + * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} + * to use for storing and looking up rank profiles */ public DefaultRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { super("default", schema, rankProfileRegistry, rankingConstants); } - /** - * Does nothing, the default rank profile can not inherit anything - */ - // TODO: Why not? If that's the case, then fail attempts at it - public void setInherited(String inheritedName) { - } - - /** Returns null, the default rank profile can not inherit anything */ - public String getInheritedName() { - return null; + /** Ignore self inheriting of default as some applications may use that for historical reasons. */ + public void inherit(String inheritedName) { + if (inheritedName.equals("default")) return; + super.inherit(inheritedName); } - /** Returns the rank boost value of the given field */ - public RankSetting getRankSetting(String fieldOrIndex,RankSetting.Type type) { - RankSetting setting = super.getRankSetting(fieldOrIndex,type); + @Override + public RankSetting getRankSetting(String fieldOrIndex, RankSetting.Type type) { + RankSetting setting = super.getRankSetting(fieldOrIndex, type); if (setting != null) return setting; - ImmutableSDField field = getSearch().getConcreteField(fieldOrIndex); + ImmutableSDField field = schema().getConcreteField(fieldOrIndex); if (field != null) { - setting = toRankSetting(field,type); + setting = toRankSetting(field, type); if (setting != null) return setting; } - Index index = getSearch().getIndex(fieldOrIndex); + Index index = schema().getIndex(fieldOrIndex); if (index != null) { - setting = toRankSetting(index,type); + setting = toRankSetting(index, type); if (setting != null) return setting; } @@ -57,13 +53,13 @@ public class DefaultRankProfile extends RankProfile { return null; } - private RankSetting toRankSetting(ImmutableSDField field,RankSetting.Type type) { - if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight()>0 && field.getWeight()!=100) - return new RankSetting(field.getName(),type,field.getWeight()); + private RankSetting toRankSetting(ImmutableSDField field, RankSetting.Type type) { + if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight() > 0 && field.getWeight() != 100) + return new RankSetting(field.getName(), type, field.getWeight()); if (type.equals(RankSetting.Type.RANKTYPE)) - return new RankSetting(field.getName(),type,field.getRankType()); - if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost()>0) - return new RankSetting(field.getName(),type,field.getLiteralBoost()); + return new RankSetting(field.getName(), type, field.getRankType()); + if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost() > 0) + return new RankSetting(field.getName(), type, field.getLiteralBoost()); // Index level setting really if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { @@ -86,18 +82,19 @@ public class DefaultRankProfile extends RankProfile { * Returns the names of the fields which have a rank boost setting * explicitly in this profile or in fields */ + @Override public Set<RankSetting> rankSettings() { Set<RankSetting> settings = new LinkedHashSet<>(20); settings.addAll(this.rankSettings); - for (ImmutableSDField field : getSearch().allConcreteFields() ) { + for (ImmutableSDField field : schema().allConcreteFields() ) { addSetting(field, RankSetting.Type.WEIGHT, settings); addSetting(field, RankSetting.Type.RANKTYPE, settings); addSetting(field, RankSetting.Type.LITERALBOOST, settings); addSetting(field, RankSetting.Type.PREFERBITVECTOR, settings); } - // Foer settings that really pertains to indexes do the explicit indexes too - for (Index index : getSearch().getExplicitIndices()) { + // For settings that really pertains to indexes do the explicit indexes too + for (Index index : schema().getExplicitIndices()) { addSetting(index, RankSetting.Type.PREFERBITVECTOR, settings); } return settings; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 55f24123940..8b068381e07 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -32,14 +32,13 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; - -import static java.util.Collections.emptySet; -import static java.util.stream.Collectors.toSet; +import java.util.stream.Collectors; /** * @author baldersheim @@ -48,12 +47,12 @@ public class DocumentModelBuilder { private final DocumentModel model; - public DocumentModelBuilder(DocumentModel model) { - this.model = model; - model.getDocumentManager().add(VespaDocumentType.INSTANCE); + public DocumentModelBuilder() { + this.model = new DocumentModel(); + this.model.getDocumentManager().add(VespaDocumentType.INSTANCE); } - public void addToModel(Collection<Schema> schemaList) { + public DocumentModel build(Collection<Schema> schemaList) { List<SDDocumentType> docList = new LinkedList<>(); for (Schema schema : schemaList) { docList.add(schema.getDocument()); @@ -65,6 +64,7 @@ public class DocumentModelBuilder { toAdd = tryAdd(schemaList)) { schemaList = toAdd; } + return model; } private List<SDDocumentType> sortDocumentTypes(List<SDDocumentType> docList) { @@ -124,7 +124,7 @@ public class DocumentModelBuilder { return left; } - public void addToModel(Schema schema) { + private void addToModel(Schema schema) { // Then we add the search specific stuff SearchDef searchDef = new SearchDef(schema.getName()); addSearchFields(schema.extraFieldList(), searchDef); @@ -232,8 +232,7 @@ public class DocumentModelBuilder { @SuppressWarnings("deprecation") private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo, Collection<NewDocumentType> docs, - Set<TypeReplacement> replacements) - { + Set<TypeReplacement> replacements) { DataType original = type; if (type instanceof TemporaryStructuredDataType) { DataType other = repo.getDataType(type.getId()); @@ -298,8 +297,7 @@ public class DocumentModelBuilder { } private static DataType specialHandleAnnotationReferenceRecurse(NewDocumentType docType, String fieldName, - DataType dataType) - { + DataType dataType) { if (dataType instanceof TemporaryAnnotationReferenceDataType) { TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; if (refType.getId() != 0) { @@ -359,6 +357,7 @@ public class DocumentModelBuilder { addType(dt, s); return s; } + private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) { if (sa.getInherits() != null) { AnnotationType tmp = sdoc.findAnnotation(sa.getInherits()); @@ -367,7 +366,7 @@ public class DocumentModelBuilder { } return false; } - @SuppressWarnings("deprecation") + private NewDocumentType convert(SDDocumentType sdoc) { Map<AnnotationType, String> annotationInheritance = new HashMap<>(); Map<StructDataType, String> structInheritance = new HashMap<>(); @@ -439,17 +438,17 @@ public class DocumentModelBuilder { private static Set<NewDocumentType.Name> convertDocumentReferencesToNames(Optional<DocumentReferences> documentReferences) { if (!documentReferences.isPresent()) { - return emptySet(); + return Set.of(); } return documentReferences.get().referenceMap().values().stream() .map(documentReference -> documentReference.targetSearch().getDocument()) .map(documentType -> new NewDocumentType.Name(documentType.getName())) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { if (importedFields == null) { - return emptySet(); + return Set.of(); } return Collections.unmodifiableSet(importedFields.fields().keySet()); } @@ -463,6 +462,7 @@ public class DocumentModelBuilder { } } } + private static void extractNestedTypes(NewDocumentType dt, DataType type) { if (type instanceof StructDataType) { StructDataType tmp = (StructDataType) type; @@ -484,8 +484,11 @@ public class DocumentModelBuilder { throw new IllegalArgumentException(type.toString()); } } + private static boolean testAddType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, true); } + private static boolean addType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, false); } + private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) { DataType oldType = dt.getDataTypeRecursive(type.getId()); if (oldType == null) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java index c672b662874..1d71a9f1494 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; @@ -14,8 +15,11 @@ import com.yahoo.searchdefinition.document.SDDocumentType; */ public class DocumentOnlySchema extends Schema { - public DocumentOnlySchema(Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - super(application, fileRegistry, deployLogger, properties); + public DocumentOnlySchema(ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties) { + super(applicationPackage, fileRegistry, deployLogger, properties); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index 793fcc59f9d..14f07f224ab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -44,7 +44,7 @@ public class Index implements Cloneable, Serializable { private boolean prefix; /** The list of aliases (Strings) to this index name */ - private Set<String> aliases=new java.util.LinkedHashSet<>(1); + private Set<String> aliases = new java.util.LinkedHashSet<>(1); /** * The stemming setting of this field, or null to use the default. @@ -138,19 +138,18 @@ public class Index implements Cloneable, Serializable { } public String toString() { - String rankTypeName=rankType==null ? "(none)" : rankType.name(); - return - "index '" + name + - "' [ranktype: " + rankTypeName + - ", prefix: " + prefix + "]"; + String rankTypeName = rankType == null ? "(none)" : rankType.name(); + return "index '" + name + + "' [ranktype: " + rankTypeName + + ", prefix: " + prefix + "]"; } /** Makes a deep copy of this index */ @Override public Object clone() { try { - Index copy=(Index)super.clone(); - copy.aliases=new LinkedHashSet<>(this.aliases); + Index copy = (Index)super.clone(); + copy.aliases = new LinkedHashSet<>(this.aliases); return copy; } catch (CloneNotSupportedException e) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index fef7ff56763..08475813317 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -104,7 +104,6 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement @Override public TensorType getType(Reference reference) { // computeIfAbsent without concurrent modification due to resolve adding more resolved entries: - boolean canBeResolvedGlobally = referenceCanBeResolvedGlobally(reference); TensorType resolvedType = resolvedTypes.get(reference); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java index d40fe975721..4b849af9662 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java @@ -18,8 +18,8 @@ import java.util.Optional; public class OnnxModel extends DistributableResource { private OnnxModelInfo modelInfo = null; - private Map<String, String> inputMap = new HashMap<>(); - private Map<String, String> outputMap = new HashMap<>(); + private final Map<String, String> inputMap = new HashMap<>(); + private final Map<String, String> outputMap = new HashMap<>(); private String statelessExecutionMode = null; private Integer statelessInterOpThreads = null; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index d484d32b02f..49c52b21907 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -43,6 +43,7 @@ import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; import java.util.Set; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Level; import java.util.stream.Collectors; @@ -57,15 +58,17 @@ public class RankProfile implements Cloneable { public final static String FIRST_PHASE = "firstphase"; public final static String SECOND_PHASE = "secondphase"; + /** The search definition-unique name of this rank profile */ private final String name; - /** The search definition owning this profile, or null if global (owned by a model) */ - private final ImmutableSchema search; + /** The schema owning this profile, or null if global (owned by a model) */ + private final ImmutableSchema schema; /** The name of the rank profile inherited by this */ - private String inheritedName = null; - private RankProfile inherited = null; + private final List<String> inheritedNames = new ArrayList<>(); + /** Stores the resolved inherited profiles, or null when not resolved. */ + private List<RankProfile> inherited; /** The match settings of this profile */ private MatchPhaseSettings matchPhaseSettings = null; @@ -95,10 +98,10 @@ public class RankProfile implements Cloneable { private double rankScoreDropLimit = -Double.MAX_VALUE; private Set<ReferenceNode> summaryFeatures; - private String inheritedSummaryFeatures; + private String inheritedSummaryFeaturesProfileName; private Set<ReferenceNode> matchFeatures; - private String inheritedMatchFeatures; + private String inheritedMatchFeaturesProfileName; private Set<ReferenceNode> rankFeatures; @@ -132,20 +135,6 @@ public class RankProfile implements Cloneable { private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; - private static class CachedFunctions { - private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; - private final ImmutableMap<String, ExpressionFunction> allExpressionFunctions; - CachedFunctions(Map<String, RankingExpressionFunction> functions) { - allRankingExpressionFunctions = functions; - ImmutableMap.Builder<String,ExpressionFunction> mapBuilder = new ImmutableMap.Builder<>(); - for (var entry : functions.entrySet()) { - ExpressionFunction function = entry.getValue().function(); - mapBuilder.put(function.getName(), function); - } - allExpressionFunctions = mapBuilder.build(); - } - } - /** * Creates a new rank profile for a particular search definition * @@ -156,7 +145,7 @@ public class RankProfile implements Cloneable { */ public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.search = Objects.requireNonNull(schema, "search cannot be null"); + this.schema = Objects.requireNonNull(schema, "search cannot be null"); this.onnxModels = null; this.rankingConstants = rankingConstants; this.rankProfileRegistry = rankProfileRegistry; @@ -172,7 +161,7 @@ public class RankProfile implements Cloneable { public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants, OnnxModels onnxModels) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.search = null; + this.schema = null; this.rankProfileRegistry = rankProfileRegistry; this.rankingConstants = rankingConstants; this.onnxModels = onnxModels; @@ -180,10 +169,10 @@ public class RankProfile implements Cloneable { this.deployLogger = deployLogger; } - public String getName() { return name; } + public String name() { return name; } /** Returns the search definition owning this, or null if it is global */ - public ImmutableSchema getSearch() { return search; } + public ImmutableSchema schema() { return schema; } /** Returns the application this is part of */ public ApplicationPackage applicationPackage() { @@ -196,69 +185,76 @@ public class RankProfile implements Cloneable { } public Map<String, OnnxModel> onnxModels() { - return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); + return schema != null ? schema.onnxModels().asMap() : onnxModels.asMap(); } private Stream<ImmutableSDField> allFields() { - if (search == null) return Stream.empty(); + if (schema == null) return Stream.empty(); if (allFieldsList == null) { - allFieldsList = search.allFieldsList(); + allFieldsList = schema.allFieldsList(); } return allFieldsList.stream(); } private Stream<ImmutableSDField> allImportedFields() { - return search != null ? search.allImportedFields() : Stream.empty(); + return schema != null ? schema.allImportedFields() : Stream.empty(); } /** - * Sets the name of the rank profile this inherits. Both rank profiles must be present in the same search - * definition + * Adds a profile to those inherited by this. + * The profile must belong to this schema (directly or by inheritance). */ - public void setInherited(String inheritedName) { - this.inheritedName = inheritedName; - } - - /** Returns the name of the profile this one inherits, or null if none is inherited */ - public String getInheritedName() { return inheritedName; } - - /** Returns the inherited rank profile, or null if there is none */ - private RankProfile getInherited() { - if (inheritedName == null) return null; - if (inherited == null) { - inherited = resolveInherited(); - if (inherited == null) { - String msg = "rank-profile '" + getName() + "' inherits '" + inheritedName + - "', but it does not exist anywhere in the inheritance of search '" + - ((getSearch() != null) ? getSearch().getName() : " global rank profiles") + "'."; - throw new IllegalArgumentException(msg); - } else { - List<String> children = new ArrayList<>(); - children.add(createFullyQualifiedName()); - verifyNoInheritanceCycle(children, inherited); - } - } + public void inherit(String inheritedName) { + inherited = null; + inheritedNames.add(inheritedName); + } + + /** Returns the names of the profiles this inherits, if any. */ + public List<String> inheritedNames() { return Collections.unmodifiableList(inheritedNames); } + + /** Returns the rank profiles inherited by this. */ + private List<RankProfile> inherited() { + if (inheritedNames.isEmpty()) return List.of(); + if (inherited != null) return inherited; + + inherited = resolveInheritedProfiles(schema); + List<String> children = new ArrayList<>(); + children.add(createFullyQualifiedName()); + inherited.forEach(profile -> verifyNoInheritanceCycle(children, profile)); return inherited; } private String createFullyQualifiedName() { - return (search != null) - ? (search.getName() + "." + getName()) - : getName(); + return (schema != null) + ? (schema.getName() + "." + name()) + : name(); } private void verifyNoInheritanceCycle(List<String> children, RankProfile parent) { children.add(parent.createFullyQualifiedName()); String root = children.get(0); - if (root.equals(parent.createFullyQualifiedName())) { + if (root.equals(parent.createFullyQualifiedName())) throw new IllegalArgumentException("There is a cycle in the inheritance for rank-profile '" + root + "' = " + children); + for (RankProfile parentInherited : parent.inherited()) + verifyNoInheritanceCycle(children, parentInherited); + } + + private List<RankProfile> resolveInheritedProfiles(ImmutableSchema schema) { + List<RankProfile> inherited = new ArrayList<>(); + for (String inheritedName : inheritedNames) { + RankProfile inheritedProfile = schema == null + ? rankProfileRegistry.getGlobal(inheritedName) + : resolveInheritedProfile(schema, inheritedName); + if (inheritedProfile == null) + throw new IllegalArgumentException("rank-profile '" + name() + "' inherits '" + inheritedName + + "', but this is not found in " + + ((schema() != null) ? schema() : " global rank profiles")); + inherited.add(inheritedProfile); } - if (parent.getInherited() != null) { - verifyNoInheritanceCycle(children, parent.getInherited()); - } + return inherited; } - private RankProfile resolveInherited(ImmutableSchema schema) { + private RankProfile resolveInheritedProfile(ImmutableSchema schema, String inheritedName) { SDDocumentType documentType = schema.getDocument(); if (documentType != null) { if (name.equals(inheritedName)) { @@ -273,25 +269,11 @@ public class RankProfile implements Cloneable { return rankProfileRegistry.get(schema.getName(), inheritedName); } - private RankProfile resolveInherited() { - if (inheritedName == null) return null; - return (getSearch() != null) - ? resolveInherited(search) - : rankProfileRegistry.getGlobal(inheritedName); - } - - /** - * Returns whether this profile inherits (directly or indirectly) the given profile - * - * @param name the profile name to compare this to. - * @return whether or not this inherits from the named profile. - */ + /** Returns whether this profile inherits (directly or indirectly) the given profile name. */ public boolean inherits(String name) { - RankProfile parent = getInherited(); - while (parent != null) { - if (parent.getName().equals(name)) - return true; - parent = parent.getInherited(); + for (RankProfile inheritedProfile : inherited()) { + if (inheritedProfile.name().equals(name)) return true; + if (inheritedProfile.inherits(name)) return true; } return false; } @@ -302,10 +284,20 @@ public class RankProfile implements Cloneable { } public MatchPhaseSettings getMatchPhaseSettings() { - MatchPhaseSettings settings = this.matchPhaseSettings; - if (settings != null) return settings; - if (getInherited() != null) return getInherited().getMatchPhaseSettings(); - return null; + if (matchPhaseSettings != null) return matchPhaseSettings; + return inheritedWith(p -> p.getMatchPhaseSettings() != null,"match phase settings") + .map(p -> p.getMatchPhaseSettings()).orElse(null); + } + + /** Returns the single profile having the property checked by the given filter, or empty if none */ + private Optional<RankProfile> inheritedWith(Predicate<RankProfile> property, String propertyDescription) { + List<RankProfile> matchingInherited = + inherited().stream().filter(profile -> property.test(profile)).collect(Collectors.toList()); + if (matchingInherited.isEmpty()) return Optional.empty(); + if (matchingInherited.size() == 1) return Optional.of(matchingInherited.get(0)); + throw new IllegalArgumentException("Only one of the profiles inherited by " + this + " can contain " + + propertyDescription + ", but it is present in all of " + matchingInherited); + } public void addRankSetting(RankSetting rankSetting) { @@ -319,15 +311,14 @@ public class RankProfile implements Cloneable { /** * Returns the a rank setting of a field, or null if there is no such rank setting in this profile * - * @param field the field whose settings to return. - * @param type the type that the field is required to be. - * @return the rank setting found, or null. + * @param field the field whose settings to return + * @param type the type that the field is required to be + * @return the rank setting found, or null */ RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext(); ) { RankSetting setting = i.next(); - if (setting.getFieldName().equals(field) && - setting.getType().equals(type)) { + if (setting.getFieldName().equals(field) && setting.getType() == type) { return setting; } } @@ -346,9 +337,8 @@ public class RankProfile implements Cloneable { RankSetting rankSetting = getDeclaredRankSetting(field, type); if (rankSetting != null) return rankSetting; - if (getInherited() != null) return getInherited().getRankSetting(field, type); - - return null; + return inheritedWith(p -> p.getRankSetting(field, type) != null, "rank setting " + type + " on " + field) + .map(p -> p.getRankSetting(field, type)).orElse(null); } /** @@ -374,12 +364,20 @@ public class RankProfile implements Cloneable { * Changes to the returned set will not be reflected in this rank profile. */ public Set<RankSetting> rankSettings() { - Set<RankSetting> allSettings = new LinkedHashSet<>(rankSettings); - RankProfile parent = getInherited(); - if (parent != null) - allSettings.addAll(parent.rankSettings()); + Set<RankSetting> settings = new LinkedHashSet<>(); + for (RankProfile inheritedProfile : inherited()) { + for (RankSetting setting : inheritedProfile.rankSettings()) { + if (settings.contains(setting)) + throw new IllegalArgumentException(setting + " is present in " + inheritedProfile + " inherited by " + + this + ", but is also present in another profile inherited by it"); + settings.add(setting); + } + } - return allSettings; + // TODO: Here we do things in the wrong order to not break tests. Reverse this. + Set<RankSetting> finalSettings = new LinkedHashSet<>(rankSettings); + finalSettings.addAll(settings); + return finalSettings; } public void addConstant(String name, Value value) { @@ -398,14 +396,20 @@ public class RankProfile implements Cloneable { /** Returns an unmodifiable view of the constants available in this */ public Map<String, Value> getConstants() { - if (constants.isEmpty()) - return getInherited() != null ? getInherited().getConstants() : Collections.emptyMap(); - if (getInherited() == null || getInherited().getConstants().isEmpty()) - return Collections.unmodifiableMap(constants); - - Map<String, Value> combinedConstants = new HashMap<>(getInherited().getConstants()); - combinedConstants.putAll(constants); - return combinedConstants; + if (inherited().isEmpty()) return new HashMap<>(constants); + + Map<String, Value> allConstants = new HashMap<>(); + for (var inheritedProfile : inherited()) { + for (var constant : inheritedProfile.getConstants().entrySet()) { + if (allConstants.containsKey(constant.getKey())) + throw new IllegalArgumentException("Constant '" + constant.getKey() + "' is present in " + + inheritedProfile + " inherited by " + + this + ", but is also present in another profile inherited by it"); + allConstants.put(constant.getKey(), constant.getValue()); + } + } + allConstants.putAll(constants); + return allConstants; } public void addAttributeType(String attributeName, String attributeType) { @@ -436,9 +440,8 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getFirstPhase() { if (firstPhaseRanking != null) return firstPhaseRanking; - RankProfile inherited = getInherited(); - if (inherited != null) return inherited.getFirstPhase(); - return null; + return inheritedWith(p -> p.getFirstPhase() != null, "first-phase expression") + .map(p -> p.getFirstPhase()).orElse(null); } void setFirstPhaseRanking(RankingExpression rankingExpression) { @@ -465,9 +468,8 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - RankProfile inherited = getInherited(); - if (inherited != null) return inherited.getSecondPhase(); - return null; + return inheritedWith(p -> p.getSecondPhase() != null, "second-phase expression") + .map(p -> p.getSecondPhase()).orElse(null); } public void setSecondPhaseRanking(String expression) { @@ -479,75 +481,80 @@ public class RankProfile implements Cloneable { } } - /** Returns a read-only view of the summary features to use in this profile. This is never null */ - public Set<ReferenceNode> getSummaryFeatures() { - if (inheritedSummaryFeatures != null && summaryFeatures != null) { - Set<ReferenceNode> combined = new HashSet<>(); - combined.addAll(getInherited().getSummaryFeatures()); - combined.addAll(summaryFeatures); - return Collections.unmodifiableSet(combined); - } - if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); - if (getInherited() != null) return getInherited().getSummaryFeatures(); - return Set.of(); - } - - private void addSummaryFeature(ReferenceNode feature) { - if (summaryFeatures == null) - summaryFeatures = new LinkedHashSet<>(); - summaryFeatures.add(feature); - } - - /** Adds the content of the given feature list to the internal list of summary features. */ - public void addSummaryFeatures(FeatureList features) { - for (ReferenceNode feature : features) { - addSummaryFeature(feature); - } - } + // TODO: Below we have duplicate methods for summary and match features: Encapsulate this in a single parametrized + // class instead (and probably make rank features work the same). /** * Sets the name this should inherit the summary features of. - * Without setting this, this will either have the summary features of the parent, + * Without setting this, this will either have the summary features of the single parent setting them, * or if summary features are set in this, only have the summary features in this. * With this set the resulting summary features of this will be the superset of those defined in this and * the final (with inheritance included) summary features of the given parent. - * The profile must be the profile which is directly inherited by this. - * + * The profile must be one which is directly inherited by this. */ public void setInheritedSummaryFeatures(String parentProfile) { - if ( ! parentProfile.equals(inheritedName)) - throw new IllegalArgumentException("This can only inherit the summary features of its parent, '" + - inheritedName + ", but attempting to inherit '" + parentProfile); - this.inheritedSummaryFeatures = parentProfile; + if ( ! inheritedNames().contains(parentProfile)) + throw new IllegalArgumentException("This can only inherit the summary features of a directly inherited profile, '" + + ", but attempting to inherit '" + parentProfile); + this.inheritedSummaryFeaturesProfileName = parentProfile; } /** * Sets the name of a profile this should inherit the match features of. - * Without setting this, this will either have the match features of the parent, + * Without setting this, this will either have the match features of the single parent setting them, * or if match features are set in this, only have the match features in this. * With this set the resulting match features of this will be the superset of those defined in this and * the final (with inheritance included) match features of the given parent. - * The profile must be the profile which is directly inherited by this. + * The profile must be one which which is directly inherited by this. * */ public void setInheritedMatchFeatures(String parentProfile) { - if ( ! parentProfile.equals(inheritedName)) - throw new IllegalArgumentException("This rank profile ("+name+") can only inherit the match features of its parent, '" + - inheritedName + ", but attemtping to inherit '" + parentProfile); - this.inheritedMatchFeatures = parentProfile; + if ( ! inheritedNames().contains(parentProfile)) + throw new IllegalArgumentException("This can only inherit the match features of a directly inherited profile, '" + + ", but attempting to inherit '" + parentProfile); + this.inheritedMatchFeaturesProfileName = parentProfile; + } + + /** Returns a read-only view of the summary features to use in this profile. This is never null */ + public Set<ReferenceNode> getSummaryFeatures() { + if (inheritedSummaryFeaturesProfileName != null && summaryFeatures != null) { + Set<ReferenceNode> combined = new HashSet<>(); + RankProfile inherited = inherited().stream() + .filter(p -> p.name().equals(inheritedSummaryFeaturesProfileName)) + .findAny() + .orElseThrow(); + combined.addAll(inherited.getSummaryFeatures()); + combined.addAll(summaryFeatures); + return Collections.unmodifiableSet(combined); + } + if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); + return inheritedWith(p -> ! p.getSummaryFeatures().isEmpty(), "summary features") + .map(p -> p.getSummaryFeatures()) + .orElse(Set.of()); } /** Returns a read-only view of the match features to use in this profile. This is never null */ public Set<ReferenceNode> getMatchFeatures() { - if (inheritedMatchFeatures != null && matchFeatures != null) { + if (inheritedMatchFeaturesProfileName != null && matchFeatures != null) { Set<ReferenceNode> combined = new HashSet<>(); - combined.addAll(getInherited().getMatchFeatures()); + RankProfile inherited = inherited().stream() + .filter(p -> p.name().equals(inheritedMatchFeaturesProfileName)) + .findAny() + .orElseThrow(); + combined.addAll(inherited.getMatchFeatures()); combined.addAll(matchFeatures); return Collections.unmodifiableSet(combined); } if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); - if (getInherited() != null) return getInherited().getMatchFeatures(); - return Set.of(); + return inheritedWith(p -> ! p.getMatchFeatures().isEmpty(), "match features") + .map(p -> p.getMatchFeatures()) + .orElse(Set.of()); + } + + private void addSummaryFeature(ReferenceNode feature) { + if (summaryFeatures == null) + summaryFeatures = new LinkedHashSet<>(); + summaryFeatures.add(feature); } private void addMatchFeature(ReferenceNode feature) { @@ -556,6 +563,13 @@ public class RankProfile implements Cloneable { matchFeatures.add(feature); } + /** Adds the content of the given feature list to the internal list of summary features. */ + public void addSummaryFeatures(FeatureList features) { + for (ReferenceNode feature : features) { + addSummaryFeature(feature); + } + } + /** Adds the content of the given feature list to the internal list of match features. */ public void addMatchFeatures(FeatureList features) { for (ReferenceNode feature : features) { @@ -566,8 +580,8 @@ public class RankProfile implements Cloneable { /** Returns a read-only view of the rank features to use in this profile. This is never null */ public Set<ReferenceNode> getRankFeatures() { if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); - if (getInherited() != null) return getInherited().getRankFeatures(); - return Collections.emptySet(); + return inheritedWith(p -> ! p.getRankFeatures().isEmpty(), "summary-features") + .map(p -> p.getRankFeatures()).orElse(Set.of()); } private void addRankFeature(ReferenceNode feature) { @@ -598,12 +612,15 @@ public class RankProfile implements Cloneable { /** Returns a read only map view of the rank properties to use in this profile. This is never null. */ public Map<String, List<RankProperty>> getRankPropertyMap() { - if (rankProperties.size() == 0 && getInherited() == null) return Collections.emptyMap(); - if (rankProperties.size() == 0) return getInherited().getRankPropertyMap(); - if (getInherited() == null) return Collections.unmodifiableMap(rankProperties); + if (rankProperties.size() == 0 && inherited().isEmpty()) return Map.of(); + if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); + + var inheritedProperties = inheritedWith(p -> ! p.getRankPropertyMap().isEmpty(), "rank-properties") + .map(p -> p.getRankPropertyMap()).orElse(Map.of()); + if (rankProperties.isEmpty()) return inheritedProperties; // Neither is null - Map<String, List<RankProperty>> combined = new LinkedHashMap<>(getInherited().getRankPropertyMap()); + Map<String, List<RankProperty>> combined = new LinkedHashMap<>(inheritedProperties); combined.putAll(rankProperties); // Don't combine values across inherited properties return Collections.unmodifiableMap(combined); } @@ -617,57 +634,44 @@ public class RankProfile implements Cloneable { rankProperties.computeIfAbsent(rankProperty.getName(), (String key) -> new ArrayList<>(1)).add(rankProperty); } - @Override - public String toString() { - return "rank profile '" + getName() + "'"; - } + public void setRerankCount(int rerankCount) { this.rerankCount = rerankCount; } public int getRerankCount() { - return (rerankCount < 0 && (getInherited() != null)) - ? getInherited().getRerankCount() - : rerankCount; + if (rerankCount >= 0) return rerankCount; + return inheritedWith(p -> p.getRerankCount() >= 0, "rerank-count") + .map(p -> p.getRerankCount()).orElse(-1); } + public void setNumThreadsPerSearch(int numThreads) { this.numThreadsPerSearch = numThreads; } + public int getNumThreadsPerSearch() { - return (numThreadsPerSearch < 0 && (getInherited() != null)) - ? getInherited().getNumThreadsPerSearch() - : numThreadsPerSearch; + if (numThreadsPerSearch >= 0) return numThreadsPerSearch; + return inheritedWith(p -> p.getNumThreadsPerSearch() >= 0, "num-threads-per-search") + .map(p -> p.getNumThreadsPerSearch()).orElse(-1); } - public void setNumThreadsPerSearch(int numThreads) { - this.numThreadsPerSearch = numThreads; - } + public void setMinHitsPerThread(int minHits) { this.minHitsPerThread = minHits; } public int getMinHitsPerThread() { - return (minHitsPerThread < 0 && (getInherited() != null)) - ? getInherited().getMinHitsPerThread() - : minHitsPerThread; - } - - public void setMinHitsPerThread(int minHits) { - this.minHitsPerThread = minHits; + if (minHitsPerThread >= 0) return minHitsPerThread; + return inheritedWith(p -> p.getMinHitsPerThread() >= 0, "min-hits-per-search") + .map(p -> p.getMinHitsPerThread()).orElse(-1); } - public void setNumSearchPartitions(int numSearchPartitions) { - this.numSearchPartitions = numSearchPartitions; - } + public void setNumSearchPartitions(int numSearchPartitions) { this.numSearchPartitions = numSearchPartitions; } public int getNumSearchPartitions() { - return (numSearchPartitions < 0 && (getInherited() != null)) - ? getInherited().getNumSearchPartitions() - : numSearchPartitions; + if (numSearchPartitions >= 0) return numSearchPartitions; + return inheritedWith(p -> p.getNumSearchPartitions() >= 0, "num-search-partitions") + .map(p -> p.getNumSearchPartitions()).orElse(-1); } - public OptionalDouble getTermwiseLimit() { - return ((termwiseLimit == null) && (getInherited() != null)) - ? getInherited().getTermwiseLimit() - : (termwiseLimit != null) ? OptionalDouble.of(termwiseLimit) : OptionalDouble.empty(); - } public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } - /** Sets the rerank count. Set to -1 to use inherited */ - public void setRerankCount(int rerankCount) { - this.rerankCount = rerankCount; + public OptionalDouble getTermwiseLimit() { + if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); + return inheritedWith(p -> p.getTermwiseLimit().isPresent(), "termwise-limit") + .map(p -> p.getTermwiseLimit()).orElse(OptionalDouble.empty()); } /** Whether we should ignore the default rank features. Set to null to use inherited */ @@ -677,10 +681,26 @@ public class RankProfile implements Cloneable { public boolean getIgnoreDefaultRankFeatures() { if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; - return (getInherited() != null) && getInherited().getIgnoreDefaultRankFeatures(); + return inheritedWith(p -> p.ignoreDefaultRankFeatures != null, "ignore-default-rank-features") + .map(p -> p.getIgnoreDefaultRankFeatures()).orElse(false); + } + + public void setKeepRankCount(int rerankArraySize) { this.keepRankCount = rerankArraySize; } + + public int getKeepRankCount() { + if (keepRankCount >= 0) return keepRankCount; + return inheritedWith(p -> p.getKeepRankCount() >= 0, "keep-rank-count") + .map(p -> p.getKeepRankCount()).orElse(-1); + } + + public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } + + public double getRankScoreDropLimit() { + if (rankScoreDropLimit > -Double.MAX_VALUE) return rankScoreDropLimit; + return inheritedWith(p -> p.getRankScoreDropLimit() > -Double.MAX_VALUE, "rank.score-drop-limit") + .map(p -> p.getRankScoreDropLimit()).orElse(rankScoreDropLimit); } - /** Adds a function */ public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { addFunction(parseRankingExpression(name, arguments, expression), inline); @@ -742,10 +762,11 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction findFunction(String name) { RankingExpressionFunction function = functions.get(name); - return ((function == null) && (getInherited() != null)) - ? getInherited().findFunction(name) - : function; + if (function != null) return function; + return inheritedWith(p -> p.findFunction(name) != null, "function '" + name + "'") + .map(p -> p.findFunction(name)).orElse(null); } + /** Returns an unmodifiable snapshot of the functions in this */ public Map<String, RankingExpressionFunction> getFunctions() { updateCachedFunctions(); @@ -762,75 +783,58 @@ public class RankProfile implements Cloneable { } private Map<String, RankingExpressionFunction> gatherAllFunctions() { - if (functions.isEmpty() && getInherited() == null) return Collections.emptyMap(); - if (functions.isEmpty()) return getInherited().getFunctions(); - if (getInherited() == null) return Collections.unmodifiableMap(new LinkedHashMap<>(functions)); - - // Neither is null - Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(getInherited().getFunctions()); + if (functions.isEmpty() && inherited().isEmpty()) return Map.of(); + if (inherited().isEmpty()) return Collections.unmodifiableMap(new LinkedHashMap<>(functions)); + + // Combine + Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(); + for (var inheritedProfile : inherited()) { + for (var function : inheritedProfile.getFunctions().entrySet()) { + if (allFunctions.containsKey(function.getKey())) + throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + + function.getValue() + ", but this function is already " + + "defined in another profile this inherits"); + allFunctions.put(function.getKey(), function.getValue()); + } + } allFunctions.putAll(functions); return Collections.unmodifiableMap(allFunctions); } private boolean needToUpdateFunctionCache() { - if (getInherited() != null) - return (allFunctionsCached == null) || getInherited().needToUpdateFunctionCache(); + if (inherited().stream().anyMatch(profile -> profile.needToUpdateFunctionCache())) return true; return allFunctionsCached == null; } - public int getKeepRankCount() { - if (keepRankCount >= 0) return keepRankCount; - if (getInherited() != null) return getInherited().getKeepRankCount(); - return -1; - } - - public void setKeepRankCount(int rerankArraySize) { - this.keepRankCount = rerankArraySize; - } + public Set<String> filterFields() { return filterFields; } - public double getRankScoreDropLimit() { - if (rankScoreDropLimit >- Double.MAX_VALUE) return rankScoreDropLimit; - if (getInherited() != null) return getInherited().getRankScoreDropLimit(); - return rankScoreDropLimit; - } + /** Returns all filter fields in this profile and any profile it inherits. */ + public Set<String> allFilterFields() { + Set<String> inheritedFilterFields = + inheritedWith(p -> ! p.allFilterFields().isEmpty(), "filter fields") + .map(p -> p.allFilterFields()).orElse(Set.of()); - public void setRankScoreDropLimit(double rankScoreDropLimit) { - this.rankScoreDropLimit = rankScoreDropLimit; - } + if (inheritedFilterFields.isEmpty()) return Collections.unmodifiableSet(filterFields); - public Set<String> filterFields() { - return filterFields; - } - - /** - * Returns all filter fields in this profile and any profile it inherits. - * - * @return the set of all filter fields - */ - public Set<String> allFilterFields() { - RankProfile parent = getInherited(); - Set<String> retval = new LinkedHashSet<>(); - if (parent != null) { - retval.addAll(parent.allFilterFields()); - } - retval.addAll(filterFields()); - return retval; + Set<String> combined = new LinkedHashSet<>(inheritedFilterFields); + combined.addAll(filterFields()); + return combined; } private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { if (expression.trim().length() == 0) - throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + name + "."); + throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + - "' in " + getName()+ ", " + name + "."); + "' in " + name() + ", " + name + "."); throw (ParseException)exception.initCause(e); } catch (IOException e) { - throw new RuntimeException("IOException parsing ranking expression '" + name + "'"); + throw new RuntimeException("IOException parsing ranking expression '" + name + "'", e); } } @@ -848,10 +852,10 @@ public class RankProfile implements Cloneable { String fileName = extractFileName(expression); File file = new File(fileName); if (!file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122 - throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file + - "' in subdirectory, which is not supported."); + throw new IllegalArgumentException("In " + name() + ", " + expName + ", ranking references file '" + file + + "' in subdirectory, which is not supported."); - return search.getRankingExpression(fileName); + return schema.getRankingExpression(fileName); } /** Shallow clones this */ @@ -888,7 +892,7 @@ public class RankProfile implements Cloneable { return compiled; } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Rank profile '" + getName() + "' is invalid", e); + throw new IllegalArgumentException("Rank profile '" + name() + "' is invalid", e); } } @@ -905,7 +909,7 @@ public class RankProfile implements Cloneable { secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); // Function compiling second pass: compile all functions and insert previously compiled inline functions - // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. + // TODO: This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); allFunctionsCached = null; } @@ -960,6 +964,7 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { if (function == null) return null; + RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, featureTypes, @@ -993,7 +998,8 @@ public class RankProfile implements Cloneable { return featureTypes; } - public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles, Map<Reference, TensorType> featureTypes) { + public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles, + Map<Reference, TensorType> featureTypes) { MapEvaluationTypeContext context = new MapEvaluationTypeContext(getExpressionFunctions(), featureTypes); // Add small and large constants, respectively @@ -1076,6 +1082,11 @@ public class RankProfile implements Cloneable { }); } + @Override + public String toString() { + return "rank profile '" + name() + "'"; + } + /** * A rank setting. The identity of a rank setting is its field name and type (not value). * A rank setting is immutable. @@ -1118,8 +1129,9 @@ public class RankProfile implements Cloneable { return name; } + @Override public String toString() { - return "type: " + name; + return "type " + name; } } @@ -1232,7 +1244,7 @@ public class RankProfile implements Cloneable { @Override public String toString() { - return "function " + function; + return function.toString(); } } @@ -1321,4 +1333,22 @@ public class RankProfile implements Cloneable { } + private static class CachedFunctions { + + private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; + + private final ImmutableMap<String, ExpressionFunction> allExpressionFunctions; + + CachedFunctions(Map<String, RankingExpressionFunction> functions) { + allRankingExpressionFunctions = functions; + ImmutableMap.Builder<String,ExpressionFunction> mapBuilder = new ImmutableMap.Builder<>(); + for (var entry : functions.entrySet()) { + ExpressionFunction function = entry.getValue().function(); + mapBuilder.put(function.getName(), function); + } + allExpressionFunctions = mapBuilder.build(); + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java index 08ae3d838ec..75c3aa313c0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java @@ -6,7 +6,6 @@ import com.yahoo.searchdefinition.document.SDDocumentType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -44,22 +43,22 @@ public class RankProfileRegistry { /** Adds a rank profile to this registry */ public void add(RankProfile rankProfile) { - String searchName = extractName(rankProfile.getSearch()); - if ( ! rankProfiles.containsKey(searchName)) { - rankProfiles.put(searchName, new LinkedHashMap<>()); + String schemaName = extractName(rankProfile.schema()); + if ( ! rankProfiles.containsKey(schemaName)) { + rankProfiles.put(schemaName, new LinkedHashMap<>()); } checkForDuplicate(rankProfile); - rankProfiles.get(searchName).put(rankProfile.getName(), rankProfile); + rankProfiles.get(schemaName).put(rankProfile.name(), rankProfile); } private void checkForDuplicate(RankProfile rankProfile) { - String rankProfileName = rankProfile.getName(); - RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.getSearch())).get(rankProfileName); + String rankProfileName = rankProfile.name(); + RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.schema())).get(rankProfileName); if (existingRankProfileWithSameName == null) return; if ( ! overridableRankProfileNames.contains(rankProfileName)) { throw new IllegalArgumentException("Duplicate rank profile '" + rankProfileName + "' in " + - rankProfile.getSearch()); + rankProfile.schema()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java index aa43c00f461..ba34045e7de 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java @@ -76,12 +76,12 @@ public class SDDocumentTypeOrderer { SDDocumentType inherited; if (type.isStruct()) { inherited = owningDocument.allTypes().get(new NewDocumentType.Name(name.getName())); - if (inherited == null) throw new IllegalStateException("Struct '" + name + "' not found in " + owningDocument); + if (inherited == null) throw new IllegalArgumentException("Struct '" + name + "' not found in " + owningDocument); process(inherited, owningDocument); } else { inherited = createdSDTypes.get(name); - if (inherited == null) throw new IllegalStateException("Document type '" + name + "' not found"); + if (inherited == null) throw new IllegalArgumentException("Document type '" + name + "' not found"); process(inherited, inherited); } type.inherit(inherited); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java index ddad67324ba..12b0394b090 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java @@ -55,6 +55,9 @@ public class Schema implements ImmutableSchema { /** The unique name of this schema */ private String name; + /** The application package this is constructed from */ + private final ApplicationPackage applicationPackage; + /** The name of the schema this should inherit all the content of, if any */ private final Optional<String> inherited; @@ -93,21 +96,22 @@ public class Schema implements ImmutableSchema { /** The resulting processed field */ private Optional<ImportedFields> importedFields = Optional.empty(); - private final Application owner; private final DeployLogger deployLogger; private final ModelContext.Properties properties; + private Application owner; + /** Testing only */ - public Schema(String name) { - this(name, Optional.empty(), null, null, new BaseDeployLogger(), new TestProperties()); + public Schema(String name, ApplicationPackage applicationPackage) { + this(name, applicationPackage, Optional.empty(), null, new BaseDeployLogger(), new TestProperties()); } public Schema(String name, - Application application, + ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(name, Optional.empty(), application, fileRegistry, deployLogger, properties); + this(name, applicationPackage, Optional.empty(), fileRegistry, deployLogger, properties); } /** @@ -115,30 +119,30 @@ public class Schema implements ImmutableSchema { * * @param name of the schema * @param inherited the schema this inherits, if any - * @param application the application containing this */ public Schema(String name, + ApplicationPackage applicationPackage, Optional<String> inherited, - Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(inherited, application, fileRegistry, deployLogger, properties, false); - this.name = name; + this(inherited, applicationPackage, fileRegistry, deployLogger, properties, false); + this.name = Objects.requireNonNull(name, "A schema must have a name"); } - protected Schema(Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(Optional.empty(), application, fileRegistry, deployLogger, properties, true); + protected Schema(ApplicationPackage applicationPackage, FileRegistry fileRegistry, + DeployLogger deployLogger, ModelContext.Properties properties) { + this(Optional.empty(), applicationPackage, fileRegistry, deployLogger, properties, true); } private Schema(Optional<String> inherited, - Application application, + ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties, boolean documentsOnly) { this.inherited = inherited; - this.owner = application; + this.applicationPackage = applicationPackage; this.deployLogger = deployLogger; this.properties = properties; this.documentsOnly = documentsOnly; @@ -147,14 +151,21 @@ public class Schema implements ImmutableSchema { onnxModels = new OnnxModels(fileRegistry, Optional.of(this)); } - protected void setName(String name) { - this.name = name; + /** + * Assigns the owner of this + * + * @throws IllegalStateException if an owner is already assigned + */ + public void setOwner(Application owner) { + if (this.owner != null) + throw new IllegalStateException("Cannot reassign the owner of " + this); + this.owner = owner; } + protected void setName(String name) { this.name = name; } + @Override - public String getName() { - return name; - } + public String getName() {return name; } /** Returns true if this only defines a document type, not a full schema */ public boolean isDocumentsOnly() { @@ -174,11 +185,12 @@ public class Schema implements ImmutableSchema { */ public boolean isRawAsBase64() { if (rawAsBase64 != null) return rawAsBase64; + // TODO Vespa 8: flip default: if (inherited.isEmpty()) return false; return requireInherited().isRawAsBase64(); } - public void enableRawAsBase64() { rawAsBase64 = true; } + public void enableRawAsBase64(boolean value) { rawAsBase64 = value; } /** * Sets the stemming default of fields. Default is ALL @@ -311,16 +323,13 @@ public class Schema implements ImmutableSchema { */ @Override public Reader getRankingExpression(String fileName) { - return owner.applicationPackage().getRankingExpression(fileName); + return applicationPackage.getRankingExpression(fileName); } public Application application() { return owner; } @Override - public ApplicationPackage applicationPackage() { - if (owner == null) return null; - return owner.applicationPackage(); - } + public ApplicationPackage applicationPackage() { return applicationPackage; } @Override public DeployLogger getDeployLogger() { return deployLogger; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 04ef85856cd..dad8385e0e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -179,6 +179,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce return "attributes"; } + @SuppressWarnings("removal") // TODO Vespa 8: remove private Map<String, AttributesConfig.Attribute.Builder> toMap(List<AttributesConfig.Attribute.Builder> ls) { Map<String, AttributesConfig.Attribute.Builder> ret = new LinkedHashMap<>(); for (AttributesConfig.Attribute.Builder builder : ls) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java index 30cb236984d..3a8268029d0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java @@ -76,7 +76,7 @@ public abstract class Derived implements Exportable { protected abstract String getDerivedName(); /** Returns the value of getName if true, the given number as a string otherwise */ - protected String getIndex(int number,boolean labels) { + protected String getIndex(int number, boolean labels) { return labels ? getName() : String.valueOf(number); } @@ -87,12 +87,12 @@ public abstract class Derived implements Exportable { * @param toDirectory the directory to export to, or null * */ - public final void export(String toDirectory) - throws IOException { - Writer writer=null; + public final void export(String toDirectory) throws IOException { + Writer writer = null; try { - String fileName=getDerivedName() + ".cfg"; - if (toDirectory!=null) writer=IOUtils.createWriter(toDirectory + "/" + fileName,false); + String fileName = getDerivedName() + ".cfg"; + if (toDirectory != null) + writer = IOUtils.createWriter(toDirectory + "/" + fileName,false); try { exportBuilderConfig(writer); } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException e) { @@ -100,7 +100,7 @@ public abstract class Derived implements Exportable { } } finally { - if (writer!=null) IOUtils.closeWriter(writer); + if (writer != null) IOUtils.closeWriter(writer); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java index 14e303522e0..bcdc9e656e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; @@ -11,22 +11,21 @@ import java.util.Collections; import java.util.List; /** - * Auxiliary facade for deriving configs from search definitions + * Facade for deriving configs from schemas * * @author bratseth */ public class Deriver { - public static SchemaBuilder getSearchBuilder(List<String> sds) { - SchemaBuilder builder = new SchemaBuilder(); + public static ApplicationBuilder getSchemaBuilder(List<String> schemas) { + ApplicationBuilder builder = new ApplicationBuilder(); try { - for (String s : sds) { - builder.importFile(s); - } + for (String schema : schemas) + builder.addSchemaFile(schema); } catch (ParseException | IOException e) { throw new IllegalArgumentException(e); } - builder.build(); + builder.build(true); return builder; } @@ -34,22 +33,22 @@ public class Deriver { return getDocumentManagerConfig(Collections.singletonList(sd)); } - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds) { - return new DocumentManager().produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> schemas) { + return new DocumentManager().produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); } - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds, boolean useV8DocManagerCfg) { + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> schemas, boolean useV8DocManagerCfg) { return new DocumentManager() .useV8DocManagerCfg(useV8DocManagerCfg) - .produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + .produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); } - public static DocumenttypesConfig.Builder getDocumentTypesConfig(String sd) { - return getDocumentTypesConfig(Collections.singletonList(sd)); + public static DocumenttypesConfig.Builder getDocumentTypesConfig(String schema) { + return getDocumentTypesConfig(Collections.singletonList(schema)); } - public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> sds) { - return new DocumentTypes().produce(getSearchBuilder(sds).getModel(), new DocumenttypesConfig.Builder()); + public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> schemas) { + return new DocumentTypes().produce(getSchemaBuilder(schemas).getModel(), new DocumenttypesConfig.Builder()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java index c6f59fbe596..2a2950f71e9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java @@ -4,11 +4,9 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.collections.Pair; import java.util.ArrayList; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.logging.Logger; /** * The rank settings of a field used for native rank features. @@ -17,9 +15,7 @@ import java.util.logging.Logger; */ public class FieldRankSettings { - private static final Logger logger = Logger.getLogger(FieldRankSettings.class.getName()); - - private String fieldName; + private final String fieldName; private final Map<String, NativeTable> tables = new LinkedHashMap<>(); @@ -30,7 +26,7 @@ public class FieldRankSettings { public void addTable(NativeTable table) { NativeTable existing = tables.get(table.getType().getName()); if (existing != null) { - logger.info("Using already specified rank table " + existing + " for field " + fieldName + ", not " + table); + // TODO: Throw? return; } tables.put(table.getType().getName(), table); @@ -60,8 +56,7 @@ public class FieldRankSettings { public List<Pair<String, String>> deriveRankProperties() { List<Pair<String, String>> properties = new ArrayList<>(); - for (Iterator<NativeTable> i = tables.values().iterator(); i.hasNext();) { - NativeTable table = i.next(); + for (NativeTable table : tables.values()) { if (isFieldMatchTable(table)) properties.add(new Pair<>("nativeFieldMatch." + table.getType().getName() + "." + fieldName, table.getName())); if (isAttributeMatchTable(table)) @@ -72,6 +67,7 @@ public class FieldRankSettings { return properties; } + @Override public String toString() { return "rank settings of field " + fieldName; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 3081637c975..bcebdf3a916 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -73,9 +73,9 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry) { - return (rank.getInheritedName() == null) || - rankProfiles.containsKey(rank.getInheritedName()) || - (rank.getSearch() != null && registry.resolve(rank.getSearch().getDocument(), rank.getInheritedName()) != null); + return rank.inheritedNames().isEmpty() || + rankProfiles.keySet().containsAll(rank.inheritedNames()) || + (rank.schema() != null && rank.inheritedNames().stream().allMatch(name -> registry.resolve(rank.schema().getDocument(), name) != null)); } private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, @@ -92,7 +92,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } Map<String, RankProfile> remaining = new LinkedHashMap<>(); - rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.getName(), rank)); + rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); remaining.remove("default"); while (!remaining.isEmpty()) { List<RankProfile> ready = new ArrayList<>(); @@ -100,7 +100,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ if (areDependenciesReady(rank, rankProfileRegistry)) ready.add(rank); }); processRankProfiles(ready, queryProfiles, importedModels, schema, attributeFields, deployProperties, executor); - ready.forEach(rank -> remaining.remove(rank.getName())); + ready.forEach(rank -> remaining.remove(rank.name())); } } private void processRankProfiles(List<RankProfile> ready, @@ -116,8 +116,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ onnxModels.add(rank.onnxModels()); } - futureRawRankProfiles.put(rank.getName(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties))); + futureRawRankProfiles.put(rank.name(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties))); } try { for (Future<RawRankProfile> rawFuture : futureRawRankProfiles.values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index f166e533ab3..f024eb6eb77 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -63,9 +63,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.name = rankProfile.getName(); - compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), - attributeFields, deployProperties).derive(largeExpressions)); + this.name = rankProfile.name(); + compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), attributeFields, deployProperties, queryProfiles) + .derive(largeExpressions)); } private Compressor.Compression compress(List<Pair<String, String>> properties) { @@ -159,9 +159,11 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) - { - rankprofileName = compiled.getName(); + Deriver(RankProfile compiled, + AttributeFields attributeFields, + ModelContext.Properties deployProperties, + QueryProfileRegistry queryProfiles) { + rankprofileName = compiled.name(); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); firstPhaseRanking = compiled.getFirstPhaseRanking(); @@ -184,7 +186,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); Map<String, String> functionProperties = new LinkedHashMap<>(); - SerializationContext functionSerializationContext = new SerializationContext(functionExpressions); + SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, + Map.of(), + compiled.typeContext(queryProfiles)); if (firstPhaseRanking != null) { functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); @@ -206,8 +210,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void derivePropertiesAndFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions, - Map<String, String> functionProperties, - SerializationContext functionContext) { + Map<String, String> functionProperties, + SerializationContext functionContext) { if (functions.isEmpty()) return; replaceFunctionFeatures(summaryFeatures, functionContext); @@ -287,14 +291,15 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void deriveRankTypeSetting(RankProfile rankProfile, AttributeFields attributeFields) { for (Iterator<RankProfile.RankSetting> i = rankProfile.rankSettingIterator(); i.hasNext(); ) { RankProfile.RankSetting setting = i.next(); - if (!setting.getType().equals(RankProfile.RankSetting.Type.RANKTYPE)) continue; + if (setting.getType() != RankProfile.RankSetting.Type.RANKTYPE) continue; deriveNativeRankTypeSetting(setting.getFieldName(), (RankType) setting.getValue(), attributeFields, - hasDefaultRankTypeSetting(rankProfile, setting.getFieldName())); + hasDefaultRankTypeSetting(rankProfile, setting.getFieldName())); } } - private void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) { + private void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, + boolean isDefaultSetting) { if (isDefaultSetting) return; NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType); @@ -422,7 +427,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Map.Entry<String, String> queryFeatureType : queryFeatureTypes.entrySet()) { properties.add(new Pair<>("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue())); } - if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties"); + if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); distributeLargeExpressionsAsFiles(properties, largeRankExpressions); return properties; } @@ -461,8 +466,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void deriveOnnxModelFunctionsAndFeatures(RankProfile rankProfile) { - if (rankProfile.getSearch() == null) return; - if (rankProfile.getSearch().onnxModels().asMap().isEmpty()) return; + if (rankProfile.schema() == null) return; + if (rankProfile.schema().onnxModels().asMap().isEmpty()) return; replaceOnnxFunctionInputs(rankProfile); replaceImplicitOnnxConfigFeatures(summaryFeatures, rankProfile); replaceImplicitOnnxConfigFeatures(matchFeatures, rankProfile); @@ -471,7 +476,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void replaceOnnxFunctionInputs(RankProfile rankProfile) { Set<String> functionNames = rankProfile.getFunctions().keySet(); if (functionNames.isEmpty()) return; - for (OnnxModel onnxModel: rankProfile.getSearch().onnxModels().asMap().values()) { + for (OnnxModel onnxModel: rankProfile.schema().onnxModels().asMap().values()) { for (Map.Entry<String, String> mapping : onnxModel.getInputMap().entrySet()) { String source = mapping.getValue(); if (functionNames.contains(source)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java index 2fd90d6c87e..7310a3fbd88 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java @@ -14,6 +14,7 @@ import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.searchdefinition.FieldSets; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -53,15 +54,19 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { protected void derive(StreamingDocumentType document, SDField field) { if (field.usesStructOrMap()) { + if (GeoPos.isAnyPos(field)) { + StreamingField streamingField = new StreamingField(field); + addField(streamingField.getName(), streamingField); + addFieldToIndices(document, field.getName(), streamingField); + } for (SDField structField : field.getStructFields()) { derive(document, structField); // Recursion } } else { - if (! (field.doesIndexing() || field.doesSummarying() || field.doesAttributing()) ) return; - StreamingField streamingField=new StreamingField(field); + StreamingField streamingField = new StreamingField(field); addField(streamingField.getName(),streamingField); deriveIndices(document, field, streamingField); } @@ -132,14 +137,15 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { public static Type STRING = new Type("string","AUTOUTF8"); public static Type BOOL = new Type("bool","BOOL"); public static Type UNSEARCHABLESTRING = new Type("string","NONE"); + public static Type GEO_POSITION = new Type("position", "GEOPOS"); private String name; private String searchMethod; - private Type(String name,String searchMethod) { - this.name=name; - this.searchMethod=searchMethod; + private Type(String name, String searchMethod) { + this.name = name; + this.searchMethod = searchMethod; } @Override @@ -166,10 +172,10 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } public StreamingField(SDField field) { - this(field.getName(),field.getDataType(),field.getMatching(), field.doesAttributing()); + this(field.getName(), field.getDataType(), field.getMatching(), field.doesAttributing()); } - private StreamingField(String name,DataType sourceType, Matching matching, boolean isAttribute) { + private StreamingField(String name, DataType sourceType, Matching matching, boolean isAttribute) { this.name = name; this.type = convertType(sourceType); this.matching = matching; @@ -191,6 +197,8 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { return Type.BOOL; } else if (fieldType.equals(DataType.BYTE)) { return Type.INT8; + } else if (GeoPos.isAnyPos(fieldType)) { + return Type.GEO_POSITION; } else if (fieldType instanceof NumericDataType) { return Type.INT32; } else if (fval instanceof StringFieldValue) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java index 03b9e795317..998343bc467 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java @@ -37,7 +37,7 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { if (doMapField(schema, summaryField)) { SDField sdField = schema.getConcreteField(summaryField.getName()); - if (sdField != null && GeoPos.isPos(sdField)) { + if (sdField != null && GeoPos.isAnyPos(sdField)) { summaryMap.put(summaryField, Collections.singletonList(summaryField.getName())); } else { summaryMap.put(summaryField, from); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 5ac6dd46102..fce128d80f3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -337,8 +337,8 @@ public final class Attribute implements Cloneable, Serializable { @SuppressWarnings("deprecation") private DataType createReferenceDataType() { - if (!referenceDocumentType.isPresent()) { - throw new IllegalStateException("Referenced document type is not set!"); + if (referenceDocumentType.isEmpty()) { + throw new IllegalStateException("Referenced document type is not set"); } StructuredDataType type = referenceDocumentType.get(); if (type instanceof DocumentType) { @@ -350,9 +350,9 @@ public final class Attribute implements Cloneable, Serializable { public DataType getDataType() { DataType dataType = toDataType(type); - if (collectionType.equals(Attribute.CollectionType.ARRAY)) { + if (collectionType == Attribute.CollectionType.ARRAY) { return DataType.getArray(dataType); - } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) { + } else if (collectionType == Attribute.CollectionType.WEIGHTEDSET) { return DataType.getWeightedSet(dataType, createIfNonExistent, removeIfZero); } else { return dataType; @@ -371,7 +371,7 @@ public final class Attribute implements Cloneable, Serializable { public boolean equals(Object object) { if (! (object instanceof Attribute)) return false; - Attribute other=(Attribute)object; + Attribute other = (Attribute)object; if (!this.name.equals(other.name)) return false; return isCompatible(other); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java index 691426ee413..d506b22297d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java @@ -102,8 +102,8 @@ public class Matching implements Cloneable, Serializable { } if (m.isTypeUserSet()) { this.setType(m.getType()); - if (m.getType()==Type.GRAM) - gramSize=m.gramSize; + if (m.getType() == Type.GRAM) + gramSize = m.gramSize; } if (m.getExactMatchTerminator() != null) { this.setExactMatchTerminator(m.getExactMatchTerminator()); @@ -125,8 +125,10 @@ public class Matching implements Cloneable, Serializable { this.exactMatchTerminator = exactMatchTerminator; } + @Override public String toString() { - return type + " matching [" + (type==Type.GRAM ? "gram size " + gramSize : "supports " + algorithm) + "], [exact-terminator "+exactMatchTerminator+"]"; + return type + " matching [" + (type==Type.GRAM ? "gram size " + gramSize : "supports " + algorithm) + + "], [exact-terminator "+exactMatchTerminator+"]"; } public Matching clone() { @@ -145,10 +147,10 @@ public class Matching implements Cloneable, Serializable { Matching other=(Matching)o; if ( ! other.type.equals(this.type)) return false; if ( ! other.algorithm.equals(this.algorithm)) return false; - if ( this.exactMatchTerminator==null && other.exactMatchTerminator!=null) return false; - if ( this.exactMatchTerminator!=null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) + if ( this.exactMatchTerminator == null && other.exactMatchTerminator != null) return false; + if ( this.exactMatchTerminator != null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) return false; - if ( gramSize!=other.gramSize) return false; + if ( gramSize != other.gramSize) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java index 7bfff2d56ee..235677cd05e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java @@ -69,7 +69,7 @@ public class Ranking implements Cloneable, Serializable { return (Ranking)super.clone(); } catch (CloneNotSupportedException e) { - throw new RuntimeException("Programming error",e); + throw new RuntimeException("Programming error", e); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 566342403d4..edb317f4a99 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -435,7 +435,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, config.setInputStream(new IndexingInput(script)); setIndexingScript(ScriptExpression.newInstance(config)); } catch (ParseException e) { - throw new RuntimeException("Failed to parser script '" + script + "'.", e); + throw new IllegalArgumentException("Failed to parse script '" + script + "'", e); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java index 96f2f2f1d24..b4d76445507 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java @@ -29,11 +29,7 @@ public class TemporaryImportedFields { } public boolean hasField(String fieldName) { - boolean has = fields.get(fieldName) != null; - if (has) return true; - if (owner.inherited().isEmpty()) return false; - if (owner.inherited().get().temporaryImportedFields().isEmpty()) return false; - return owner.inherited().get().temporaryImportedFields().get().hasField(fieldName); + return fields.get(fieldName) != null; } public Map<String, TemporaryImportedField> fields() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java index 166255799bd..f6ed5abaa7f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java @@ -63,8 +63,8 @@ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfile TensorValue tensorValue = (TensorValue)value; String tensorType = tensorValue.asTensor().type().toString(); - context.rankProperties().put(constantReference.toString() + ".value", tensorValue.toString()); - context.rankProperties().put(constantReference.toString() + ".type", tensorType); + context.rankProperties().put(constantReference + ".value", tensorValue.toString()); + context.rankProperties().put(constantReference + ".type", tensorType); return new ReferenceNode(constantReference); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java index ae8f918dd4b..6944a1f9dd1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java @@ -39,7 +39,6 @@ public class OnnxFeatureConverter extends ExpressionTransformer<RankProfileTrans private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { if ( ! feature.getName().equals("onnx_vespa")) return feature; - try { FeatureArguments arguments = asFeatureArguments(feature.getArguments()); ConvertedModel convertedModel = diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java index 6d9f4cdec92..35ee9ddb9ed 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java @@ -49,12 +49,12 @@ public class OnnxModelTransformer extends ExpressionTransformer<RankProfileTrans private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { if (context.rankProfile() == null) return feature; - if (context.rankProfile().getSearch() == null) return feature; + if (context.rankProfile().schema() == null) return feature; return transformFeature(feature, context.rankProfile()); } public static ExpressionNode transformFeature(ReferenceNode feature, RankProfile rankProfile) { - ImmutableSchema search = rankProfile.getSearch(); + ImmutableSchema search = rankProfile.schema(); final String featureName = feature.getName(); if ( ! featureName.equals("onnxModel") && ! featureName.equals("onnx")) return feature; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java index cde172d00b9..0b275c6a722 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java @@ -6,11 +6,11 @@ import com.yahoo.javacc.FastCharStream; /** * @author Simon Thoresen Hult */ -@SuppressWarnings("deprecation") public class SimpleCharStream extends FastCharStream implements com.yahoo.searchdefinition.parser.CharStream, - com.yahoo.vespa.indexinglanguage.parser.CharStream -{ + com.yahoo.vespa.indexinglanguage.parser.CharStream { + public SimpleCharStream(String input) { super(input); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index 57d5a9c8e75..51defffa00b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,11 +26,6 @@ public class AddExtraFieldsToDocument extends Processor { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } - //TODO This is a tempoarry hack to avoid producing illegal code for fields not wanted anyway. - private boolean dirtyLegalFieldNameCheck(String fieldName) { - return ! fieldName.contains(".") && !"rankfeatures".equals(fieldName) && !"summaryfeatures".equals(fieldName); - } - @Override public void process(boolean validate, boolean documentsOnly) { SDDocumentType document = schema.getDocument(); @@ -38,10 +33,21 @@ public class AddExtraFieldsToDocument extends Processor { for (SDField field : schema.extraFieldList()) { addSdField(schema, document, field, validate); } - //TODO Vespa 8 or sooner we should avoid the dirty addition of fields from dirty 'default' summary to document at all - for (SummaryField field : schema.getSummary("default").getSummaryFields().values()) { - if (dirtyLegalFieldNameCheck(field.getName())) { - addSummaryField(schema, document, field, validate); + for (var docsum : schema.getSummaries().values()) { + for (var summaryField : docsum.getSummaryFields().values()) { + switch (summaryField.getTransform()) { + case NONE: + case BOLDED: + case DYNAMICBOLDED: + case DYNAMICTEASER: + case TEXTEXTRACTOR: + addSummaryField(schema, document, summaryField, validate); + break; + default: + // skip: generated from attribute or similar, + // so does not need to be included as an extra + // field in the document type + } } } } @@ -52,14 +58,7 @@ public class AddExtraFieldsToDocument extends Processor { return; } for (Attribute atr : field.getAttributes().values()) { - // TODO Vespa 8 or before: Check if this should be removed or changed to _zcurve. - if (atr.getName().equals(field.getName() + "_position")) { - DataType type = PositionDataType.INSTANCE; - if (atr.getCollectionType().equals(Attribute.CollectionType.ARRAY)) { - type = DataType.getArray(type); - } - addField(schema, document, new SDField(document, atr.getName(), type), validate); - } else if (!atr.getName().equals(field.getName())) { + if (!atr.getName().equals(field.getName())) { addField(schema, document, new SDField(document, atr.getName(), atr.getDataType()), validate); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java index 766b6ed3fec..254b3743f52 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; @@ -26,6 +27,14 @@ public class AdjustPositionSummaryFields extends Processor { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } + private boolean useV8GeoPositions = false; + + @Override + public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { + this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); + process(validate, documentsOnly); + } + @Override public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary summary : schema.getSummaries().values()) { @@ -80,6 +89,7 @@ public class AdjustPositionSummaryFields extends Processor { private void ensureSummaryField(DocumentSummary summary, String fieldName, DataType dataType, Source source, SummaryTransform transform) { SummaryField oldField = schema.getSummaryField(fieldName); if (oldField == null) { + if (useV8GeoPositions) return; SummaryField newField = new SummaryField(fieldName, dataType, transform); newField.addSource(source); summary.add(newField); @@ -94,6 +104,7 @@ public class AdjustPositionSummaryFields extends Processor { if (oldField.getSourceCount() != 1 || !oldField.getSingleSource().equals(source.getName())) { fail(oldField, "has source '" + oldField.getSources().toString() + "', should have source '" + source + "'"); } + if (useV8GeoPositions) return; summary.add(oldField); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index f5c1d8d8197..0bb1b7da769 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; @@ -38,6 +39,14 @@ public class CreatePositionZCurve extends Processor { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } + private boolean useV8GeoPositions = false; + + @Override + public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { + this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); + process(validate, documentsOnly); + } + @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : schema.allConcreteFields()) { @@ -63,14 +72,16 @@ public class CreatePositionZCurve extends Processor { // configure summary Collection<String> summaryTo = removeSummaryTo(field); - ensureCompatibleSummary(field, zName, - PositionDataType.getPositionSummaryFieldName(fieldName), - DataType.getArray(DataType.STRING), // will become "xmlstring" - SummaryTransform.POSITIONS, summaryTo, validate); - ensureCompatibleSummary(field, zName, - PositionDataType.getDistanceSummaryFieldName(fieldName), - DataType.INT, - SummaryTransform.DISTANCE, summaryTo, validate); + if (! useV8GeoPositions) { + ensureCompatibleSummary(field, zName, + PositionDataType.getPositionSummaryFieldName(fieldName), + DataType.getArray(DataType.STRING), // will become "xmlstring" + SummaryTransform.POSITIONS, summaryTo, validate); + ensureCompatibleSummary(field, zName, + PositionDataType.getDistanceSummaryFieldName(fieldName), + DataType.INT, + SummaryTransform.DISTANCE, summaryTo, validate); + } // clear indexing script field.setIndexingScript(null); SDField posX = field.getStructField(PositionDataType.FIELD_X); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java index 5643bb660f1..3759fc453df 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -30,7 +30,7 @@ public class DiversitySettingsValidator extends Processor { } private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) { String attributeName = settings.getAttribute(); - new AttributeValidator(schema.getName(), rankProfile.getName(), + new AttributeValidator(schema.getName(), rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java index d89f83c333f..3f97bf83565 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -63,7 +63,7 @@ public class FilterFieldNames extends Processor { filterFields.add(fieldName); } } else { - deployLogger.logApplicationPackage(Level.WARNING, "For rank profile '" + profile.getName() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); + deployLogger.logApplicationPackage(Level.WARNING, "For rank profile '" + profile.name() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index 9eb8b921e81..0db6f4f05ba 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -48,7 +48,7 @@ public class ImplicitSummaries extends Processor { sdField.addSummaryFieldSources(summaryField); } - private void collectSummaries(SDField field , Schema schema, boolean validate) { + private void collectSummaries(SDField field, Schema schema, boolean validate) { SummaryField addedSummaryField = null; // Implicit @@ -91,9 +91,16 @@ public class ImplicitSummaries extends Processor { if (field.doesSummarying()) { for (Attribute attribute : field.getAttributes().values()) { if ( ! attribute.isPosition()) continue; - DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); - attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName))); - attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName))); + var distField = field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName)); + if (distField != null) { + DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); + attributePrefetchSummary.add(distField); + } + var posField = field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName)); + if (posField != null) { + DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); + attributePrefetchSummary.add(posField); + } } } @@ -104,7 +111,6 @@ public class ImplicitSummaries extends Processor { if (attribute != null && summaryField.getTransform() == SummaryTransform.NONE) { summaryField.setTransform(SummaryTransform.ATTRIBUTE); } - if (isValid(summaryField, schema, validate)) { addToDestinations(summaryField, schema); } @@ -203,8 +209,9 @@ public class ImplicitSummaries extends Processor { addToDestination("default", summaryField, schema); } else { - for (String destinationName : summaryField.getDestinations()) + for (String destinationName : summaryField.getDestinations()) { addToDestination(destinationName, summaryField, schema); + } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index e11f7d370c5..df103dcb096 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -31,8 +31,8 @@ public class ImplicitSummaryFields extends Processor { private void addField(DocumentSummary docsum, SummaryField field, boolean validate) { if (validate && docsum.getSummaryField(field.getName()) != null) { - throw new IllegalStateException("Summary class '" + docsum.getName() + "' uses reserved field name '" + - field.getName() + "'."); + throw new IllegalArgumentException("Summary class '" + docsum.getName() + "' uses reserved field name '" + + field.getName() + "'."); } docsum.add(field); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java index b697c584ece..4eae6b47833 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -35,7 +35,7 @@ public class MatchPhaseSettingsValidator extends Processor { private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) { String attributeName = settings.getAttribute(); new AttributeValidator(schema.getName(), - rankProfile.getName(), + rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java index 2a4f4f18759..2ca4abae2c4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java @@ -9,6 +9,8 @@ import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.Optional; + /** * Validates the 'paged' attribute setting and throws if specified on unsupported types. * @@ -38,11 +40,23 @@ public class PagedAttributeValidator extends Processor { } private void validatePagedSetting(Field field, Attribute attribute) { - var tensorType = attribute.tensorType(); - if (tensorType.isEmpty() - || !isDenseTensorType(tensorType.get())) { - fail(schema, field, "The 'paged' attribute setting is only supported for dense tensor types"); + if (!isSupportedType(attribute)) { + fail(schema, field, "The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types"); + } + } + + private boolean isSupportedType(Attribute attribute) { + var type = attribute.getType(); + return (type != Attribute.Type.PREDICATE) && + (type != Attribute.Type.REFERENCE) && + (isSupportedTensorType(attribute.tensorType())); + } + + private boolean isSupportedTensorType(Optional<TensorType> tensorType) { + if (tensorType.isPresent()) { + return isDenseTensorType(tensorType.get()); } + return true; } private boolean isDenseTensorType(TensorType type) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java index f5c3fa35e34..e6adb8b683b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java @@ -89,10 +89,10 @@ public class RankingExpressionTypeResolver extends Processor { if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty() && ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { - deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in '" + profile.getName() + - "' are not declared in query profile " + - "types and will be interpreted as scalars, not tensors: " + - context.queryFeaturesNotDeclared()); + deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in '" + profile.name() + + "' are not declared in query profile " + + "types and will be interpreted as scalars, not tensors: " + + context.queryFeaturesNotDeclared()); warnedAbout.addAll(context.queryFeaturesNotDeclared()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java index 8194b9f9e06..f4f920d9ec8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java @@ -35,9 +35,9 @@ public class ReservedFunctionNames extends Processor { for (String functionName : rp.getFunctions().keySet()) { if (reservedNames.contains(functionName)) { deployLogger.logApplicationPackage(Level.WARNING, "Function '" + functionName + "' " + - "in rank profile '" + rp.getName() + "' " + - "has a reserved name. This might mean that the function shadows " + - "the built-in function with the same name." + "in rank profile '" + rp.name() + "' " + + "has a reserved name. This might mean that the function shadows " + + "the built-in function with the same name." ); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java index bea7c95d412..15599c567ab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java @@ -9,23 +9,15 @@ import com.yahoo.documentmodel.DocumentTypeRepo; * It contains a search manager managing all specified search definitions. * It contains a storage manager managing all specified storage definitions. * - * @author baldersheim - * @since 2010-02-19 + * @author baldersheim */ public class DocumentModel { - private DocumentTypeRepo documentMan = new DocumentTypeRepo(); - private SearchManager searchMan = new SearchManager(); - /** - * - * @return Returns the DocumentManager - */ + private final DocumentTypeRepo documentMan = new DocumentTypeRepo(); + private final SearchManager searchMan = new SearchManager(); + public DocumentTypeRepo getDocumentManager() { return documentMan; } - /** - * - * @return Returns the SearchManager - */ - public SearchManager getSearchManager() { return searchMan; } + public SearchManager getSearchManager() { return searchMan; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java index e146d3c8798..acfdd4f8671 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java @@ -8,10 +8,10 @@ import java.util.ArrayList; import java.util.List; /** - * @author baldersheim - * @since 2010-02-19 + * @author baldersheim */ public class SearchField extends Field { + /// Indicate if field shall be stored in memory for attribute usage. private boolean attribute = false; /// Indicate if the field is Vespa indexed. @@ -45,7 +45,6 @@ public class SearchField extends Field { validate(); } - @SuppressWarnings({ "deprecation" }) private void validate() { if (attribute || !indexed) { return; @@ -55,17 +54,14 @@ public class SearchField extends Field { if (DataType.STRING.equals(primiType) || DataType.URI.equals(primiType)) { return; } - throw new IllegalStateException("Expected type " + DataType.STRING.getName() + " for indexed field '" + - getName() + "', got " + fieldType.getName() + "."); + throw new IllegalArgumentException("Expected type " + DataType.STRING.getName() + " for indexed field '" + + getName() + "', got " + fieldType.getName() + "."); } - public SearchField setIndexed() { indexed = true; validate(); return this; } + public SearchField setIndexed() { indexed = true; validate(); return this; } public SearchField setAttribute() { attribute = true; validate(); return this; } public boolean isAttribute() { return attribute; } - /** - * True if field is Vespa indexed - * @return true if indexed - */ + public boolean isIndexed() { return indexed; } public SearchField addFeature(Feature feature) { featureList.add(feature); validate(); return this; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 0e7ee02d7ba..75a2a808a89 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -197,32 +197,30 @@ public class SummaryField extends Field implements Cloneable, TypedKey { * Returns a summary field which merges the settings in the given field * into this field * - * @param merge the field to merge with this, if null, the merged field is - * <code>this</code> field + * @param merge the field to merge with this, if null, the merged field is *this* field * @throws RuntimeException if the two fields can not be merged */ public SummaryField mergeWith(SummaryField merge) { - if (merge==null) return this; + if (merge == null) return this; if (this.isImplicit()) return merge; if (merge.isImplicit()) return this; if (!merge.getName().equals(getName())) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different names"); - if (!merge.getTransform().equals(getTransform())) + if (merge.getTransform() != getTransform()) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different transforms"); if (!merge.getDataType().equals(getDataType())) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different types"); - if (!merge.isImplicit()) - setImplicit(false); + setImplicit(false); - if (isHeadOf(this.sourceIterator(),merge.sourceIterator())) { + if (isHeadOf(this.sourceIterator(), merge.sourceIterator())) { // Ok } - else if (isHeadOf(merge.sourceIterator(),this.sourceIterator())) { - sources=new LinkedHashSet<>(merge.sources); + else if (isHeadOf(merge.sourceIterator(), this.sourceIterator())) { + sources = new LinkedHashSet<>(merge.sources); } else { throw new IllegalArgumentException(merge + " conflicts with " + this + @@ -277,13 +275,14 @@ public class SummaryField extends Field implements Cloneable, TypedKey { return "'summary " + getName() + " type " + toLowerCase(getDataType().getName()) + "' in '" + getDestinationString() + "'"; } + @Override public SummaryField clone() { try { - SummaryField clone=(SummaryField)super.clone(); - if (this.sources!=null) - clone.sources=new LinkedHashSet<>(this.sources); - if (this.destinations!=null) - clone.destinations=new LinkedHashSet<>(destinations); + SummaryField clone = (SummaryField)super.clone(); + if (this.sources != null) + clone.sources = new LinkedHashSet<>(this.sources); + if (this.destinations != null) + clone.destinations = new LinkedHashSet<>(destinations); return clone; } catch (CloneNotSupportedException e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 73b106ed393..6777e2fb741 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; @@ -16,6 +17,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; import static com.yahoo.text.Lowercase.toLowerCase; @@ -54,19 +57,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon // Please keep non-null, as passed to command line in service startup private String preload = null; - // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump. - private long mmapNoCoreLimit = -1L; - - // If this is true it will dump core when OOM - private boolean coreOnOOM = false; - - // If greater than 0, controls the number of threads used by open mp - private int ompNumThreads = 0; - - private String noVespaMalloc = ""; - private String vespaMalloc = ""; - private String vespaMallocDebug = ""; - private String vespaMallocDebugStackTrace = ""; + private final Map<String, Object> environmentVariables = new TreeMap<>(); /** The ports metainfo object */ protected PortsMeta portsMeta = new PortsMeta(); @@ -97,6 +88,8 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon */ public AbstractService(AbstractConfigProducer<?> parent, String name) { super(parent, name); + environmentVariables.put("OMP_NUM_THREADS", 1); + environmentVariables.put("VESPA_SILENCE_CORE_ON_OOM", true); } /** @@ -105,7 +98,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * @param name the name of this service. */ public AbstractService(String name) { - super(name); + this(null, name); } @Override @@ -139,25 +132,28 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * @param hostResource The physical host on which this service should run. * @param userPort The wanted port given by the user. */ - private void initService(DeployLogger deployLogger, HostResource hostResource, int userPort) { + private void initService(DeployState deployState, HostResource hostResource, int userPort) { if (initialized) { throw new IllegalStateException("Service '" + getConfigId() + "' already initialized."); } if (hostResource == null) { - throw new RuntimeException("No host found for service '" + getServiceName() + "'. " + - "The hostalias is probably missing from hosts.xml."); + throw new IllegalArgumentException("No host found for service '" + getServiceName() + "'. " + + "The hostalias is probably missing from hosts.xml."); } id = getIndex(hostResource); - ports = hostResource.allocateService(deployLogger, this, getInstanceWantedPort(userPort)); + ports = hostResource.allocateService(deployState.getDeployLogger(), this, getInstanceWantedPort(userPort)); initialized = true; + for(String envVar : deployState.getProperties().environmentVariables()) { + addEnvironmentVariable(envVar); + } } /** * Called by builder class which has not given the host or port in a constructor, hence * initService is not yet run for this. */ - public void initService(DeployLogger deployLogger) { - initService(deployLogger, this.hostResource, this.basePort); + public void initService(DeployState deployState) { + initService(deployState, this.hostResource, this.basePort); } /** @@ -381,60 +377,44 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon public void setPreLoad(String preload) { this.preload = preload; } - public long getMMapNoCoreLimit() { return mmapNoCoreLimit; } - public void setMMapNoCoreLimit(long noCoreLimit) { this.mmapNoCoreLimit = noCoreLimit; } - public boolean getCoreOnOOM() { return coreOnOOM; } - public void setCoreOnOOM(boolean coreOnOOM) { this.coreOnOOM = coreOnOOM; } - public int getOmpNumThreads() { return ompNumThreads; } - public void setOmpNumThreads(int value) { ompNumThreads = value; } - - public String getNoVespaMalloc() { return noVespaMalloc; } - public String getVespaMalloc() { return vespaMalloc; } - public String getVespaMallocDebug() { return vespaMallocDebug; } - public String getVespaMallocDebugStackTrace() { return vespaMallocDebugStackTrace; } - public void setNoVespaMalloc(String s) { noVespaMalloc = s; } - public void setVespaMalloc(String s) { vespaMalloc = s; } - public void setVespaMallocDebug(String s) { vespaMallocDebug = s; } - public void setVespaMallocDebugStackTrace(String s) { vespaMallocDebugStackTrace = s; } - - public String getMMapNoCoreEnvVariable() { - return (getMMapNoCoreLimit() >= 0L) - ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " " - : ""; - } - - public String getCoreOnOOMEnvVariable() { - return getCoreOnOOM() ? "" : "VESPA_SILENCE_CORE_ON_OOM=true "; - } - public String getOmpNumThreadsEnvVariable() { - return (getOmpNumThreads() == 0) - ? "" - : "OMP_NUM_THREADS=" + getOmpNumThreads() + " "; - } - public String getNoVespaMallocEnvVariable() { - return "".equals(getNoVespaMalloc()) - ? "" - : "VESPA_USE_NO_VESPAMALLOC=\"" + getNoVespaMalloc() + "\" "; - } - public String getVespaMallocEnvVariable() { - return "".equals(getVespaMalloc()) - ? "" - : "VESPA_USE_VESPAMALLOC=\"" + getVespaMalloc() + "\" "; - } - public String getVespaMallocDebugEnvVariable() { - return "".equals(getVespaMallocDebug()) - ? "" - : "VESPA_USE_VESPAMALLOC_D=\"" + getVespaMallocDebug() + "\" "; - } - public String getVespaMallocDebugStackTraceEnvVariable() { - return "".equals(getVespaMallocDebugStackTrace()) - ? "" - : "VESPA_USE_VESPAMALLOC_DST=\"" + getVespaMallocDebugStackTrace() + "\" "; - } - - public String getEnvVariables() { - return getCoreOnOOMEnvVariable() + getOmpNumThreadsEnvVariable() + getMMapNoCoreEnvVariable() + getNoVespaMallocEnvVariable() + - getVespaMallocEnvVariable() + getVespaMallocDebugEnvVariable() + getVespaMallocDebugStackTraceEnvVariable(); + /** If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump.*/ + public void setMMapNoCoreLimit(long noCoreLimit) { + if (noCoreLimit >= 0) { + environmentVariables.put("VESPA_MMAP_NOCORE_LIMIT", noCoreLimit); + } else { + environmentVariables.remove("VESPA_MMAP_NOCORE_LIMIT"); + } + } + public void setCoreOnOOM(boolean coreOnOOM) { + if ( ! coreOnOOM) { + environmentVariables.put("VESPA_SILENCE_CORE_ON_OOM", true); + } else { + environmentVariables.remove("VESPA_SILENCE_CORE_ON_OOM"); + } + } + + public void setNoVespaMalloc(String s) { environmentVariables.put("VESPA_USE_NO_VESPAMALLOC", s); } + public void setVespaMalloc(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC", s); } + public void setVespaMallocDebug(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC_D", s); } + public void setVespaMallocDebugStackTrace(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC_DST", s); } + + private static String toEnvValue(Object o) { + if (o instanceof Number || o instanceof Boolean) { + return o.toString(); + } + return '"' + o.toString() + '"'; + } + + public void addEnvironmentVariable(String nameAndValue) { + int pos = nameAndValue.indexOf('='); + environmentVariables.put(nameAndValue.substring(0, pos), nameAndValue.substring(pos+1)); + } + public void addEnvironmentVariable(String name, Object value) { + environmentVariables.put(name, value); + } + + public String getEnv() { + return environmentVariables.entrySet().stream().map(e -> e.getKey() + '=' + toEnvValue(e.getValue())).collect(Collectors.joining(" ")); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index 07dc0a5f9e9..b31e6028c05 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -193,9 +193,9 @@ public class HostPorts { String msg = (service.getClass().equals(otherService.getClass()) && service.requiresWantedPort()) ? "You must set port explicitly for all instances of this service type, except the first one. " : ""; - throw new RuntimeException(service.getServiceName() + " cannot reserve port " + port + - " on " + hostname + ": Already reserved for " + otherService.getServiceName() + - ". " + msg + "Next available port is: " + nextAvailablePort + " ports used: " + portDB); + throw new IllegalArgumentException(service.getServiceName() + " cannot reserve port " + port + + " on " + hostname + ": Already reserved for " + otherService.getServiceName() + + ". " + msg + "Next available port is: " + nextAvailablePort + " ports used: " + portDB); } private void noMoreAvailablePorts() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 8f804229dd0..05aa125b808 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeResources; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -103,7 +104,7 @@ public class HostResource implements Comparable<HostResource> { public HostInfo getHostInfo() { return new HostInfo(getHostname(), services.values().stream() .map(Service::getServiceInfo) - .collect(Collectors.toSet())); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>()))); } /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java index 70a238b4a0c..3420ab26e20 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java @@ -14,7 +14,7 @@ import java.util.List; public class PortsMeta implements Serializable { /** A list of all ports. The list elements are lists of strings. */ - private List<LinkedList<String>> ports; + private final List<LinkedList<String>> ports; /** Remember the rpc admin port offset. */ private Integer rpcAdminOffset = null; @@ -35,8 +35,9 @@ public class PortsMeta implements Serializable { /** * Set up the port to tag, for chained usage. - * @param offset The relative port to tag. - * @return this portsmeta, to allow .tag calls. + * + * @param offset the relative port to tag + * @return this portsmeta, to allow .tag calls */ public PortsMeta on(int offset) { this.currentOffset = offset; @@ -45,8 +46,9 @@ public class PortsMeta implements Serializable { /** * Tag a previously setup port (using 'on') with the specified tag. - * @param meta The tag to apply to the current port. - * @return this portsmeta, to allow further .tag calls. + * + * @param meta the tag to apply to the current port + * @return this portsmeta, to allow further .tag calls */ public PortsMeta tag(String meta) { if (currentOffset == null) { @@ -57,9 +59,10 @@ public class PortsMeta implements Serializable { /** * Register a given metainfo string to the port at offset. + * * @param offset 0-based index to identify the port - * @param meta a String to be added to the given ports meta set. - * @return this for convenient chaining. + * @param meta a String to be added to the given ports meta set + * @return this for convenient chaining */ private PortsMeta register(int offset, String meta) { // Allocate new LinkedLists on each element up-to-and-including offset @@ -73,8 +76,9 @@ public class PortsMeta implements Serializable { /** * Check if the port at a specific offset contains a particular meta attribute. - * @param offset The relative port offset - * @param meta The meta info we want to check for + * + * @param offset the relative port offset + * @param meta the meta info we want to check for * @return boolean true if the specific port has registered the meta */ public boolean contains(int offset, String meta) { @@ -83,7 +87,8 @@ public class PortsMeta implements Serializable { /** * Get the number of ports with registered meta. - * @return the number of ports that have been registered. + * + * @return the number of ports that have been registered */ public int getNumPorts() { return ports.size(); @@ -91,21 +96,23 @@ public class PortsMeta implements Serializable { /** * Get an iterator of the Strings registered at the specific point. - * @param offset The relative offset to inquire about tags. + * + * @param offset the relative offset to inquire about tags * @return List of tags. */ public List<String> getTagsAt(int offset) { try { return ports.get(offset); } catch (IndexOutOfBoundsException e) { - throw new RuntimeException("Trying to get ports meta with offset " + offset + - ", which is outside the range 0 to " + ports.size(), e); + throw new IllegalArgumentException("Trying to get ports meta with offset " + offset + + ", which is outside the range 0 to " + ports.size(), e); } } /** * Get the offset to the rpc port used for admin. - * @return Integer the offset, or null if none set. + * + * @return the offset, or null if none set */ public Integer getRpcAdminOffset() { if (rpcAdminOffset == null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 13b0f6216b2..8390cc59b6f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -87,9 +87,7 @@ import java.util.stream.Collectors; import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; import static com.yahoo.text.StringUtilities.quote; import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toUnmodifiableMap; -import static java.util.stream.Collectors.toUnmodifiableSet; /** * <p> @@ -229,7 +227,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private static Set<String> documentTypesWithIndex(ContentCluster content) { Set<String> typesWithIndexMode = content.getSearch().getDocumentTypesWithIndexedCluster().stream() .map(type -> type.getFullName().getName()) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); Set<String> typesWithIndexedFields = content.getSearch().getIndexed() == null ? Set.of() @@ -239,9 +237,10 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri .allConcreteFields() .stream().anyMatch(SDField::doesIndexing)) .map(database -> database.getInputDocType()) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); - return typesWithIndexMode.stream().filter(typesWithIndexedFields::contains).collect(toUnmodifiableSet()); + return typesWithIndexMode.stream().filter(typesWithIndexedFields::contains) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } private void propagateRestartOnDeploy() { @@ -324,7 +323,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri for (var futureConvertedModel : futureModels) { try { futureConvertedModel.get(); - } catch (ExecutionException |InterruptedException e) { + } catch (ExecutionException | InterruptedException e) { throw new RuntimeException(e); } } @@ -620,7 +619,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * @param configId the id to register with, not necessarily equal to descendant.getConfigId(). * @param descendant The configProducer descendant to add */ - public void addDescendant(String configId, AbstractConfigProducer descendant) { + public void addDescendant(String configId, AbstractConfigProducer<?> descendant) { if (id2producer.containsKey(configId)) { throw new RuntimeException ("Config ID '" + configId + "' cannot be reserved by an instance of class '" + @@ -695,7 +694,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri .map(HostResource::spec) .filter(spec -> spec.membership().isPresent()) .map(spec -> spec.membership().get().cluster().id()) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index bd8fefdda2e..f2d0ab03e27 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -24,6 +24,7 @@ import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.VespaVersion; import com.yahoo.vespa.model.application.validation.Validation; +import com.yahoo.vespa.model.application.validation.Validator; import com.yahoo.yolean.Exceptions; import org.xml.sax.SAXException; @@ -50,11 +51,13 @@ public class VespaModelFactory implements ModelFactory { private final Zone zone; private final Clock clock; private final Version version; + private final List<Validator> additionalValidators; /** Creates a factory for Vespa models for this version of the source */ @Inject public VespaModelFactory(ComponentRegistry<ConfigModelPlugin> pluginRegistry, ComponentRegistry<MlModelImporter> modelImporters, + ComponentRegistry<Validator> additionalValidators, Zone zone) { this.version = new Version(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); List<ConfigModelBuilder> modelBuilders = new ArrayList<>(); @@ -66,6 +69,7 @@ public class VespaModelFactory implements ModelFactory { this.configModelRegistry = new MapConfigModelRegistry(modelBuilders); this.modelImporters = modelImporters.allComponents(); this.zone = zone; + this.additionalValidators = List.copyOf(additionalValidators.allComponents()); this.clock = Clock.systemUTC(); } @@ -91,6 +95,7 @@ public class VespaModelFactory implements ModelFactory { this.configModelRegistry = configModelRegistry; } this.modelImporters = Collections.emptyList(); + this.additionalValidators = List.of(); this.zone = zone; this.clock = clock; } @@ -158,7 +163,7 @@ public class VespaModelFactory implements ModelFactory { try { return new VespaModel(configModelRegistry, deployState); } catch (IOException | SAXException e) { - throw new RuntimeException(e); + throw new IllegalArgumentException(e); } } @@ -197,7 +202,7 @@ public class VespaModelFactory implements ModelFactory { private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, ValidationParameters validationParameters) { try { - return Validation.validate(model, validationParameters, deployState); + return new Validation(additionalValidators).validate(model, validationParameters, deployState); } catch (ValidationOverrides.ValidationException e) { if (deployState.isHosted() && zone.environment().isManuallyDeployed()) deployState.getDeployLogger().logApplicationPackage(Level.WARNING, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 2692d676d2b..e299689d1c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -168,7 +167,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable Slobrok slobrok = new Slobrok(this, clusterController.index(), deployState.featureFlags()); slobrok.setHostResource(clusterController.getHostResource()); slobroks.add(slobrok); - slobrok.initService(deployState.getDeployLogger()); + slobrok.initService(deployState); } return slobroks; } @@ -237,16 +236,16 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable // Send hostname to be used in configId (instead of index), as the sorting of hosts seems to be unstable // between config changes, even when the set of hosts is unchanged. var container = new MetricsProxyContainer(metricsProxyCluster, host, index, deployState); - addAndInitializeService(deployState.getDeployLogger(), host, container); + addAndInitializeService(deployState, host, container); metricsProxyCluster.addContainer(container); } } private void addCommonServices(HostResource host, DeployState deployState) { - addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone(), + addConfigSentinel(deployState, host, deployState.getProperties().applicationId(), deployState.zone(), deployState.featureFlags()); - addLogd(deployState.getDeployLogger(), host); - addConfigProxy(deployState.getDeployLogger(), host); + addLogd(deployState, host); + addConfigProxy(deployState, host); addFileDistribution(host); if (logForwarderConfig != null) { boolean actuallyAdd = true; @@ -259,34 +258,34 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } } if (actuallyAdd) { - addLogForwarder(deployState.getDeployLogger(), host); + addLogForwarder(deployState, host); } } } - private void addConfigSentinel(DeployLogger deployLogger, HostResource host, + private void addConfigSentinel(DeployState deployState, HostResource host, ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) { ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone, featureFlags); - addAndInitializeService(deployLogger, host, configSentinel); + addAndInitializeService(deployState, host, configSentinel); host.getHost().setConfigSentinel(configSentinel); } - private void addLogForwarder(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new LogForwarder(host.getHost(), logForwarderConfig)); + private void addLogForwarder(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new LogForwarder(host.getHost(), logForwarderConfig)); } - private void addLogd(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new Logd(host.getHost())); + private void addLogd(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new Logd(host.getHost())); } - private void addConfigProxy(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new ConfigProxy(host.getHost())); + private void addConfigProxy(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new ConfigProxy(host.getHost())); } - public void addAndInitializeService(DeployLogger deployLogger, HostResource host, AbstractService service) { + public void addAndInitializeService(DeployState deployState, HostResource host, AbstractService service) { service.setHostResource(host); - service.initService(deployLogger); + service.initService(deployState); } private void addFileDistribution(HostResource host) { @@ -300,7 +299,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable List<Slobrok> slobs = new ArrayList<>(); if (logserver != null) { Slobrok slobrok = new Slobrok(this, 0, deployState.featureFlags()); - addAndInitializeService(deployState.getDeployLogger(), logserver.getHostResource(), slobrok); + addAndInitializeService(deployState, logserver.getHostResource(), slobrok); slobs.add(slobrok); } @@ -309,7 +308,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable HostResource host = hosts.get(n); if ((logserver== null || host != logserver.getHostResource()) && ! host.getHost().runsConfigServer()) { Slobrok newSlobrok = new Slobrok(this, slobs.size(), deployState.featureFlags()); - addAndInitializeService(deployState.getDeployLogger(), host, newSlobrok); + addAndInitializeService(deployState, host, newSlobrok); slobs.add(newSlobrok); } n++; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 43f66f2c727..75b13a89e83 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -17,7 +17,7 @@ import java.util.Optional; public class LogserverContainerCluster extends ContainerCluster<LogserverContainer> { public LogserverContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) { - super(parent, name, name, deployState, true); + super(parent, name, name, deployState, true, deployState.featureFlags().defaultPoolNumThreads()); addDefaultHandlersWithVip(); addLogHandler(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java index 6e6f027b520..bf3ba084091 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java @@ -9,19 +9,18 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.HostSpec; -import com.yahoo.net.HostName; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.Configserver; import com.yahoo.vespa.model.container.Container; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import static java.util.stream.Collectors.toUnmodifiableSet; - /** * Used if clustercontroller is run standalone (not as part of the config server ZooKeeper cluster) * to provide common configs to container components. @@ -38,26 +37,24 @@ public class ClusterControllerCluster extends AbstractConfigProducer<ClusterCont public ClusterControllerCluster(AbstractConfigProducer<?> parent, String subId, DeployState deployState) { super(parent, subId); - this.previousHosts = deployState.getPreviousModel().stream() - .map(Model::allocatedHosts) - .map(AllocatedHosts::getHosts) - .flatMap(Collection::stream) - .map(HostSpec::hostname) - .collect(toUnmodifiableSet()); + this.previousHosts = Collections.unmodifiableSet(deployState.getPreviousModel().stream() + .map(Model::allocatedHosts) + .map(AllocatedHosts::getHosts) + .flatMap(Collection::stream) + .map(HostSpec::hostname) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>()))); } @Override public void getConfig(ZookeeperServerConfig.Builder builder) { builder.clientPort(ZK_CLIENT_PORT); builder.juteMaxBuffer(1024 * 1024); // 1 Mb should be more than enough for cluster controller - boolean oldQuorumExists = containerCluster.getContainers().stream() // More than half the previous hosts must be present in the new config for quorum to persist. - .filter(container -> previousHosts.contains(container.getHostName())) // Set intersection is symmetric. - .count() > previousHosts.size() / 2; for (ClusterControllerContainer container : containerCluster.getContainers()) { ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder(); serverBuilder.hostname(container.getHostName()); serverBuilder.id(container.index()); - serverBuilder.joining(oldQuorumExists && ! previousHosts.contains(container.getHostName())); + serverBuilder.joining( ! previousHosts.isEmpty() && ! previousHosts.contains(container.getHostName())); + serverBuilder.retired(container.isRetired()); builder.server(serverBuilder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index eafc460bfb5..20c3e007e3b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -40,9 +40,11 @@ public class ClusterControllerContainer extends Container implements private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps"); private static final ComponentSpecification ZOOKEEPER_SERVER_BUNDLE = new ComponentSpecification("zookeeper-server"); private static final ComponentSpecification REINDEXING_CONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-reindexer"); - // The below adjustments to default netty settings reduces default chunkSize from 16M to 1M + // The below adjustments to default netty settings reduces default chunkSize from 16M to 128K private static final int DEFAULT_NETTY_PAGE_SIZE = 4096; // Reduced from nettys default of 8192 - private static final int DEFAULT_NETTY_MAX_ORDER = 8; // Reduced from nettys default of 11 + private static final int DEFAULT_NETTY_MAX_ORDER = 5; // Reduced from nettys default of 11 + private static final int DEFAULT_NETTY_NUM_DIRECT_ARENAS = 1; // Reduced from nettys default of 2*cores + private static final int DEFAULT_NETTY_NUM_HEAP_ARENAS = 1; // Reduced from nettys default of 2*cores private final Set<String> bundles = new TreeSet<>(); @@ -72,12 +74,16 @@ public class ClusterControllerContainer extends Container implements addFileBundle("zookeeper-server"); configureReindexing(); configureZooKeeperServer(runStandaloneZooKeeper); - prependJvmOptions(defaultNettyBufferSize(DEFAULT_NETTY_PAGE_SIZE, DEFAULT_NETTY_MAX_ORDER)); + prependJvmOptions(defaultNettyBufferSize(DEFAULT_NETTY_NUM_HEAP_ARENAS, DEFAULT_NETTY_NUM_DIRECT_ARENAS, + DEFAULT_NETTY_PAGE_SIZE, DEFAULT_NETTY_MAX_ORDER)); } - private static String defaultNettyBufferSize(int pageSize, int maxOrder) { + private static String defaultNettyBufferSize(int numHeapArenas, int numDirectArenas, int pageSize, int maxOrder) { return new StringBuffer("-Dio.netty.allocator.pageSize=").append(pageSize) - .append(" -Dio.netty.allocator.maxOrder=").append(maxOrder).toString(); + .append(" -Dio.netty.allocator.maxOrder=").append(maxOrder) + .append(" -Dio.netty.allocator.numHeapArenas=").append(numHeapArenas) + .append(" -Dio.netty.allocator.numDirectArenas=").append(numDirectArenas) + .toString(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 728e46f2ff7..a7f3a6224f2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -22,7 +22,7 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC public ClusterControllerContainerCluster( AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { - super(parent, subId, name, deployState, false); + super(parent, subId, name, deployState, false, deployState.featureFlags().defaultPoolNumThreads()); addDefaultHandlersWithVip(); this.reindexingContext = createReindexingContext(deployState); setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java index e894fba079d..ae2139be43c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java @@ -8,9 +8,11 @@ import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; /** @@ -71,7 +73,7 @@ class ConsumersConfigGenerator { static Consumer.Builder toConsumerBuilder(MetricsConsumer consumer) { Consumer.Builder builder = new Consumer.Builder().name(consumer.id()); - consumer.metrics().values().forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); + consumer.metrics().values().stream().sorted(Comparator.comparing(a -> a.name)).forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); return builder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 88d224863c6..b6ddeb57cd4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -159,12 +159,12 @@ public class MetricsProxyContainer extends Container implements cluster.getConfig(builder); if (clusterMembership.isPresent()) { - int maxHeapSize = clusterMembership.get().cluster().type() == ClusterSpec.Type.admin - ? 128 - : 256; + boolean adminCluster = clusterMembership.get().cluster().type() == ClusterSpec.Type.admin; + int maxHeapSize = adminCluster ? 96 : 256; builder.jvm .gcopts(jvmGCOptions) .heapsize(maxHeapSize); + if (adminCluster) builder.jvm.minHeapsize(maxHeapSize); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index dd6f77ed093..a29647b062a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -87,7 +87,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC private final ApplicationId applicationId; public MetricsProxyContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) { - super(parent, name, name, deployState, true); + super(parent, name, name, deployState, true, deployState.featureFlags().defaultPoolNumThreads()); this.parent = parent; applicationId = deployState.getProperties().applicationId(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 50f3a21a4fb..26e3d952080 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.admin.monitoring; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -48,7 +49,7 @@ public class AutoscalingMetrics { } private static Set<Metric> toMetrics(List<String> metrics) { - return metrics.stream().map(Metric::new).collect(Collectors.toSet()); + return metrics.stream().map(Metric::new).collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java index d4b4a747717..5e10001baf6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java @@ -21,7 +21,7 @@ public class DefaultMetrics { public static final String defaultMetricSetId = "default"; - public static MetricSet defaultMetricSet = createMetricSet(); + public static final MetricSet defaultMetricSet = createMetricSet(); private static MetricSet createMetricSet() { return new MetricSet(defaultMetricSetId, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java index e5e5a30e34f..c2b0a358a54 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java @@ -25,11 +25,11 @@ public class Metric { this.name = name; this.outputName = outputName; this.description = description; - this.dimensions = Collections.unmodifiableMap(dimensions); + this.dimensions = Map.copyOf(dimensions); } public Metric(String name, String outputName, String description) { - this(name, outputName, description, new HashMap<>()); + this(name, outputName, description, new LinkedHashMap<>()); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java index cd6035204e2..41204ce6e9a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import javax.annotation.concurrent.Immutable; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; @@ -14,10 +13,10 @@ import static java.util.Collections.unmodifiableMap; /** * Models a metric set containing a set of metrics and child metric sets. + * This should be immutable. * * @author gjoranv */ -@Immutable public class MetricSet { private final String id; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index d2294402cd2..59fb541158e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -144,6 +144,11 @@ public class VespaMetricSet { metrics.add(new Metric("serverNumConnections.max")); metrics.add(new Metric("serverNumConnections.last")); + metrics.add(new Metric("serverBytesReceived.sum")); + metrics.add(new Metric("serverBytesReceived.count")); + metrics.add(new Metric("serverBytesSent.sum")); + metrics.add(new Metric("serverBytesSent.count")); + { List<String> suffixes = List.of("sum", "count", "last", "min", "max"); addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffixes); @@ -330,6 +335,7 @@ public class VespaMetricSet { metrics.add(new Metric("dispatch_internal.rate")); metrics.add(new Metric("dispatch_fdispatch.rate")); addMetric(metrics, "jdisc.render.latency", Set.of("min", "max", "count", "sum", "last", "average")); + addMetric(metrics, "query_item_count", Set.of("max", "sum", "count")); metrics.add(new Metric("totalhits_per_query.max")); metrics.add(new Metric("totalhits_per_query.sum")); @@ -474,11 +480,17 @@ public class VespaMetricSet { // resource usage metrics.add(new Metric("content.proton.resource_usage.disk.average")); - metrics.add(new Metric("content.proton.resource_usage.disk_utilization.average")); + metrics.add(new Metric("content.proton.resource_usage.disk_usage.total.max")); + metrics.add(new Metric("content.proton.resource_usage.disk_usage.total_utilization.max")); + metrics.add(new Metric("content.proton.resource_usage.disk_usage.transient.max")); metrics.add(new Metric("content.proton.resource_usage.memory.average")); - metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average")); - metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); - metrics.add(new Metric("content.proton.resource_usage.transient_disk.average")); + metrics.add(new Metric("content.proton.resource_usage.memory_usage.total.max")); + metrics.add(new Metric("content.proton.resource_usage.memory_usage.total_utilization.max")); + metrics.add(new Metric("content.proton.resource_usage.memory_usage.transient.max")); + metrics.add(new Metric("content.proton.resource_usage.disk_utilization.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.resource_usage.transient_disk.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); @@ -487,6 +499,23 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.max")); + // CPU util + metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.setup.count")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.read.count")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.write.count")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.compact.count")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.max")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.sum")); + metrics.add(new Metric("content.proton.resource_usage.cpu_util.other.count")); + // transaction log metrics.add(new Metric("content.proton.transactionlog.entries.average")); metrics.add(new Metric("content.proton.transactionlog.disk_usage.average")); @@ -555,9 +584,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count")); - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); // TODO: Consider remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.max")); - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate")); @@ -581,8 +608,6 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.rate")); // TODO: Consider remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.limited_queries.rate")); @@ -603,8 +628,7 @@ public class VespaMetricSet { // TODO: For the purpose of this file and likely elsewhere, all but the last aggregate specifier, // TODO: such as 'average' and 'sum' in the metric names below are just confusing and can be mentally - // TODO: disregarded when considering metric names. Consider cleaning up for Vespa 8. - // TODO Vespa 8 all metrics with .sum in the name should have that removed. + // TODO: disregarded when considering metric names. Consider cleaning up for Vespa 9. metrics.add(new Metric("vds.datastored.alldisks.docs.average")); metrics.add(new Metric("vds.datastored.alldisks.bytes.average")); metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.max")); @@ -636,6 +660,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.count")); + metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.max")); + metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.sum")); + metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.count")); @@ -651,6 +678,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.count")); + metrics.add(new Metric("vds.filestor.alldisks.allstripes.throttled_rpc_direct_dispatches.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allstripes.throttled_persistence_thread_polls.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allstripes.timeouts_waiting_for_throttle_token.rate")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.max")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java new file mode 100644 index 00000000000..1642a8cb3cc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import aQute.bnd.header.Parameters; +import aQute.bnd.osgi.Domain; +import aQute.bnd.version.VersionRange; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.vespa.model.VespaModel; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.logging.Level; +import java.util.regex.Pattern; + +/** + * Base class for OSGi bundle validator. Uses BND library for some of the validation. + * + * @author bjorncs + */ +public abstract class AbstractBundleValidator extends Validator { + + protected abstract void validateManifest(DeployState state, JarFile jar, Manifest mf); + protected abstract void validatePomXml(DeployState state, JarFile jar, Document pom); + + @Override + public final void validate(VespaModel model, DeployState state) { + ApplicationPackage app = state.getApplicationPackage(); + for (ComponentInfo info : app.getComponentsInfo(state.getVespaVersion())) { + Path path = Path.fromString(info.getPathRelativeToAppDir()); + try { + state.getDeployLogger() + .log(Level.FINE, String.format("Validating bundle at '%s'", path)); + JarFile jarFile = new JarFile(app.getFileReference(path)); + validateJarFile(state, jarFile); + } catch (IOException e) { + throw new IllegalArgumentException( + "Failed to validate JAR file '" + path.last() + "'", e); + } + } + } + + final void validateJarFile(DeployState state, JarFile jar) throws IOException { + Manifest manifest = jar.getManifest(); + if (manifest == null) { + throw new IllegalArgumentException("Non-existing or invalid manifest in " + filename(jar)); + } + validateManifest(state, jar, manifest); + getPomXmlContent(state.getDeployLogger(), jar) + .ifPresent(pom -> validatePomXml(state, jar, pom)); + } + + protected final String filename(JarFile jarFile) { return Paths.get(jarFile.getName()).getFileName().toString(); } + + protected final void forEachPomXmlElement(Document pom, String xpath, Consumer<Element> consumer) throws XPathExpressionException { + NodeList dependencies = (NodeList) XPathFactory.newDefaultInstance().newXPath() + .compile("/project/" + xpath) + .evaluate(pom, XPathConstants.NODESET); + for (int i = 0; i < dependencies.getLength(); i++) { + Element element = (Element) dependencies.item(i); + consumer.accept(element); + } + } + + protected final void forEachImportPackage(Manifest mf, BiConsumer<String, VersionRange> consumer) { + Parameters importPackage = Domain.domain(mf).getImportPackage(); + importPackage.forEach((packageName, attrs) -> { + VersionRange versionRange = attrs.getVersion() != null + ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) + : null; + consumer.accept(packageName, versionRange); + }); + } + + protected final void log(DeployState state, Level level, String fmt, Object... args) { + state.getDeployLogger().logApplicationPackage(level, String.format(fmt, args)); + } + + private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); + private Optional<Document> getPomXmlContent(DeployLogger deployLogger, JarFile jar) { + return jar.stream() + .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) + .findFirst() + .map(f -> { + try { + String text = new String(jar.getInputStream(f).readAllBytes()); + return DocumentBuilderFactory.newDefaultInstance().newDocumentBuilder() + .parse(new InputSource(new StringReader(text))); + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } catch (SAXException e) { + deployLogger.log(Level.INFO, String.format("Unable to parse pom.xml from %s", filename(jar))); + return null; + } catch (IOException e) { + deployLogger.log(Level.INFO, + String.format("Unable to read '%s' from '%s'", f.getName(), jar.getName())); + return null; + } + }); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java deleted file mode 100644 index a07e07169d1..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; - -import java.util.ArrayList; -import java.util.List; - -import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator.needsAccessControlValidation; -import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection; - -/** - * @author gjoranv - */ -public class AwsAccessControlValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - - if (! needsAccessControlValidation(model, deployState)) return; - if(! deployState.zone().getCloud().requireAccessControl()) return; - - List<String> offendingClusters = new ArrayList<>(); - for (var cluster : model.getContainerClusters().values()) { - var http = cluster.getHttp(); - if (http == null - || ! http.getAccessControl().isPresent() - || ! http.getAccessControl().get().writeEnabled - || ! http.getAccessControl().get().readEnabled) - - if (hasHandlerThatNeedsProtection(cluster)) - offendingClusters.add(cluster.getName()); - } - if (! offendingClusters.isEmpty()) - deployState.validationOverrides() - .invalid(ValidationId.accessControl, - "Access-control must be enabled for read/write operations to container clusters in AWS production zones: " + - mkString(offendingClusters, "[", ", ", "]"), deployState.now()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index b6b9190fedf..a1024425124 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -1,17 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import aQute.bnd.header.Parameters; -import aQute.bnd.osgi.Domain; import aQute.bnd.version.VersionRange; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.path.Path; -import com.yahoo.vespa.model.VespaModel; +import org.w3c.dom.Document; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -25,44 +18,21 @@ import java.util.jar.Attributes; import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.logging.Level; +import java.util.regex.Pattern; +import java.util.stream.Collectors; /** - * A validator for bundles. Uses BND library for some of the validation. + * A validator for bundles. * * @author hmusum * @author bjorncs */ -public class BundleValidator extends Validator { +public class BundleValidator extends AbstractBundleValidator { public BundleValidator() {} @Override - public void validate(VespaModel model, DeployState deployState) { - ApplicationPackage app = deployState.getApplicationPackage(); - for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) { - try { - Path path = Path.fromString(info.getPathRelativeToAppDir()); - DeployLogger deployLogger = deployState.getDeployLogger(); - deployLogger.log(Level.FINE, String.format("Validating bundle at '%s'", path)); - JarFile jarFile = new JarFile(app.getFileReference(path)); - validateJarFile(deployLogger, jarFile); - } catch (IOException e) { - throw new IllegalArgumentException( - "Failed to validate JAR file '" + info.getPathRelativeToAppDir() + "'", e); - } - } - } - - void validateJarFile(DeployLogger deployLogger, JarFile jarFile) throws IOException { - Manifest manifest = jarFile.getManifest(); - String jarPath = jarFile.getName(); - if (manifest == null) { - throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarPath); - } - validateManifest(deployLogger, jarPath, manifest); - } - - void validateManifest(DeployLogger deployLogger, String jarPath, Manifest mf) { + protected void validateManifest(DeployState state, JarFile jar, Manifest mf) { // Check for required OSGI headers Attributes attributes = mf.getMainAttributes(); HashSet<String> mfAttributes = new HashSet<>(); @@ -74,69 +44,65 @@ public class BundleValidator extends Validator { for (String header : requiredOSGIHeaders) { if (!mfAttributes.contains(header)) { throw new IllegalArgumentException("Required OSGI header '" + header + - "' was not found in manifest in '" + jarPath + "'"); + "' was not found in manifest in '" + filename(jar) + "'"); } } if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { - deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + jarPath + - ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml."); + log(state, Level.WARNING, + "Deploying snapshot bundle " + filename(jar) + ".\nTo use this bundle, you must include the " + + "qualifier 'SNAPSHOT' in the version specification in services.xml."); } if (attributes.getValue("Import-Package") != null) { - validateImportedPackages(deployLogger, jarPath, mf); + validateImportedPackages(state, jar, mf); } } - private static void validateImportedPackages(DeployLogger deployLogger, String jarPath, Manifest manifest) { - Domain osgiHeaders = Domain.domain(manifest); - Parameters importPackage = osgiHeaders.getImportPackage(); - Map<DeprecatedArtifact, List<String>> deprecatedPackagesInUse = new HashMap<>(); - - importPackage.forEach((packageName, attrs) -> { - VersionRange versionRange = attrs.getVersion() != null - ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) - : null; - - for (DeprecatedArtifact deprecatedArtifact : DeprecatedArtifact.values()) { - if (deprecatedArtifact.javaPackages.contains(packageName) - && (versionRange == null || deprecatedArtifact.versionDiscriminator.test(versionRange))) { - deprecatedPackagesInUse.computeIfAbsent(deprecatedArtifact, __ -> new ArrayList<>()) - .add(packageName); + @Override protected void validatePomXml(DeployState state, JarFile jar, Document pom) {} + + private void validateImportedPackages(DeployState state, JarFile jar, Manifest manifest) { + Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); + forEachImportPackage(manifest, (packageName, versionRange) -> { + for (DeprecatedProvidedBundle deprecatedBundle : DeprecatedProvidedBundle.values()) { + for (Predicate<String> matcher : deprecatedBundle.javaPackageMatchers) { + if (matcher.test(packageName) + && (versionRange == null || deprecatedBundle.versionDiscriminator.test(versionRange))) { + deprecatedPackagesInUse.computeIfAbsent(deprecatedBundle, __ -> new ArrayList<>()) + .add(packageName); + } } } }); - deprecatedPackagesInUse.forEach((artifact, packagesInUse) -> { - deployLogger.logApplicationPackage(Level.WARNING, - String.format("For JAR file '%s': \n" + - "Manifest imports the following Java packages from '%s': %s. \n" + - "%s", - jarPath, artifact.name, packagesInUse, artifact.description)); + log(state, Level.WARNING, "JAR file '%s' imports the packages %s from '%s'. \n%s", + filename(jar), packagesInUse, artifact.name, artifact.description); }); } - private enum DeprecatedArtifact { + private enum DeprecatedProvidedBundle { ORG_JSON("org.json:json", - "The org.json library will no longer provided by jdisc runtime on Vespa 8. " + - "See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", - Set.of("org.json")); + "This bundle is no longer provided on Vespa 8 - " + + "see https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", + Set.of("org\\.json")); final String name; - final Collection<String> javaPackages; + final Collection<Predicate<String>> javaPackageMatchers; final Predicate<VersionRange> versionDiscriminator; final String description; - DeprecatedArtifact(String name, String description, Collection<String> javaPackages) { - this(name, description, __ -> true, javaPackages); + DeprecatedProvidedBundle(String name, String description, Collection<String> javaPackagePatterns) { + this(name, description, __ -> true, javaPackagePatterns); } - DeprecatedArtifact(String name, - String description, - Predicate<VersionRange> versionDiscriminator, - Collection<String> javaPackages) { + DeprecatedProvidedBundle(String name, + String description, + Predicate<VersionRange> versionDiscriminator, + Collection<String> javaPackagePatterns) { this.name = name; - this.javaPackages = javaPackages; + this.javaPackageMatchers = javaPackagePatterns.stream() + .map(s -> Pattern.compile(s).asMatchPredicate()) + .collect(Collectors.toList()); this.versionDiscriminator = versionDiscriminator; this.description = description; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index e2b08a621d1..b977e4c657e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -34,7 +34,7 @@ public class ComplexAttributeFieldsValidator extends Validator { } SearchCluster searchCluster = (SearchCluster) cluster; for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) { - validateComplexFields(searchCluster.getClusterName(), spec.getSearchDefinition().getSearch()); + validateComplexFields(searchCluster.getClusterName(), spec.getSchema()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index ef0a8ddda8e..66da43856b1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -90,7 +90,7 @@ public class ConstantTensorJsonValidator { validateTensorValue(); } } else { - throw new InvalidConstantTensor(parser, "Only 'address' or 'value' fields are permitted within a cell object"); + throw new InvalidConstantTensorException(parser, "Only 'address' or 'value' fields are permitted within a cell object"); } } @@ -110,11 +110,11 @@ public class ConstantTensorJsonValidator { String dimensionName = parser.getCurrentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { - throw new InvalidConstantTensor(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); } if (!cellDimensions.contains(dimensionName)) { - throw new InvalidConstantTensor(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); } cellDimensions.remove(dimensionName); @@ -122,7 +122,7 @@ public class ConstantTensorJsonValidator { } if (!cellDimensions.isEmpty()) { - throw new InvalidConstantTensor(parser, String.format("Tensor address missing dimension(s) %s", Joiner.on(", ").join(cellDimensions))); + throw new InvalidConstantTensorException(parser, String.format("Tensor address missing dimension(s) %s", Joiner.on(", ").join(cellDimensions))); } } @@ -134,7 +134,7 @@ public class ConstantTensorJsonValidator { private void validateLabel(TensorType.Dimension dimension) throws IOException { JsonToken token = parser.nextToken(); if (token != JsonToken.VALUE_STRING) - throw new InvalidConstantTensor(parser, String.format("Tensor label is not a string (%s)", token.toString())); + throw new InvalidConstantTensorException(parser, String.format("Tensor label is not a string (%s)", token.toString())); if (dimension instanceof TensorType.IndexedBoundDimension) { validateBoundIndex((TensorType.IndexedBoundDimension) dimension); @@ -148,7 +148,7 @@ public class ConstantTensorJsonValidator { try { int value = Integer.parseInt(parser.getValueAsString()); if (value >= dimension.size().get()) - throw new InvalidConstantTensor(parser, String.format("Index %s not within limits of bound dimension '%s'", value, dimension.name())); + throw new InvalidConstantTensorException(parser, String.format("Index %s not within limits of bound dimension '%s'", value, dimension.name())); } catch (NumberFormatException e) { throwCoordinateIsNotInteger(parser.getValueAsString(), dimension.name()); } @@ -166,14 +166,14 @@ public class ConstantTensorJsonValidator { } private void throwCoordinateIsNotInteger(String value, String dimensionName) { - throw new InvalidConstantTensor(parser, String.format("Index '%s' for dimension '%s' is not an integer", value, dimensionName)); + throw new InvalidConstantTensorException(parser, String.format("Index '%s' for dimension '%s' is not an integer", value, dimensionName)); } private void validateTensorValue() throws IOException { JsonToken token = parser.nextToken(); if (token != JsonToken.VALUE_NUMBER_FLOAT && token != JsonToken.VALUE_NUMBER_INT) { - throw new InvalidConstantTensor(parser, String.format("Tensor value is not a number (%s)", token.toString())); + throw new InvalidConstantTensorException(parser, String.format("Tensor value is not a number (%s)", token.toString())); } } @@ -187,7 +187,7 @@ public class ConstantTensorJsonValidator { private void assertTokenIs(JsonToken token, JsonToken wantedToken) { if (token != wantedToken) { - throw new InvalidConstantTensor(parser, String.format("Expected JSON token %s, but got %s", wantedToken.toString(), token.toString())); + throw new InvalidConstantTensorException(parser, String.format("Expected JSON token %s, but got %s", wantedToken.toString(), token.toString())); } } @@ -195,17 +195,17 @@ public class ConstantTensorJsonValidator { String actualFieldName = parser.getCurrentName(); if (!actualFieldName.equals(wantedFieldName)) { - throw new InvalidConstantTensor(parser, String.format("Expected field name '%s', got '%s'", wantedFieldName, actualFieldName)); + throw new InvalidConstantTensorException(parser, String.format("Expected field name '%s', got '%s'", wantedFieldName, actualFieldName)); } } - static class InvalidConstantTensor extends RuntimeException { + static class InvalidConstantTensorException extends IllegalArgumentException { - InvalidConstantTensor(JsonParser parser, String message) { + InvalidConstantTensorException(JsonParser parser, String message) { super(message + " " + parser.getCurrentLocation().toString()); } - InvalidConstantTensor(JsonParser parser, Exception base) { + InvalidConstantTensorException(JsonParser parser, Exception base) { super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); } @@ -220,7 +220,7 @@ public class ConstantTensorJsonValidator { try { lambda.invoke(); } catch (IOException e) { - throw new InvalidConstantTensor(parser, e); + throw new InvalidConstantTensorException(parser, e); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 841dbb204fd..e4a64e8d476 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.model.VespaModel; import java.math.BigDecimal; +import java.util.LinkedHashSet; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -61,7 +62,7 @@ public class QuotaValidator extends Validator { .map(hostSpec -> hostSpec.membership().orElseThrow().cluster()) .filter(cluster -> cluster.type() == ClusterSpec.Type.admin) .map(ClusterSpec::id) - .collect(Collectors.toUnmodifiableSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } /** Check that all clusters in the application do not exceed the quota max cluster size. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index e9b23461527..82c793af70a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -7,9 +7,9 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; +import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; -import com.yahoo.vespa.model.search.NamedSchema; +import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; import java.io.FileNotFoundException; @@ -36,8 +36,8 @@ public class RankingConstantsValidator extends Validator { } } - static class TensorValidationFailed extends RuntimeException { - TensorValidationFailed(String message) { + static class TensorValidationException extends IllegalArgumentException { + TensorValidationException(String message) { super(message); } } @@ -47,18 +47,18 @@ public class RankingConstantsValidator extends Validator { ApplicationPackage applicationPackage = deployState.getApplicationPackage(); ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):"); - for (NamedSchema sd : deployState.getSchemas()) { - for (RankingConstant rc : sd.getSearch().rankingConstants().asMap().values()) { + for (Schema schema : deployState.getSchemas()) { + for (RankingConstant rc : schema.rankingConstants().asMap().values()) { try { validateRankingConstant(rc, applicationPackage); - } catch (InvalidConstantTensor | FileNotFoundException ex) { + } catch (InvalidConstantTensorException | FileNotFoundException ex) { exceptionMessageCollector.add(ex, rc.getName(), rc.getFileName()); } } } if (exceptionMessageCollector.exceptionsOccurred) { - throw new TensorValidationFailed(exceptionMessageCollector.combinedMessage); + throw new TensorValidationException(exceptionMessageCollector.combinedMessage); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index 14f5c345025..976ed0a0219 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -11,11 +11,11 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; +import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.NamedSchema; import java.util.List; @@ -35,26 +35,26 @@ public class SearchDataTypeValidator extends Validator { continue; } for (AbstractSearchCluster.SchemaSpec spec : cluster.getLocalSDS()) { - SDDocumentType docType = spec.getSearchDefinition().getSearch().getDocument(); + SDDocumentType docType = spec.getSchema().getDocument(); if (docType == null) { continue; } - validateDocument(cluster, spec.getSearchDefinition(), docType); + validateDocument(cluster, spec.getSchema(), docType); } } } - private void validateDocument(AbstractSearchCluster cluster, NamedSchema def, SDDocumentType doc) { + private void validateDocument(AbstractSearchCluster cluster, Schema schema, SDDocumentType doc) { for (SDDocumentType child : doc.getTypes()) { - validateDocument(cluster, def, child); + validateDocument(cluster, schema, child); } for (Field field : doc.fieldSet()) { DataType fieldType = field.getDataType(); - disallowIndexingOfMaps(cluster, def, field); + disallowIndexingOfMaps(cluster, schema, field); if ( ! isSupportedInSearchClusters(fieldType)) { throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " + - "clusters (field '" + field.getName() + "' in definition '" + - def.getName() + "' for cluster '" + cluster.getClusterName() + "')."); + "clusters (field '" + field.getName() + "' in schema '" + + schema.getName() + "' for cluster '" + cluster.getClusterName() + "')."); } } } @@ -84,12 +84,12 @@ public class SearchDataTypeValidator extends Validator { } } - private void disallowIndexingOfMaps(AbstractSearchCluster cluster, NamedSchema def, Field field) { + private void disallowIndexingOfMaps(AbstractSearchCluster cluster, Schema schema, Field field) { DataType fieldType = field.getDataType(); if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) { throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " + "clusters (field '" + field.getName() + "' in definition '" + - def.getName() + "' for cluster '" + cluster.getClusterName() + "')."); + schema.getName() + "' for cluster '" + cluster.getClusterName() + "')."); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 08dc73a1bd0..4ba809a58ba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.model.application.validation.change.RedundancyIncreaseVal import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; -import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator; import com.yahoo.vespa.model.application.validation.first.RedundancyOnFirstDeploymentValidator; import java.time.Instant; @@ -50,6 +49,13 @@ import static java.util.stream.Collectors.toList; */ public class Validation { + private final List<Validator> additionalValidators; + + public Validation() { this(List.of()); } + + /** Create instance taking additional validators (e.g for cloud applications) */ + public Validation(List<Validator> additionalValidators) { this.additionalValidators = additionalValidators; } + /** * Validates the model supplied, and if there already exists a model for the application validates changes * between the previous and current model @@ -57,7 +63,7 @@ public class Validation { * @return a list of required changes needed to make this configuration live * @throws ValidationOverrides.ValidationException if the change fails validation */ - public static List<ConfigChangeAction> validate(VespaModel model, ValidationParameters validationParameters, DeployState deployState) { + public List<ConfigChangeAction> validate(VespaModel model, ValidationParameters validationParameters, DeployState deployState) { if (validationParameters.checkRouting()) { new RoutingValidator().validate(model, deployState); new RoutingSelectorValidator().validate(model, deployState); @@ -76,10 +82,11 @@ public class Validation { new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); new CloudWatchValidator().validate(model, deployState); - new AwsAccessControlValidator().validate(model, deployState); new QuotaValidator().validate(model, deployState); new UriBindingsValidator().validate(model, deployState); + additionalValidators.forEach(v -> v.validate(model, deployState)); + List<ConfigChangeAction> result = Collections.emptyList(); if (deployState.getProperties().isFirstTimeDeployment()) { validateFirstTimeDeployment(model, deployState); @@ -126,7 +133,6 @@ public class Validation { } private static void validateFirstTimeDeployment(VespaModel model, DeployState deployState) { - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); new RedundancyOnFirstDeploymentValidator().validate(model, deployState); } @@ -134,7 +140,7 @@ public class Validation { Set<ClusterSpec.Id> clustersToBeRestarted = actions.stream() .filter(action -> action.getType() == ConfigChangeAction.Type.RESTART) .map(action -> action.clusterId()) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); for (var clusterToRestart : clustersToBeRestarted) { var containerCluster = model.getContainerClusters().get(clusterToRestart.value()); if (containerCluster != null) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java index 30abb9b445f..a3872d9354b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -13,13 +13,12 @@ import com.yahoo.vespa.model.search.SearchNode; import java.time.Instant; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static java.util.stream.Collectors.toSet; - /** * Returns any change to the indexing mode of a cluster. * @@ -91,7 +90,7 @@ public class IndexingModeChangeValidator implements ChangeValidator { private static Set<String> toDocumentTypeNames(List<NewDocumentType> types) { return types.stream() .map(type -> type.getFullName().getName()) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java deleted file mode 100644 index dd6e6ad590d..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation.first; - -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.model.ConfigModelContext.ApplicationType; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.Validator; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.ContainerCluster; - -import java.util.ArrayList; -import java.util.List; - -import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.config.provision.InstanceName.defaultName; -import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection; - -/** - * Validates that hosted applications in prod zones have write protection enabled. - * - * @author gjoranv - */ -public class AccessControlOnFirstDeploymentValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - - if (! needsAccessControlValidation(model, deployState)) return; - - List<String> offendingClusters = new ArrayList<>(); - for (ContainerCluster<? extends Container> c : model.getContainerClusters().values()) { - if (! (c instanceof ApplicationContainerCluster)) continue; - ApplicationContainerCluster cluster = (ApplicationContainerCluster)c; - if (cluster.getHttp() == null - || ! cluster.getHttp().getAccessControl().isPresent() - || ! cluster.getHttp().getAccessControl().get().writeEnabled) - - if (hasHandlerThatNeedsProtection(cluster)) - offendingClusters.add(cluster.getName()); - } - if (! offendingClusters.isEmpty()) - deployState.validationOverrides().invalid(ValidationId.accessControl, - "Access-control must be enabled for write operations to container clusters in production zones: " + - mkString(offendingClusters, "[", ", ", "]"), deployState.now()); - } - - public static boolean needsAccessControlValidation(VespaModel model, DeployState deployState) { - if (! deployState.isHosted()) return false; - if (! deployState.zone().environment().isProduction()) return false; - if (deployState.zone().system().isPublic()) return false; - if (! deployState.getApplicationPackage().getApplicationId().instance().equals(defaultName())) return false; - if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return false; - - return true; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java index b163f9f0699..06747bdbd2c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java @@ -11,8 +11,9 @@ import java.util.regex.Pattern; * @author Tony Vaagenes */ public class BinaryScaledAmountParser { - //The pattern must match the one given in the schema - private static Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); + + // The pattern must match the one given in the schema + private static final Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); public static BinaryScaledAmount parse(String valueString) { Matcher matcher = pattern.matcher(valueString); @@ -34,4 +35,5 @@ public class BinaryScaledAmountParser { return BinaryPrefix.fromSymbol(binaryPrefixString.toUpperCase().charAt(0)); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java index 1317f96e109..4a46b62d096 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java @@ -10,8 +10,9 @@ import static com.yahoo.text.Lowercase.toLowerCase; * @author Tony Vaagenes */ public class BinaryUnit { - //The pattern must match the one given in the schema - private static Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); + + // The pattern must match the one given in the schema + private static final Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); public static double valueOf(String valueString) { Matcher matcher = pattern.matcher(valueString); @@ -28,13 +29,13 @@ public class BinaryUnit { } private static double unitToValue(char unit) { - final char units[] = {'k', 'm', 'g'}; - for (int i=0; i<units.length; ++i) { + char[] units = {'k', 'm', 'g'}; + for (int i = 0; i < units.length; ++i) { if (units[i] == unit) { return Math.pow(2, 10*(i+1)); } } - throw new RuntimeException("No such unit: '" + unit + "'"); + throw new IllegalArgumentException("No such unit: '" + unit + "'"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 56f462805bd..4c74282c061 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -45,7 +45,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu this.configServerSpecs = configServerSpecs; } - List<Configserver> getConfigServersFromSpec(DeployLogger deployLogger, AbstractConfigProducer<?> parent) { + List<Configserver> getConfigServersFromSpec(DeployState deployState, AbstractConfigProducer<?> parent) { List<Configserver> configservers = new ArrayList<>(); for (ConfigServerSpec spec : configServerSpecs) { HostSystem hostSystem = parent.hostSystem(); @@ -54,7 +54,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu Configserver configserver = new Configserver(parent, spec.getHostName(), spec.getConfigServerPort()); configserver.setHostResource(host); configserver.setBasePort(configserver.getWantedPort()); - configserver.initService(deployLogger); + configserver.initService(deployState); configservers.add(configserver); } return configservers; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 5aaba9550d2..8e7c543b67b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -53,7 +53,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) { List<Configserver> configservers; if (multitenant) - configservers = getConfigServersFromSpec(deployState.getDeployLogger(), admin); + configservers = getConfigServersFromSpec(deployState, admin); else configservers = getConfigServers(deployState, admin, adminE); if (configservers.isEmpty() && ! multitenant) @@ -129,7 +129,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { SimpleConfigProducer<?> configServers = new SimpleConfigProducer<>(parent, "configservers"); Configserver configServer = new Configserver(configServers, "configserver", Configserver.defaultRpcPort); configServer.setHostResource(parent.hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); - configServer.initService(deployState.getDeployLogger()); + configServer.initService(deployState); return List.of(configServer); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 41baf2db3aa..a91a7949ad8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; @@ -44,7 +43,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { @Override protected void doBuildAdmin(DeployState deployState, Admin admin, Element w3cAdminElement) { ModelElement adminElement = new ModelElement(w3cAdminElement); - admin.addConfigservers(getConfigServersFromSpec(deployState.getDeployLogger(), admin)); + admin.addConfigservers(getConfigServersFromSpec(deployState, admin)); // Note: These two elements only exists in admin version 4.0 // This build handles admin version 3.0 by ignoring its content (as the content is not useful) @@ -79,13 +78,13 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) - Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts); + Logserver logserver = createLogserver(deployState, admin, hosts); createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); } else if (containerModels.iterator().hasNext()) { List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) - createLogserver(deployState.getDeployLogger(), admin, hosts); + createLogserver(deployState, admin, hosts); } else { context.getDeployLogger().logApplicationPackage(Level.INFO, "No container host available to use for running logserver"); } @@ -110,9 +109,9 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { LogserverContainer container = new LogserverContainer(logServerCluster, deployState); container.setHostResource(hostResource); - container.initService(deployState.getDeployLogger()); + container.initService(deployState); logServerCluster.addContainer(container); - admin.addAndInitializeService(deployState.getDeployLogger(), hostResource, container); + admin.addAndInitializeService(deployState, hostResource, container); admin.setLogserverContainerCluster(logServerCluster); context.getConfigModelRepoAdder().add(logserverClusterModel); } @@ -169,11 +168,11 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { return hosts.subList(0, Math.min(count, hosts.size())); } - private Logserver createLogserver(DeployLogger deployLogger, Admin admin, Collection<HostResource> hosts) { + private Logserver createLogserver(DeployState deployState, Admin admin, Collection<HostResource> hosts) { Logserver logserver = new Logserver(admin); logserver.setHostResource(hosts.iterator().next()); admin.setLogserver(logserver); - logserver.initService(deployLogger); + logserver.initService(deployState); return logserver; } @@ -185,7 +184,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Slobrok slobrok = new Slobrok(admin, index++, deployState.featureFlags()); slobrok.setHostResource(host); slobroks.add(slobrok); - slobrok.initService(deployState.getDeployLogger()); + slobrok.initService(deployState); } admin.addSlobroks(slobroks); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 0b32c77fe58..bbc18ab2254 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -15,7 +15,9 @@ import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; +import java.util.List; import java.util.Set; +import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_2; @@ -49,7 +51,12 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< for (Element binding : XML.getChildren(handlerElement, "binding")) addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); - for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding")) + List<Element> clientBindingsElements = XML.getChildren(handlerElement, "clientBinding"); + if (! clientBindingsElements.isEmpty()) { + deployState.getDeployLogger().logApplicationPackage( + Level.WARNING, "The 'clientBindings' element is deprecated for removal in Vespa 8, with no replacement"); + } + for (Element clientBinding : clientBindingsElements) handler.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(clientBinding))); DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java index cecc258a3b8..40e457ba69b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java @@ -13,7 +13,6 @@ import org.w3c.dom.Element; * producer in between. This should not be used by new model plugins. * * @author Ulf Lilleengen - * @since 5.1 */ public abstract class LegacyConfigModelBuilder<MODEL extends ConfigModel> extends ConfigModelBuilder<MODEL> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 81923db15dd..0be3ac01f71 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -69,20 +69,20 @@ public class VespaDomBuilder extends VespaModelBuilder { // TODO Move and change scope public static List<String> getHostAliases(NodeList hostAliases) { List<String> aliases = new LinkedList<>(); - for (int i=0; i < hostAliases.getLength(); i++) { + for (int i = 0; i < hostAliases.getLength(); i++) { Node n = hostAliases.item(i); if (! (n instanceof Element)) { continue; } Element e = (Element)n; if (! e.getNodeName().equals("alias")) { - throw new RuntimeException("Unexpected tag: '" + e.getNodeName() + "' at node " + - XML.getNodePath(e, " > ") + ", expected 'alias'."); + throw new IllegalArgumentException("Unexpected tag: '" + e.getNodeName() + "' at node " + + XML.getNodePath(e, " > ") + ", expected 'alias'."); } String alias = e.getFirstChild().getNodeValue(); if ((alias == null) || (alias.equals(""))) { - throw new RuntimeException("Missing value for the alias tag at node " + - XML.getNodePath(e, " > ") + "'."); + throw new IllegalArgumentException("Missing value for the alias tag at node " + + XML.getNodePath(e, " > ") + "'."); } aliases.add(alias); } @@ -113,7 +113,6 @@ public class VespaDomBuilder extends VespaModelBuilder { * include hostalias, baseport and user config overrides generically. * * @param <T> an {@link com.yahoo.config.model.producer.AbstractConfigProducer} - * @author vegardh */ public static abstract class DomConfigProducerBuilder<T extends AbstractConfigProducer<?>> { @@ -184,7 +183,7 @@ public class VespaDomBuilder extends VespaModelBuilder { // This depends on which constructor in AbstractService is used, but the best way // is to let this method do initialize. if (!t.isInitialized()) { - t.initService(deployState.getDeployLogger()); + t.initService(deployState); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java index 560ede554e6..0d54b7b49f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java @@ -134,15 +134,15 @@ public class ComponentsBuilder<T extends ChainedComponent<?>> { private void ensureNotDefinition(String componentName, Element componentSpec) { if (componentSpec.getAttributes().getLength() > 1 || !XML.getChildren(componentSpec).isEmpty()) - throw new RuntimeException("Expecting " + componentName + - " to be a reference to a global component with the same name," + - " so no additional attributes or nested elements are allowed"); + throw new IllegalArgumentException("Expecting " + componentName + + " to be a reference to a global component with the same name," + + " so no additional attributes or nested elements are allowed"); } private void ensureTypesMatch(ComponentType type1, ComponentType type2, String componentName) { if (!type1.equals(type2)) { - throw new RuntimeException("Two different types declared for the component with name '" + componentName + "' (" - + type1.name + " != " + type2.name + ")."); + throw new IllegalArgumentException("Two different types declared for the component with name '" + componentName + + "' (" + type1.name + " != " + type2.name + ")."); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java index 92887bc3227..b62c1a01fa3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java @@ -6,9 +6,11 @@ import java.lang.reflect.InvocationTargetException; /** * Utility class for instantiating a builder using reflection. + * * @author Tony Vaagenes */ public class DomBuilderCreator { + public static <T> T create(Class<T> builderClass, Object... parameters) { try { return getConstructor(builderClass).newInstance(parameters); @@ -23,4 +25,5 @@ public class DomBuilderCreator { assert(constructors.length == 1); return (Constructor<T>) constructors[0]; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java index 9db28c83a93..19c5c489f7e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java @@ -157,7 +157,6 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { } } - @SuppressWarnings("deprecation") private Provider buildProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) { @@ -166,7 +165,7 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { } else if (LocalProviderSpec.includesType(providerReader.type)) { return buildLocalProvider(specWithoutInnerSearchers, providerReader, federationOptions); } else { - throw new RuntimeException("Unknown provider type '" + providerReader.type + "'"); + throw new IllegalArgumentException("Unknown provider type '" + providerReader.type + "'"); } } @@ -194,7 +193,7 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { private void ensureEmpty(ComponentId componentId, Object... objects) { for (Object object : objects) { if (object != null) { - throw new RuntimeException("Invalid provider option in provider '" + componentId + "': value='" + object + "'"); + throw new IllegalArgumentException("Invalid provider option in provider '" + componentId + "': value='" + object + "'"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 7c386875d02..ba5e8ed5544 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -11,6 +11,10 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.util.Optional; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + /** * A container that is typically used by container clusters set up from the user application. * @@ -24,6 +28,7 @@ public final class ApplicationContainer extends Container implements private final boolean isHostedVespa; private final boolean enableServerOcspStapling; + private final boolean useQrserverServiceName; public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -33,6 +38,7 @@ public final class ApplicationContainer extends Container implements super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); + this.useQrserverServiceName = deployState.featureFlags().useQrserverServiceName(); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -56,7 +62,7 @@ public final class ApplicationContainer extends Container implements if (parent instanceof ContainerCluster) { ContainerCluster<?> cluster = (ContainerCluster<?>)parent; // TODO: The 'qrserver' name is retained for legacy reasons (e.g. system tests and log parsing). - if (cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) { + if (useQrserverServiceName && cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) { return ContainerServiceType.QRSERVER; } } @@ -99,4 +105,11 @@ public final class ApplicationContainer extends Container implements return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); } + @Override + public Optional<String> getPreShutdownCommand() { + int preshutdownTimeoutSeconds = 360; + int rpcTimeoutSeconds = preshutdownTimeoutSeconds + 10; + String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + preshutdownTimeoutSeconds; + return Optional.of(getDefaults().underVespaHome("bin/vespa-rpc-invoke") + " " + rpcParams); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index c4d420f2d44..8f95e390b07 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -40,13 +40,13 @@ import com.yahoo.vespa.model.utils.FileSender; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; /** @@ -94,14 +94,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private List<ApplicationClusterEndpoint> endpointList = List.of(); public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { - super(parent, configSubId, clusterId, deployState, true); + super(parent, configSubId, clusterId, deployState, true, 10); this.tlsClientAuthority = deployState.tlsClientAuthority(); - previousHosts = deployState.getPreviousModel().stream() - .map(Model::allocatedHosts) - .map(AllocatedHosts::getHosts) - .flatMap(Collection::stream) - .map(HostSpec::hostname) - .collect(Collectors.toUnmodifiableSet()); + previousHosts = Collections.unmodifiableSet(deployState.getPreviousModel().stream() + .map(Model::allocatedHosts) + .map(AllocatedHosts::getHosts) + .flatMap(Collection::stream) + .map(HostSpec::hostname) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>()))); addSimpleComponent("com.yahoo.language.provider.DefaultLinguisticsProvider"); addSimpleComponent("com.yahoo.language.provider.DefaultEmbedderProvider"); @@ -179,21 +179,18 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat return Optional.empty(); } - /* - Create list of endpoints, these will be consumed later by the LBservicesProducer - */ + /** Create list of endpoints, these will be consumed later by LbServicesProducer */ private void createEndpointList(DeployState deployState) { if(!deployState.isHosted()) return; if(deployState.getProperties().applicationId().instance().isTester()) return; List<ApplicationClusterEndpoint> endpoints = new ArrayList<>(); - // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id. - // For now support both L7 and L4 routing + // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id. List<String> hosts = getContainers().stream() .map(AbstractService::getHostName) .collect(Collectors.toList()); - for(String suffix : deployState.getProperties().zoneDnsSuffixes()) { - // L4 + + for (String suffix : deployState.getProperties().zoneDnsSuffixes()) { ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( deployState.zone().system(), ClusterSpec.Id.from(getName()), @@ -206,28 +203,13 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .hosts(hosts) .clusterId(getName()) .build()); - - // L7 - ApplicationClusterEndpoint.DnsName l7Name = ApplicationClusterEndpoint.DnsName.sharedNameFrom( - deployState.zone().system(), - ClusterSpec.Id.from(getName()), - deployState.getProperties().applicationId(), - suffix); - endpoints.add(ApplicationClusterEndpoint.builder() - .zoneScope() - .sharedRouting() - .dnsName(l7Name) - .hosts(hosts) - .clusterId(getName()) - .build()); } - // Then get all endpoints provided by controller. - Set<ApplicationClusterEndpoint.RoutingMethod> supportedRoutingMethods = Set.of(shared, sharedLayer4); + // Include all endpoints provided by controller Set<ContainerEndpoint> endpointsFromController = deployState.getEndpoints(); endpointsFromController.stream() .filter(ce -> ce.clusterId().equals(getName())) - .filter(ce -> supportedRoutingMethods.contains(ce.routingMethod())) + .filter(ce -> ce.routingMethod() == sharedLayer4) .forEach(ce -> ce.names().forEach( name -> endpoints.add(ApplicationClusterEndpoint.builder() .scope(ce.scope()) @@ -309,8 +291,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder(); serverBuilder.hostname(container.getHostName()) .id(container.index()) - .joining(!previousHosts.isEmpty() && - !previousHosts.contains(container.getHostName())); + .joining( ! previousHosts.isEmpty() && + ! previousHosts.contains(container.getHostName())) + .retired(container.isRetired()); builder.server(serverBuilder); builder.dynamicReconfiguration(true); } @@ -343,6 +326,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat return endpointList; } + @Override + public String name() { return getName(); } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 5c5b065ac29..4b1c03a170c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -168,11 +168,11 @@ public abstract class Container extends AbstractService implements public void addBuiltinHandlers() { } @Override - public void initService(DeployLogger deployLogger) { + public void initService(DeployState deployState) { if (isInitialized()) return; // XXX: Must be called first, to set the baseport - super.initService(deployLogger); + super.initService(deployState); if (getHttp() == null) { initDefaultJettyConnector(); @@ -281,7 +281,7 @@ public abstract class Container extends AbstractService implements } protected int allocatedRpcPort = 0; - private int getRpcPort() { + protected int getRpcPort() { return allocatedRpcPort; } protected int numRpcPorts() { return rpcServerEnabled() ? 1 : 0; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index c73a3b2a676..b6ce1a7f457 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -146,6 +146,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private final ComponentGroup<Component<?, ?>> componentGroup; private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; + private final int numAvailableProcessors; private final Map<String, String> concreteDocumentTypes = new LinkedHashMap<>(); @@ -160,23 +161,25 @@ public abstract class ContainerCluster<CONTAINER extends Container> private boolean deferChangesUntilRestart = false; - public ContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState, boolean zooKeeperLocalhostAffinity) { + public ContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState, boolean zooKeeperLocalhostAffinity, int defaultPoolNumThreads) { super(parent, configSubId); this.name = clusterId; this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.zooKeeperLocalhostAffinity = zooKeeperLocalhostAffinity; + numAvailableProcessors = deployState.featureFlags().availableProcessors(); componentGroup = new ComponentGroup<>(this, "component"); addCommonVespaBundles(); - // TODO Vespa 8: remove LoggingRequestHandler.Context (replaced by ThreadedHttpRequestHandler.Context) - addSimpleComponent(com.yahoo.container.jdisc.LoggingRequestHandler.Context.class); + // TODO Vespa 8: remove LoggingRequestHandler.Context component if we can break binary compatibility + // (ThreadedHttpRequestHandler.Context is source compatible.) + addSimpleComponent("com.yahoo.container.jdisc.LoggingRequestHandler$Context"); addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); - addComponent(new DefaultThreadpoolProvider(this, deployState.featureFlags().metricsproxyNumThreads())); + addComponent(new DefaultThreadpoolProvider(this, defaultPoolNumThreads)); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider"); @@ -331,8 +334,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public SearchChains getSearchChains() { if (containerSearch == null) - throw new IllegalStateException("Search components not found in container cluster '" + getSubId() + - "': Add <search/> to the cluster in services.xml"); + throw new IllegalArgumentException("Search components not found in container cluster '" + getSubId() + + "': Add <search/> to the cluster in services.xml"); return containerSearch.getChains(); } @@ -371,8 +374,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public DocprocChains getDocprocChains() { if (containerDocproc == null) - throw new IllegalStateException("Document processing components not found in container cluster '" + getSubId() + - "': Add <document-processing/> to the cluster in services.xml"); + throw new IllegalArgumentException("Document processing components not found in container cluster '" + getSubId() + + "': Add <document-processing/> to the cluster in services.xml"); return containerDocproc.getChains(); } @@ -493,7 +496,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void getConfig(QrStartConfig.Builder builder) { builder.jvm .verbosegc(false) - .availableProcessors(2) + .availableProcessors(numAvailableProcessors) .compressedClassSpaceSize(32) .minHeapsize(32) .heapsize(256) @@ -576,6 +579,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> builder.system(zone.system().value()); builder.environment(zone.environment().value()); builder.region(zone.region().value()); + builder.cloud(zone.getCloud().name().value()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java index e0d4f3c0692..0b37abaded9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -5,7 +5,6 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.ThreadPoolProvider; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster; import com.yahoo.vespa.model.container.component.SimpleComponent; /** @@ -16,38 +15,27 @@ import com.yahoo.vespa.model.container.component.SimpleComponent; class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolConfig.Producer { private final ContainerCluster<?> cluster; - private final int metricsproxyNumThreads; + private final int defaultWorkerThreads; - DefaultThreadpoolProvider(ContainerCluster<?> cluster, int metricsproxyNumThreads) { + DefaultThreadpoolProvider(ContainerCluster<?> cluster, int defaultWorkerThreads) { super(new ComponentModel( BundleInstantiationSpecification.getFromStrings( "default-threadpool", ThreadPoolProvider.class.getName(), null))); this.cluster = cluster; - this.metricsproxyNumThreads = metricsproxyNumThreads; - } - - private int defaultThreadsByClusterType() { - if (cluster instanceof MetricsProxyContainerCluster) { - return metricsproxyNumThreads; - } - return 10; + this.defaultWorkerThreads = defaultWorkerThreads; } @Override public void getConfig(ThreadpoolConfig.Builder builder) { - if (!(cluster instanceof ApplicationContainerCluster)) { + if (cluster instanceof ApplicationContainerCluster) { + // Core pool size of 2xcores, and max of 100xcores and using a synchronous Q + // This is the deafault pool used by both federation and generally when you ask for an Executor. + builder.corePoolSize(-2).maxthreads(-100).queueSize(0); + } else { // Container clusters such as logserver, metricsproxy and clustercontroller - int defaultWorkerThreads = defaultThreadsByClusterType(); - builder.maxthreads(defaultWorkerThreads); - builder.corePoolSize(defaultWorkerThreads); - builder.queueSize(50); - return; + builder.corePoolSize(defaultWorkerThreads).maxthreads(defaultWorkerThreads).queueSize(50); } - - // Core pool size of 2xcores, and max of 100xcores and using a synchronous Q - // This is the deafault pool used by both federation and generally when you ask for an Executor. - builder.corePoolSize(-2).maxthreads(-100).queueSize(0); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index e58adf8d6b9..78fea787b7c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -9,6 +9,8 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import java.util.Objects; + /** * @author Tony Vaagenes * @author gjoranv @@ -52,7 +54,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL Integer bufferSize) { super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); - this.fileNamePattern = fileNamePattern; + this.fileNamePattern = Objects.requireNonNull(fileNamePattern, "File name pattern required when configuring access log"); this.rotationInterval = rotationInterval; this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; @@ -60,9 +62,6 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.compressionType = compressionType; this.queueSize = (queueSize == null) ? 256 : queueSize; this.bufferSize = bufferSize; - - if (fileNamePattern == null) - throw new RuntimeException("File name pattern required when configuring access log."); } private static String accessLogClass(AccessLogType logType) { @@ -114,4 +113,5 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL public String getFileNamePattern() { return fileNamePattern; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 302e8eff2d8..413c6f07efd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -83,8 +83,6 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.zookeeperClientPort().isPresent()) { builder.clientPort(options.zookeeperClientPort().get()); } - - builder.snapshotMethod(options.zooKeeperSnapshotMethod()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index c61c140c05b..f342aa1a2bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -21,22 +21,20 @@ public interface CloudConfigOptions { Optional<Integer> rpcPort(); Optional<Boolean> multiTenant(); Optional<Boolean> hostedVespa(); - ConfigServer[] allConfigServers(); int[] configServerZookeeperIds(); Optional<Integer> zookeeperClientPort(); String[] configModelPluginDirs(); Optional<Long> sessionLifeTimeSecs(); - Optional<Long> zookeeperBarrierTimeout(); //in seconds Optional<Integer> zookeeperElectionPort(); Optional<Integer> zookeeperQuorumPort(); Optional<String> environment(); Optional<String> region(); Optional<String> system(); + default Optional<String> cloud() { return Optional.empty(); } Optional<Boolean> useVespaVersionInRequest(); Optional<String> loadBalancerAddress(); Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); - String zooKeeperSnapshotMethod(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java index da6da75f409..167dac4c57e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java @@ -6,13 +6,13 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.dependencies.Dependencies; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig; -import com.yahoo.path.Path; import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.ContainerCluster; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -41,10 +41,11 @@ public class BlockFeedGlobalEndpointsFilter extends Filter implements RuleBasedF public void getConfig(RuleBasedFilterConfig.Builder builder) { Set<String> hostNames = endpoints.stream() .flatMap(e -> e.names().stream()) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); if(hostNames.size() > 0) { + Collection<String> hostnamesSorted = hostNames.stream().sorted().collect(Collectors.toList()); RuleBasedFilterConfig.Rule.Builder rule = new RuleBasedFilterConfig.Rule.Builder() - .hostNames(hostNames) + .hostNames(hostnamesSorted) .pathExpressions(ContainerCluster.RESERVED_URI_PREFIX + "/{*}") .pathExpressions(ContainerDocumentApi.DOCUMENT_V1_PREFIX + "/{*}") .methods(List.of(PUT, POST, DELETE)) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 59e8c493709..f5d1d3e6afd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -98,7 +98,7 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl for (FilterBinding binding: bindings) { if (filters.getComponent(binding.chainId()) == null && chains.getComponent(binding.chainId()) == null) - throw new RuntimeException("Can't find filter " + binding.chainId() + " for binding " + binding.binding()); + throw new IllegalArgumentException("Can't find filter " + binding.chainId() + " for binding " + binding.binding()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index 7b91b4b3244..33cbb328c52 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -75,6 +75,7 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); } configureJettyThreadpool(builder); + builder.stopTimeout(300); } private void configureJettyThreadpool(ServerConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index b7bacb34b05..e3aa5bd517f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import java.time.Duration; import java.util.Collection; import java.util.List; -import java.util.Set; +import java.util.stream.Collectors; /** * Component specification for {@link com.yahoo.jdisc.http.server.jetty.ConnectorFactory} with hosted specific configuration. @@ -88,9 +88,9 @@ public class HostedSslConnectorFactory extends ConnectorFactory { connectorBuilder.ssl.enabledProtocols(List.of("TLSv1.2")); if (!tlsCiphersOverride.isEmpty()) { - connectorBuilder.ssl.enabledCipherSuites(tlsCiphersOverride); + connectorBuilder.ssl.enabledCipherSuites(tlsCiphersOverride.stream().sorted().collect(Collectors.toList())); } else { - connectorBuilder.ssl.enabledCipherSuites(Set.copyOf(TlsContext.ALLOWED_CIPHER_SUITES)); + connectorBuilder.ssl.enabledCipherSuites(TlsContext.ALLOWED_CIPHER_SUITES.stream().sorted().collect(Collectors.toList())); } connectorBuilder diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index 1c0601915e9..13503906c04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -106,7 +106,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> .orElse(null); if (tenantDomain == null) { if (explicitDomain == null) { - throw new IllegalStateException("No Athenz domain provided for 'access-control'"); + throw new IllegalArgumentException("No Athenz domain provided for 'access-control'"); } deployState.getDeployLogger().logApplicationPackage(Level.WARNING, "Athenz tenant is not provided by deploy call. This will soon be handled as failure."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java index bf5bcbff360..6532fed9abf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java @@ -84,7 +84,7 @@ public class ModelsEvaluatorTester { return new ModelsEvaluator(rankProfilesConfig, rankingConstantsConfig, rankingExpressionsConfig, onnxModelsConfig, files); } catch (IOException | SAXException e) { - throw new RuntimeException(e); + throw new IllegalArgumentException(e); } finally { if (temporaryApplicationDir != null) { IOUtils.recursiveDeleteDir(temporaryApplicationDir); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 0e53417aea9..e0b7c3b288a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -125,7 +125,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). name(sys.getClusterName()); for (AbstractSearchCluster.SchemaSpec spec : sys.getLocalSDS()) { - scB.searchdef(spec.getSearchDefinition().getName()); + scB.searchdef(spec.getSchema().getName()); } scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java index 55284c444e4..1abb62fedab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java @@ -208,14 +208,14 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem Target target = targetResolver.resolve(targetSpec); if (target == null) { - throw new RuntimeException("Can't find source " + targetSpec.sourceSpec + - " used as a source for federation '" + getComponentId() + "'"); + throw new IllegalArgumentException("Can't find source " + targetSpec.sourceSpec + + " used as a source for federation '" + getComponentId() + "'"); } Target duplicate = resolvedTargets.put(target.id, target); if (duplicate != null && !duplicate.targetOptions.equals(target.targetOptions)) { - throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" + - " to the federation searcher " + getComponentId()); + throw new IllegalArgumentException("Search chain " + target.id + " added twice with different federation options" + + " to the federation searcher " + getComponentId()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index e2579d95fc0..7eefa2e5e74 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -116,7 +116,7 @@ public class LocalProvider extends Provider implements List<String> documentTypes = new ArrayList<>(); for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) { - documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName()); + documentTypes.add(spec.getSchema().getDocument().getName()); } return documentTypes; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java index a733c2a0f30..a71e1936425 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java @@ -41,9 +41,8 @@ public class SearchChains extends Chains<SearchChain> { private void setSearchClusterForLocalProvider(Map<String, ? extends AbstractSearchCluster> clusterIndexByName) { for (LocalProvider provider : localProviders()) { AbstractSearchCluster cluster = clusterIndexByName.get(provider.getClusterName()); - if (cluster == null) { - throw new RuntimeException("No searchable content cluster with id '" + provider.getClusterName() + "'"); - } + if (cluster == null) + throw new IllegalArgumentException("No searchable content cluster with id '" + provider.getClusterName() + "'"); provider.setSearchCluster(cluster); } } @@ -53,9 +52,8 @@ public class SearchChains extends Chains<SearchChain> { sourceGroup.validate(); if (getChainGroup().getComponentMap().containsKey(sourceGroup.getComponentId())) { - throw new RuntimeException( - String.format("Same id used for a source and another search chain/provider: '%s'", - sourceGroup.getComponentId())); + throw new IllegalArgumentException("Id '" + sourceGroup.getComponentId() + + "' is used both for a source and another search chain/provider"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java index 0be6675915c..7abb3111afd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java @@ -9,25 +9,21 @@ import java.util.LinkedHashSet; import java.util.Set; /** - * A set of sources with the same name, - * each associated with a different provider, - * that fills the same role. + * A set of sources with the same name, each associated with a different provider, that fills the same role. + * * @author Tony Vaagenes */ final class SourceGroup { + private final ComponentId id; private Source leader; - private final Set<Source> participants = - new LinkedHashSet<>(); + private final Set<Source> participants = new LinkedHashSet<>(); private void setLeader(Source leader) { assert (validMember(leader)); - if (this.leader != null) { - throw new IllegalStateException( - "There can not be two default providers for the source " - + id); - } + if (this.leader != null) + throw new IllegalArgumentException("There can not be two default providers for the source '" + id + "'"); this.leader = leader; } @@ -36,10 +32,8 @@ final class SourceGroup { assert (validMember(source)); assert (!source.equals(leader)); - if (!participants.add(source)) { - throw new RuntimeException("Source added twice to the same group " - + source); - } + if (!participants.add(source)) + throw new IllegalArgumentException("Source '" + source + "' added twice to the same group"); } private boolean validMember(Source leader) { @@ -55,8 +49,8 @@ final class SourceGroup { } public void add(Source source) { - assert source.getComponentId().equals(getComponentId()): - "Ids differ: " + source.getComponentId() + " -- " + getComponentId(); + if ( ! source.getComponentId().equals(getComponentId())) + throw new IllegalStateException("Ids differ: " + source.getComponentId() + " and " + getComponentId()); if (Source.GroupOption.leader == source.groupOption) { setLeader(source); @@ -89,7 +83,8 @@ final class SourceGroup { public void validate() { if (leader == null) - throw new IllegalStateException("Missing leader for the source " + getComponentId() + - ". One of the sources must use the attribute id instead of idref."); + throw new IllegalArgumentException("Missing leader for the source " + getComponentId() + + ". One of the sources must use the attribute id instead of idref."); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 5e1cf6db770..fbdefd0afb0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -43,7 +43,7 @@ public class BundleInstantiationSpecificationBuilder { for (String forbiddenClass: forbiddenClasses) { if (forbiddenClass.equals(instSpec.getClassName())) { - throw new RuntimeException("Setting up " + forbiddenClass + " manually is not supported."); + throw new IllegalArgumentException("Setting up " + forbiddenClass + " manually is not supported"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 986d5a21c70..c88d225f527 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -96,11 +96,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; +import java.util.logging.Level; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -162,6 +164,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { checkVersion(spec); checkTagName(spec, log); + checkDeprecatedAttributes(spec, log); ApplicationContainerCluster cluster = createContainerCluster(spec, modelContext); addClusterContent(cluster, spec, modelContext); @@ -171,6 +174,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { model.setCluster(cluster); } + private void checkDeprecatedAttributes(Element spec, DeployLogger log) { + String version = spec.getAttribute("jetty"); + if (!version.isEmpty()) { + log.logApplicationPackage(WARNING, + "The 'jetty' attribute is deprecated and will be removed in Vespa 8. " + + "It has no effect - Jetty is always enabled." + + "Please remove the attribute from the 'container'/'jdisc' element in services.xml."); + } + } + private ApplicationContainerCluster createContainerCluster(Element spec, ConfigModelContext modelContext) { return new VespaDomBuilder.DomConfigProducerBuilder<ApplicationContainerCluster>() { @Override @@ -281,7 +294,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addCloudSecretStore(ApplicationContainerCluster cluster, Element secretStoreElement, DeployState deployState) { if ( ! deployState.isHosted()) return; if ( ! cluster.getZone().system().isPublic()) - throw new RuntimeException("cloud secret store is not supported in non-public system, please see documentation"); + throw new IllegalArgumentException("Cloud secret store is not supported in non-public system, see the documentation"); CloudSecretStore cloudSecretStore = new CloudSecretStore(); Map<String, TenantSecretStore> secretStoresByName = deployState.getProperties().tenantSecretStores() .stream() @@ -296,10 +309,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { TenantSecretStore secretStore = secretStoresByName.get(account); if (secretStore == null) - throw new RuntimeException("No configured secret store named " + account); + throw new IllegalArgumentException("No configured secret store named " + account); if (secretStore.getExternalId().isEmpty()) - throw new RuntimeException("No external ID has been set"); + throw new IllegalArgumentException("No external ID has been set"); cloudSecretStore.addConfig(account, region, secretStore.getAwsId(), secretStore.getRole(), secretStore.getExternalId().get()); } @@ -347,7 +360,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Only consider global endpoints. .filter(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.global) .flatMap(endpoint -> endpoint.names().stream()) - .collect(Collectors.toUnmodifiableSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); // Build the comma delimited list of endpoints this container should be known as. // Confusingly called 'rotations' for legacy reasons. @@ -358,6 +371,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (environment != Environment.prod) return; Element aliases = XML.getChild(spec, "aliases"); + if (aliases != null) { + log.logApplicationPackage(WARNING, "The 'aliases' element and its children has no effect. They have been deprecated for removal in Vespa 8"); + } for (Element alias : XML.getChildren(aliases, "service-alias")) { cluster.serviceAliases().add(XML.getValue(alias)); } @@ -389,7 +405,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addClientProviders(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { - for (Element clientSpec: XML.getChildren(spec, "client")) { + List<Element> clientElements = XML.getChildren(spec, "client"); + if (! clientElements.isEmpty()) { + log.logApplicationPackage( + Level.WARNING, "The 'client' element is deprecated for removal in Vespa 8, with no replacement"); + } + for (Element clientSpec : clientElements) { cluster.addComponent(new DomClientProviderBuilder(cluster).build(deployState, cluster, clientSpec)); } } @@ -447,13 +468,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { - throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/en/security-model#data-plane"); + throw new IllegalArgumentException("Client certificate authority security/clients.pem is missing - " + + "see: https://cloud.vespa.ai/en/security-model#data-plane"); } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); boolean enforceHandshakeClientAuth = cluster.getHttp().getAccessControl() .map(accessControl -> accessControl.clientAuthentication) - .map(clientAuth -> clientAuth.equals(AccessControl.ClientAuthentication.need)) + .map(clientAuth -> clientAuth == AccessControl.ClientAuthentication.need) .orElse(false); connectorFactory = authorizeClient @@ -632,14 +654,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void checkVersion(Element spec) { String version = spec.getAttribute("version"); - if ( ! Version.fromString(version).equals(new Version(1))) { - throw new RuntimeException("Expected container version to be 1.0, but got " + version); - } + if ( ! Version.fromString(version).equals(new Version(1))) + throw new IllegalArgumentException("Expected container version to be 1.0, but got " + version); } private void checkTagName(Element spec, DeployLogger logger) { if (spec.getTagName().equals(DEPRECATED_CONTAINER_TAG)) { - logger.logApplicationPackage(WARNING, "'" + DEPRECATED_CONTAINER_TAG + "' is deprecated as tag name. Use '" + CONTAINER_TAG + "' instead."); + logger.logApplicationPackage(WARNING, "'" + DEPRECATED_CONTAINER_TAG + + "' is deprecated as tag name. Use '" + CONTAINER_TAG + "' instead."); } } @@ -806,7 +828,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { false, !deployState.getProperties().isBootstrap()); var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); - return createNodesFromHosts(log, hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } else { return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); @@ -816,7 +838,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) { ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, context.getDeployState()); node.setHostResource(host); - node.initService(context.getDeployLogger()); + node.initService(context.getDeployState()); return List.of(node); } @@ -827,7 +849,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ClusterSpec.Id.from(cluster.getName()), log, hasZooKeeper(containerElement)); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List<ApplicationContainer> createNodesFromNodeType(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -839,7 +861,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Map<HostResource, ClusterMembership> hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, Capacity.fromRequiredNodeType(type), log); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List<ApplicationContainer> createNodesFromContentServiceReference(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -857,11 +879,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { referenceId, cluster.getRoot().hostSystem(), context.getDeployLogger()); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } - private List<ApplicationContainer> createNodesFromHosts(DeployLogger deployLogger, - Map<HostResource, ClusterMembership> hosts, + private List<ApplicationContainer> createNodesFromHosts(Map<HostResource, ClusterMembership> hosts, ApplicationContainerCluster cluster, DeployState deployState) { List<ApplicationContainer> nodes = new ArrayList<>(); @@ -869,7 +890,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String id = "container." + entry.getValue().index(); ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), deployState); container.setHostResource(entry.getKey()); - container.initService(deployLogger); + container.initService(deployState); nodes.add(container); } return nodes; @@ -1020,9 +1041,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { AthenzService service = spec.instance(app.getApplicationId().instance()) .flatMap(instanceSpec -> instanceSpec.athenzService(zone.environment(), zone.region())) .or(() -> spec.athenzService()) - .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration in instance '" + app.getApplicationId().instance() + "'")); + .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + + app.getApplicationId().instance() + "'")); String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; - IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs), ztsUrl, zoneDnsSuffix, zone); + IdentityProvider identityProvider = new IdentityProvider(domain, + service, + getLoadBalancerName(loadBalancerName, configServerSpecs), + ztsUrl, + zoneDnsSuffix, + zone); cluster.addComponent(identityProvider); cluster.getContainers().forEach(container -> { @@ -1152,7 +1179,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (invalidOptions.isEmpty()) return; - String message = "Invalid JVM options in services.xml: " + String.join(",", invalidOptions); + String message = "Invalid or misplaced JVM options in services.xml: " + + String.join(",", invalidOptions) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; if (failDeploymentWithInvalidJvmOptions) throw new IllegalArgumentException(message); else @@ -1216,7 +1245,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (options.isEmpty()) return; Collections.sort(options); - String message = "Invalid JVM GC options in services.xml: " + String.join(",", options); + String message = "Invalid or misplaced JVM GC options in services.xml: " + + String.join(",", options) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; if (failDeploymentWithInvalidJvmOptions) throw new IllegalArgumentException(message); else diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java index 628987d5930..e15c531aa32 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java @@ -8,9 +8,9 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; * Represents configuration for bucket splitting. */ public class BucketSplitting implements StorDistributormanagerConfig.Producer { - Integer maxDocuments; - Integer splitSize; - Integer minSplitCount; + private final Integer maxDocuments; + private final Integer splitSize; + private final Integer minSplitCount; public static class Builder { public BucketSplitting build(ModelElement clusterElem) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 25e00dc05dd..f0b32f9140d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -41,7 +41,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr minNodeRatioPerGroup = clusterTuning.childAsDouble("min-node-ratio-per-group"); bucketSplittingMinimumBits = clusterTuning.childAsInteger("bucket-splitting.minimum-bits"); } - boolean enableClusterFeedBlock = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); if (tuning != null) { return new ClusterControllerConfig(ancestor, clusterName, @@ -53,14 +52,13 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr tuning.childAsDouble("min-storage-up-ratio"), bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock, resourceLimits); } else { return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null, bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock, resourceLimits); + resourceLimits); } } } @@ -74,7 +72,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr private final Double minStorageUpRatio; private final Integer minSplitBits; private final Double minNodeRatioPerGroup; - private final boolean enableClusterFeedBlock; private final ResourceLimits resourceLimits; // TODO refactor; too many args @@ -88,7 +85,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr Double minStorageUpRatio, Integer minSplitBits, Double minNodeRatioPerGroup, - boolean enableClusterFeedBlock, ResourceLimits resourceLimits) { super(parent, "fleetcontroller"); @@ -101,7 +97,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr this.minStorageUpRatio = minStorageUpRatio; this.minSplitBits = minSplitBits; this.minNodeRatioPerGroup = minNodeRatioPerGroup; - this.enableClusterFeedBlock = enableClusterFeedBlock; this.resourceLimits = resourceLimits; } @@ -144,7 +139,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr if (minNodeRatioPerGroup != null) { builder.min_node_ratio_per_group(minNodeRatioPerGroup); } - builder.enable_cluster_feed_block(enableClusterFeedBlock); resourceLimits.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java index 0a08f81ffca..80dd17213f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -34,7 +34,6 @@ public class ClusterResourceLimits { public static class Builder { - private final boolean enableFeedBlockInDistributor; private final boolean hostedVespa; private final double resourceLimitDisk; private final double resourceLimitMemory; @@ -42,11 +41,9 @@ public class ClusterResourceLimits { private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); - public Builder(boolean enableFeedBlockInDistributor, - boolean hostedVespa, + public Builder(boolean hostedVespa, double resourceLimitDisk, double resourceLimitMemory) { - this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; this.hostedVespa = hostedVespa; this.resourceLimitDisk = resourceLimitDisk; this.resourceLimitMemory = resourceLimitMemory; @@ -81,17 +78,15 @@ public class ClusterResourceLimits { } private void deriveLimits() { - if (enableFeedBlockInDistributor) { - // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), - nodeBuilder.getDiskLimit(), - ctrlBuilder::setDiskLimit, - resourceLimitDisk); - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), - nodeBuilder.getMemoryLimit(), - ctrlBuilder::setMemoryLimit, - resourceLimitMemory); - } + // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), + nodeBuilder.getDiskLimit(), + ctrlBuilder::setDiskLimit, + resourceLimitDisk); + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), + nodeBuilder.getMemoryLimit(), + ctrlBuilder::setMemoryLimit, + resourceLimitMemory); deriveClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); deriveClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 533364d80c3..dcf0be48f7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -243,8 +243,8 @@ public class Content extends ConfigModel { String indexingClusterName = cluster.getIndexingClusterName(); ContainerModel containerModel = findByName(indexingClusterName, containers); if (containerModel == null) - throw new RuntimeException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + - "cluster '" + indexingClusterName + "', but this cluster does not exist."); + throw new IllegalArgumentException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + + "cluster '" + indexingClusterName + "', but this cluster does not exist."); addIndexingChainsTo(containerModel.getCluster(), cluster); } @@ -312,7 +312,7 @@ public class Content extends ConfigModel { index++; docprocService.useDynamicPorts(); docprocService.setHostResource(host); - docprocService.initService(modelContext.getDeployLogger()); + docprocService.initService(modelContext.getDeployState()); nodes.add(docprocService); processedHosts.add(host); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index d0cba617cfc..22b752777e9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.builder.UserConfigBuilder; @@ -14,7 +15,6 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; -import com.yahoo.vespa.model.search.NamedSchema; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; @@ -73,6 +73,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final double defaultFeedConcurrency; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; + private final ProtonConfig.Replay_throttling_policy.Type.Enum persistenceAsyncThrottling; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final double fractionOfMemoryReserved; @@ -202,6 +203,14 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } } + private static ProtonConfig.Replay_throttling_policy.Type.Enum convertPersistenceAsyncThrottling(String value) { + try { + return ProtonConfig.Replay_throttling_policy.Type.Enum.valueOf(value); + } catch (Throwable t) { + return ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED; + } + } + private ContentSearchCluster(AbstractConfigProducer<?> parent, String clusterName, ModelContext.FeatureFlags featureFlags, @@ -226,6 +235,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.defaultFeedConcurrency = featureFlags.feedConcurrency(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); + this.persistenceAsyncThrottling = convertPersistenceAsyncThrottling(featureFlags.persistenceAsyncThrottling()); } public void setVisibilityDelay(double delay) { @@ -249,27 +259,25 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); - NamedSchema searchDefinition = - schemaDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSchemas()); - if (searchDefinition == null) - throw new RuntimeException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + - this + " does not exist"); + Schema schema = schemaDefinitionXMLHandler.findResponsibleSchema(deployState.getSchemas()); + if (schema == null) + throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + + this + " does not exist"); // TODO: remove explicit building of user configs when the complete content model is built using builders. - sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(searchDefinition, + sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(schema, UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger()))); - //need to get the document names from this sdfile - sc.addDocumentNames(searchDefinition); + sc.addDocumentNames(schema); } } private void addCluster(AbstractSearchCluster sc) { if (clusters.containsKey(sc.getClusterName())) { - throw new IllegalArgumentException("I already have registered cluster '" + sc.getClusterName() + "'"); + throw new IllegalArgumentException("Duplicate cluster '" + sc.getClusterName() + "'"); } if (sc instanceof IndexedSearchCluster) { if (indexedCluster != null) { - throw new IllegalArgumentException("I already have one indexed cluster named '" + indexedCluster.getClusterName()); + throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); } indexedCluster = (IndexedSearchCluster)sc; } @@ -292,11 +300,11 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), fractionOfMemoryReserved); searchNode.setHostResource(node.getHostResource()); - searchNode.initService(deployState.getDeployLogger()); + searchNode.initService(deployState); tls = new TransactionLogServer(searchNode, clusterName, syncTransactionLog); tls.setHostResource(searchNode.getHostResource()); - tls.initService(deployState.getDeployLogger()); + tls.initService(deployState); } else { searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, fractionOfMemoryReserved) @@ -444,6 +452,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> builder.indexing.tasklimit(feedTaskLimit); builder.feeding.master_task_limit(feedMasterTaskLimit); builder.feeding.shared_field_writer_executor(sharedFieldWriterExecutor); + builder.replay_throttling_policy.type(persistenceAsyncThrottling); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 7a45689901f..3f01f5610f1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -42,9 +42,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean hasIndexedDocumentType; private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; - private final int mergeBusyWait; - private final boolean enhancedMaintenanceScheduling; private final boolean unorderedMergeChaining; + private final boolean inhibitDefaultMergesWhenGlobalMergesPending; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -107,16 +106,13 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); - int mergeBusyWait = deployState.getProperties().featureFlags().distributorMergeBusyWait(); - boolean useEnhancedMaintenanceScheduling = deployState.getProperties().featureFlags().distributorEnhancedMaintenanceScheduling(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); + boolean inhibitDefaultMerges = deployState.getProperties().featureFlags().inhibitDefaultMergesWhenGlobalMergesPending(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, mergeBusyWait, - useEnhancedMaintenanceScheduling, - unorderedMergeChaining); + maxInhibitedGroups, unorderedMergeChaining, inhibitDefaultMerges); } } @@ -124,9 +120,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - int mergeBusyWait, - boolean enhancedMaintenanceScheduling, - boolean unorderedMergeChaining) + boolean unorderedMergeChaining, + boolean inhibitDefaultMergesWhenGlobalMergesPending) { super(parent, "distributor"); this.parent = parent; @@ -135,9 +130,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.hasIndexedDocumentType = hasIndexedDocumentType; this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; - this.mergeBusyWait = mergeBusyWait; - this.enhancedMaintenanceScheduling = enhancedMaintenanceScheduling; this.unorderedMergeChaining = unorderedMergeChaining; + this.inhibitDefaultMergesWhenGlobalMergesPending = inhibitDefaultMergesWhenGlobalMergesPending; } @Override @@ -151,9 +145,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.disable_bucket_activation(hasIndexedDocumentType == false); builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); - builder.inhibit_merge_sending_on_busy_node_duration_sec(mergeBusyWait); - builder.implicitly_clear_bucket_priority_on_schedule(enhancedMaintenanceScheduling); builder.use_unordered_merge_chaining(unorderedMergeChaining); + builder.inhibit_default_merges_when_global_merges_pending(inhibitDefaultMergesWhenGlobalMergesPending); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java index 4ea07ae06de..235128f7a62 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java @@ -3,12 +3,13 @@ package com.yahoo.vespa.model.content; import com.yahoo.documentmodel.NewDocumentType; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toSet; /** * Performs the following validations: @@ -29,7 +30,7 @@ public class GlobalDistributionValidator { private static void verifyReferredDocumentsArePresent(Map<String, NewDocumentType> documentDefinitions) { Set<NewDocumentType.Name> unknowDocuments = getReferencedDocuments(documentDefinitions) .filter(name -> !documentDefinitions.containsKey(name.toString())) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); if (!unknowDocuments.isEmpty()) { throw new IllegalArgumentException("The following document types are referenced from other documents, " + "but are not listed in services.xml: " + asPrintableString(unknowDocuments.stream())); @@ -41,7 +42,7 @@ public class GlobalDistributionValidator { Set<NewDocumentType> nonGlobalReferencedDocuments = getReferencedDocuments(documentDefinitions) .map(name -> documentDefinitions.get(name.toString())) .filter(documentType -> !globallyDistributedDocuments.contains(documentType)) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); if (!nonGlobalReferencedDocuments.isEmpty()) { throw new IllegalArgumentException("The following document types are referenced from other documents, " + "but are not globally distributed: " + asPrintableString(toDocumentNameStream(nonGlobalReferencedDocuments))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java index 26bf8a33a73..df269c7a9a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java @@ -31,13 +31,12 @@ public class ResourceLimits implements FleetcontrollerConfig.Producer, ProtonCon @Override public void getConfig(FleetcontrollerConfig.Builder builder) { - // TODO: Choose other defaults when this is default enabled. // Note: The resource categories must match the ones used in host info reporting // between content nodes and cluster controller: // storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp builder.cluster_feed_block_limit.put("memory", memoryLimit.orElse(0.8)); - builder.cluster_feed_block_limit.put("disk", diskLimit.orElse(0.8)); - builder.cluster_feed_block_limit.put("attribute-address-space", 0.89); + builder.cluster_feed_block_limit.put("disk", diskLimit.orElse(0.75)); + builder.cluster_feed_block_limit.put("attribute-address-space", 0.9); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index a07a36d800b..88739e92567 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -211,7 +211,7 @@ public class StorageGroup { Optional<ModelElement> nodes = getNodes(clusterElement); if (group.isPresent() && nodes.isPresent()) - throw new IllegalStateException("Both group and nodes exists, only one of these tags is legal"); + throw new IllegalArgumentException("Both <group> and <nodes> is specified: Only one of these tags can be used in the same configuration"); if (group.isPresent() && (group.get().stringAttribute("name") != null || group.get().integerAttribute("distribution-key") != null)) deployState.getDeployLogger().logApplicationPackage(Level.INFO, "'distribution-key' attribute on a content cluster's root group is ignored"); @@ -280,15 +280,13 @@ public class StorageGroup { /** The nodes explicitly specified as a nodes tag in this group, or empty if none */ private final Optional<NodesSpecification> nodeRequirement; - private final DeployLogger deployLogger; private GroupBuilder(StorageGroup storageGroup, List<GroupBuilder> subGroups, List<XmlNodeBuilder> nodeBuilders, - Optional<NodesSpecification> nodeRequirement, DeployLogger deployLogger) { + Optional<NodesSpecification> nodeRequirement) { this.storageGroup = storageGroup; this.subGroups = subGroups; this.nodeBuilders = nodeBuilders; this.nodeRequirement = nodeRequirement; - this.deployLogger = deployLogger; } /** @@ -319,11 +317,11 @@ public class StorageGroup { StorageNode searchNode = new StorageNode(deployState.getProperties(), parent.getStorageCluster(), 1.0, distributionKey , false); searchNode.setHostResource(parent.hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); PersistenceEngine provider = parent.getPersistence().create(deployState, searchNode, storageGroup, null); - searchNode.initService(deployLogger); + searchNode.initService(deployState); Distributor distributor = new Distributor(deployState.getProperties(), parent.getDistributorNodes(), distributionKey, null, provider); distributor.setHostResource(searchNode.getHostResource()); - distributor.initService(deployLogger); + distributor.initService(deployState); return searchNode; } @@ -339,7 +337,7 @@ public class StorageGroup { throw new IllegalArgumentException("Specifying individual groups is not supported on hosted applications"); Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? - provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployLogger) : + provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployState.getDeployLogger()) : Collections.emptyMap(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); @@ -450,7 +448,7 @@ public class StorageGroup { else // Nodes or groups explicitly listed - resolve in GroupBuilder nodeRequirement = Optional.empty(); - return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement, context.getDeployLogger()); + return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement); } private Optional<String> childAsString(Optional<ModelElement> element, String childTagName) { @@ -503,13 +501,13 @@ public class StorageGroup { private static StorageNode createStorageNode(DeployState deployState, ContentCluster parent, HostResource hostResource, StorageGroup parentGroup, ClusterMembership clusterMembership) { StorageNode sNode = new StorageNode(deployState.getProperties(), parent.getStorageCluster(), null, clusterMembership.index(), clusterMembership.retired()); sNode.setHostResource(hostResource); - sNode.initService(deployState.getDeployLogger()); + sNode.initService(deployState); // TODO: Supplying null as XML is not very nice PersistenceEngine provider = parent.getPersistence().create(deployState, sNode, parentGroup, null); Distributor d = new Distributor(deployState.getProperties(), parent.getDistributorNodes(), clusterMembership.index(), null, provider); d.setHostResource(sNode.getHostResource()); - d.initService(deployState.getDeployLogger()); + d.initService(deployState); return sNode; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 628cf6bb4c7..0d8f7148758 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -120,9 +120,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); - boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); - var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - stateIsHosted(deployState), + var resourceLimits = new ClusterResourceLimits.Builder(stateIsHosted(deployState), deployState.featureFlags().resourceLimitDisk(), deployState.featureFlags().resourceLimitMemory()) .build(contentElement); @@ -142,7 +140,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) - throw new RuntimeException("Indexed search requires proton as engine"); + throw new IllegalArgumentException("Indexed search requires proton as engine"); if (documentsElement != null) { ModelElement e = documentsElement.child("document-processing"); @@ -220,14 +218,14 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce if (distribution != null) { String attr = distribution.stringAttribute("type"); if (attr != null) { - if (attr.toLowerCase().equals("strict")) { + if (attr.equalsIgnoreCase("strict")) { c.distributionMode = DistributionMode.STRICT; - } else if (attr.toLowerCase().equals("loose")) { + } else if (attr.equalsIgnoreCase("loose")) { c.distributionMode = DistributionMode.LOOSE; - } else if (attr.toLowerCase().equals("legacy")) { + } else if (attr.equalsIgnoreCase("legacy")) { c.distributionMode = DistributionMode.LEGACY; } else { - throw new IllegalStateException("Distribution type " + attr + " not supported."); + throw new IllegalArgumentException("Distribution type " + attr + " not supported."); } } } @@ -367,7 +365,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce boolean retired = host.spec().membership().map(ClusterMembership::retired).orElse(false); var clusterControllerContainer = new ClusterControllerContainer(clusterControllers, ccIndex, runStandaloneZooKeeper, deployState, retired); clusterControllerContainer.setHostResource(host); - clusterControllerContainer.initService(deployState.getDeployLogger()); + clusterControllerContainer.initService(deployState); clusterControllerContainer.setProp("clustertype", "admin"); containers.add(clusterControllerContainer); ++index; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java index a4231105f42..1cab6cd462d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java @@ -5,10 +5,10 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; - -import static java.util.stream.Collectors.toSet; +import java.util.stream.Collectors; /** * Determines the set of document types that are configured to be globally distributed. @@ -32,7 +32,7 @@ public class GlobalDistributionBuilder { .filter(GlobalDistributionBuilder::isGloballyDistributed) .map(GlobalDistributionBuilder::getDocumentName) .map(this::getDocumentType) - .collect(toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } private static boolean isGloballyDistributed(ModelElement e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java index d80cb0056fa..f45bc8f4eca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java @@ -23,7 +23,7 @@ public class SearchDefinitionBuilder { if (documentType != null) { docTypes.put(documentType.getName(), documentType); } else { - throw new RuntimeException("Document type '" + name + "' not found in application package"); + throw new IllegalArgumentException("Document type '" + name + "' not found in application package"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index 1f3a76b766e..f3263658717 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,9 +46,10 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; - private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType; + private final StorFilestorConfig.Async_operation_throttler.Type.Enum asyncOperationThrottlerType; + private final double persistenceThrottlingWsDecrementFactor; + private final double persistenceThrottlingWsBackoff; private final boolean useAsyncMessageHandlingOnSchedule; - private final boolean asyncApplyBucketDiff; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { try { @@ -58,11 +59,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } } - private static StorFilestorConfig.Async_operation_throttler_type.Enum toAsyncOperationThrottlerType(String throttlerType) { + private static StorFilestorConfig.Async_operation_throttler.Type.Enum toAsyncOperationThrottlerType(String throttlerType) { try { - return StorFilestorConfig.Async_operation_throttler_type.Enum.valueOf(throttlerType); + return StorFilestorConfig.Async_operation_throttler.Type.Enum.valueOf(throttlerType); } catch (Throwable t) { - return StorFilestorConfig.Async_operation_throttler_type.UNLIMITED; + return StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED; } } @@ -72,8 +73,9 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); - useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); - asyncApplyBucketDiff = featureFlags.asyncApplyBucketDiff(); + this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); + this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); + this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); } @Override @@ -85,8 +87,16 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(reponseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); - builder.async_apply_bucket_diff(asyncApplyBucketDiff); - builder.async_operation_throttler_type(asyncOperationThrottlerType); + // TODO remove deprecated throttler type config + builder.async_operation_throttler_type((asyncOperationThrottlerType == StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC) + ? StorFilestorConfig.Async_operation_throttler_type.Enum.DYNAMIC + : StorFilestorConfig.Async_operation_throttler_type.Enum.UNLIMITED); + + var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); + throttleBuilder.type(asyncOperationThrottlerType); + throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); + throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); + builder.async_operation_throttler(throttleBuilder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index c7d581fca35..2fca964a995 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,7 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; - private Boolean ignoreMergeQueueLimit; + private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -51,11 +51,19 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } + private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { + try { + return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType); + } catch (Throwable t) { + return StorServerConfig.Merge_throttling_policy.Type.STATIC; + } + } + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); - ignoreMergeQueueLimit = featureFlags.ignoreMergeQueueLimit(); + mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy()); } @Override @@ -75,8 +83,7 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } - if (ignoreMergeQueueLimit != null) { - builder.disable_queue_limits_for_chained_merges(ignoreMergeQueueLimit); - } + // TODO set throttle policy params based on existing or separate flags + builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java index ac5c456a7b1..3962aa3d612 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java @@ -22,7 +22,10 @@ public class FileReferencesRepository { } public Set<FileReference> allFileReferences() { - return fileRegistry.export().stream().map(e -> e.reference).collect(Collectors.toUnmodifiableSet()); + return fileRegistry.export() + .stream() + .map(e -> e.reference) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 1488f5fd112..38b1d42862e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -87,7 +87,7 @@ public class ConvertedModel { ApplicationPackage applicationPackage = context.rankProfile().applicationPackage(); ImportedMlModel sourceModel = // TODO: Convert to name here, make sure its done just one way context.importedModels().get(sourceModelFile(applicationPackage, modelPath)); - ModelName modelName = new ModelName(context.rankProfile().getName(), modelPath, pathIsFile); + ModelName modelName = new ModelName(context.rankProfile().name(), modelPath, pathIsFile); if (sourceModel == null && ! new ModelStore(applicationPackage, modelName).exists()) throw new IllegalArgumentException("No model '" + modelPath + "' is available. Available models: " + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index e642da6176e..754800a42e1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -317,7 +317,7 @@ public final class DocumentProtocol implements Protocol, DocprocChain defaultChain = getDefaultChain(containerCluster.getDocproc()); if (defaultChain != null) { if (result != null) - throw new RuntimeException("Only a single default docproc chain is allowed across all container clusters"); + throw new IllegalArgumentException("Only a single default docproc chain is allowed across all container clusters"); result = defaultChain; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 19b1f39c87d..93a264ae4bb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; @@ -37,9 +38,8 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer<Abstr this.index = index; } - public void addDocumentNames(NamedSchema searchDefinition) { - String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName(); - documentNames.add(dName); + public void addDocumentNames(Schema schema) { + documentNames.add(schema.getDocument().getDocumentName().getName()); } /** Returns a List with document names used in this search cluster */ @@ -51,16 +51,15 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer<Abstr public String getClusterName() { return clusterName; } public final String getIndexingModeName() { return getIndexingMode().getName(); } - public final boolean isRealtime() { return getIndexingMode() == IndexingMode.REALTIME; } public final boolean isStreaming() { return getIndexingMode() == IndexingMode.STREAMING; } public final AbstractSearchCluster setQueryTimeout(Double to) { - this.queryTimeout=to; + this.queryTimeout = to; return this; } public final AbstractSearchCluster setVisibilityDelay(double delay) { - this.visibilityDelay=delay; + this.visibilityDelay = delay; return this; } @@ -103,16 +102,16 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer<Abstr public static final class SchemaSpec { - private final NamedSchema searchDefinition; + private final Schema schema; private final UserConfigRepo userConfigRepo; - public SchemaSpec(NamedSchema searchDefinition, UserConfigRepo userConfigRepo) { - this.searchDefinition = searchDefinition; + public SchemaSpec(Schema schema, UserConfigRepo userConfigRepo) { + this.schema = schema; this.userConfigRepo = userConfigRepo; } - public NamedSchema getSearchDefinition() { - return searchDefinition; + public Schema getSchema() { + return schema; } public UserConfigRepo getUserConfigs() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index a68993be543..fb7c6696b54 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -196,10 +196,9 @@ public class IndexedSearchCluster extends SearchCluster @Override protected void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState) { for (SchemaSpec spec : localSearches) { - Schema schema = spec.getSearchDefinition().getSearch(); - if ( ! (schema instanceof DocumentOnlySchema)) { - DocumentDatabase db = new DocumentDatabase(this, schema.getName(), - new DerivedConfiguration(schema, + if ( ! (spec.getSchema() instanceof DocumentOnlySchema)) { + DocumentDatabase db = new DocumentDatabase(this, spec.getSchema().getName(), + new DerivedConfiguration(spec.getSchema(), deployState.getDeployLogger(), deployState.getProperties(), deployState.rankProfileRegistry(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java deleted file mode 100644 index 334493b4a92..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.searchdefinition.Schema; - -import java.util.Collection; - -/** - * @author Tony Vaagenes - */ -// TODO: This class is quite pointless -public class NamedSchema { - - private final Schema schema; - private final String name; - - public static final String fileNameSuffix = ".sd"; - - public Schema getSearch() { - return schema; - } - - public String getName() { - return name; - } - - public NamedSchema(String name, Schema schema) { - this.name = name; - this.schema = schema; - } - - //Find search definition from a collection with the name specified - public static NamedSchema findByName(String schemaName, Collection<NamedSchema> schemas) { - for (NamedSchema candidate : schemas) { - if (candidate.getName().equals(schemaName) ) - return candidate; - } - - return null; - } - - // Used by admin interface - public String getFilename() { - return getName() + fileNameSuffix; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java index 881b68e8396..b4082308195 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; +import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import java.io.Serializable; +import java.util.Collection; import java.util.List; /** @@ -13,19 +15,23 @@ import java.util.List; */ public class SchemaDefinitionXMLHandler implements Serializable { - private String sdName; + private String schemaName; public SchemaDefinitionXMLHandler(ModelElement elem) { - sdName = elem.stringAttribute("name"); - if (sdName == null) { - sdName = elem.stringAttribute("type"); + schemaName = elem.stringAttribute("name"); + if (schemaName == null) { + schemaName = elem.stringAttribute("type"); } } - public String getName() { return sdName; } + public String getName() { return schemaName; } - public NamedSchema getResponsibleSearchDefinition(List<NamedSchema> schemas) { - return NamedSchema.findByName(getName(), schemas ); + public Schema findResponsibleSchema(List<Schema> schemas) { + for (Schema candidate : schemas) { + if (candidate.getName().equals(schemaName) ) + return candidate; + } + return null; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 28d1fbe72ef..6e3f3e1ebf5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -123,7 +123,6 @@ public class SearchNode extends AbstractService implements boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, double fractionOfMemoryReserved) { super(parent, name); - setOmpNumThreads(1); this.isHostedVespa = isHostedVespa; this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.nodeSpec = nodeSpec; @@ -239,10 +238,7 @@ public class SearchNode extends AbstractService implements @Override public String getStartupCommand() { - if (getOmpNumThreads() != 1) { - throw new IllegalStateException("ompNumThreads must be 1"); - } - String startup = getEnvVariables() + "exec $ROOT/sbin/vespa-proton " + "--identity " + getConfigId(); + String startup = getEnv() + " exec $ROOT/sbin/vespa-proton " + "--identity " + getConfigId(); if (serviceLayerService != null) { startup = startup + " --serviceidentity " + serviceLayerService.getConfigId(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 7b328acdc98..42bb2a1da29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -85,14 +85,15 @@ public class StreamingSearchCluster extends SearchCluster implements @Override protected void deriveAllSchemas(List<SchemaSpec> local, DeployState deployState) { if (local.size() == 1) { - deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), deployState); + deriveSingleSearchDefinition(local.get(0).getSchema(), deployState); } else if (local.size() > 1){ - throw new IllegalStateException("Logical indexes are not supported: Got " + local.size() + " search definitions, expected 1"); + throw new IllegalArgumentException("Only a single schema is supported, got " + local.size()); } } private void deriveSingleSearchDefinition(Schema localSchema, DeployState deployState) { if (!localSchema.getName().equals(docTypeName)) { - throw new IllegalStateException("Mismatch between document type name (" + docTypeName + ") and name of search definition (" + localSchema.getName() + ")"); + throw new IllegalArgumentException("Document type name '" + docTypeName + + "' must be the same as the schema name '" + localSchema.getName() + "'"); } this.sdConfig = new DerivedConfiguration(localSchema, deployState.getDeployLogger(), deployState.getProperties(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index c6d94ccfea0..2f28da60e7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -294,7 +294,7 @@ public class Tuning extends AbstractConfigProducer<Tuning> implements ProtonConf public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) { if (outputInt) { - if (maxSize!=null) chunk.maxbytes(maxSize.intValue()); + if (maxSize != null) chunk.maxbytes(maxSize.intValue()); } else { throw new IllegalStateException("Fix this, chunk does not have long types"); } @@ -305,6 +305,7 @@ public class Tuning extends AbstractConfigProducer<Tuning> implements ProtonConf } public static class LogStore { + public Long maxFileSize = null; public Component chunk = null; public Double minFileSizeFactor = null; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java index 7dabefeec47..9f049f727d6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java @@ -14,6 +14,7 @@ import java.util.Set; * @author Tony Vaagenes */ public class FreezableMap<K, V> implements Map<K, V> { + private boolean frozen = false; private Map<K, V> map; @@ -85,7 +86,7 @@ public class FreezableMap<K, V> implements Map<K, V> { public void freeze() { if (frozen) - throw new RuntimeException("The map has already been frozen."); + throw new IllegalStateException("The map has already been frozen"); frozen = true; map = Collections.unmodifiableMap(map); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java index 799107ea763..0ad28d37e93 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.model.ConfigProducer; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -23,8 +24,7 @@ import java.util.stream.Collectors; */ public final class ReflectionUtil { - private ReflectionUtil() { - } + private ReflectionUtil() {} public static Set<ConfigKey<?>> getAllConfigsProduced(Class<? extends ConfigProducer> producerClass, String configId) { // TypeToken is @Beta in guava, so consider implementing a simple recursive method instead. @@ -32,7 +32,7 @@ public final class ReflectionUtil { return interfaces.rawTypes().stream() .filter(ReflectionUtil::isConcreteProducer) .map(i -> createConfigKeyFromInstance(i.getEnclosingClass(), configId)) - .collect(Collectors.toSet()); + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } /** @@ -64,6 +64,7 @@ public final class ReflectionUtil { /** * Compares the config instances and lists any differences that will require service restart. + * * @param from The previous config. * @param to The new config. * @return An object describing the difference. @@ -91,13 +92,9 @@ public final class ReflectionUtil { } private static ConfigKey<?> createConfigKeyFromInstance(Class<?> configInstClass, String configId) { - try { - String defName = ConfigInstance.getDefName(configInstClass); - String defNamespace = ConfigInstance.getDefNamespace(configInstClass); - return new ConfigKey<>(defName, configId, defNamespace); - } catch (IllegalArgumentException | SecurityException e) { - throw new RuntimeException(e); - } + String defName = ConfigInstance.getDefName(configInstClass); + String defNamespace = ConfigInstance.getDefNamespace(configInstClass); + return new ConfigKey<>(defName, configId, defNamespace); } private static boolean isConcreteProducer(Class<?> producerInterface) { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 92633f61e67..2ccb9a82bdf 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -22,7 +22,7 @@ import com.yahoo.document.*; import com.yahoo.documentmodel.*; import com.yahoo.compress.Compressor; import com.yahoo.compress.CompressionType; -import com.yahoo.searchdefinition.Application; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.searchdefinition.DistributableResource; import com.yahoo.searchdefinition.document.*; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; @@ -73,7 +73,7 @@ import java.util.logging.Level; public class SDParser { private DocumentTypeManager docMan = null; - private Application application; + private ApplicationPackage applicationPackage; private FileRegistry fileRegistry; private DeployLogger deployLogger; private ModelContext.Properties properties; @@ -86,17 +86,17 @@ public class SDParser { * @param documentsOnly true to only parse the document aspect of a schema (e.g skip rank profiles) */ public SDParser(SimpleCharStream stream, + ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties, - Application application, RankProfileRegistry rankProfileRegistry, boolean documentsOnly) { this(stream); + this.applicationPackage = applicationPackage; this.fileRegistry = fileRegistry; this.deployLogger = deployLogger; this.properties = properties; - this.application = application; this.rankProfileRegistry = rankProfileRegistry; this.documentsOnly = documentsOnly; } @@ -395,16 +395,15 @@ SPECIAL_TOKEN : * The rule consumes any schema and returns the corresponding object. This is the only production that should * ever consume leading newlines. * - * @param dir the directory containing the file being parsed * @return the schema object */ -Schema schema(DocumentTypeManager docMan, String dir) : +Schema schema(DocumentTypeManager docMan) : { this.docMan = docMan; Schema schema; } { - (<NL>)* (schema = rootSchema(dir) | schema = rootDocument(dir)) + (<NL>)* (schema = rootSchema() | schema = rootDocument()) { return schema; } } @@ -412,10 +411,9 @@ Schema schema(DocumentTypeManager docMan, String dir) : * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume * trailing newline tokens. * - * @param dir the directory containing the file being parsed. * @return the schema definition object. */ -Schema rootSchema(String dir) : +Schema rootSchema() : { String name; String inherited = null; @@ -423,7 +421,7 @@ Schema rootSchema(String dir) : } { ( ( <SCHEMA> | <SEARCH> ) name = identifier() (<INHERITS> inherited = identifier() )? { - schema = new Schema(name, Optional.ofNullable(inherited), application, fileRegistry, deployLogger, properties); + schema = new Schema(name, applicationPackage, Optional.ofNullable(inherited), fileRegistry, deployLogger, properties); rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants()));} lbrace() (rootSchemaItem(schema) (<NL>)*)* <RBRACE> (<NL>)* <EOF>) @@ -459,12 +457,11 @@ Object rootSchemaItem(Schema schema) : { } /** * Consumes a schema definition that contains only documents to be used for inheritance, etc. * - * @param dir the directory containing the file being parsed. * @return the schema definition object. */ -Schema rootDocument(String dir) : +Schema rootDocument() : { - Schema schema = new DocumentOnlySchema(application, fileRegistry, deployLogger, properties); + Schema schema = new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties); } { ( (rootDocumentItem(schema) (<NL>)*)*<EOF> ) @@ -552,9 +549,18 @@ Object documentBody(SDDocumentType document, Schema schema) : } void rawAsBase64(Schema schema) : -{} { - <RAW_AS_BASE64_IN_SUMMARY> { schema.enableRawAsBase64(); } + boolean enabled = false; +} +{ + <RAW_AS_BASE64_IN_SUMMARY> + { + enabled = true; + } + [ <COLON> ( <TRUE> | ( <FALSE> { enabled = false; } ) ) ] + { + schema.enableRawAsBase64(enabled); + } } /** @@ -1267,17 +1273,14 @@ Object summaryInDocument(DocumentSummary document) : summary.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); SummaryInFieldLongOperation op = new SummaryInFieldLongOperation(); - } lbrace() (summaryItem(op) (<NL>)*)* <RBRACE> { - if (op.destinationIterator().hasNext()) { + if (op.destinationIterator().hasNext()) { throw new ParseException("Summaries defined in a document-summary section " + "can not have a 'to' line."); } - op.applyToSummary(summary); - document.add(summary); return null; } @@ -2036,15 +2039,15 @@ Object rankProfileItem(RankProfile profile) : { } /** * This rule consumes an inherits statement of a rank-profile. * - * @param profile The profile to modify. + * @param profile the profile to modify */ void inheritsRankProfile(RankProfile profile) : { - String str; + String name; } { - <INHERITS> str = identifierWithDash() - { profile.setInherited(str); } + <INHERITS> name = identifierWithDash() { profile.inherit(name); } + ( <COMMA> name = identifierWithDash() { profile.inherit(name);; } )* } /** @@ -2561,7 +2564,7 @@ void constantTensor(RankProfile profile, String name) : { <LBRACE> (<NL>)* (( tensorString = tensorValue() | - tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.getName(), name)) ) (<NL>)* )* <RBRACE> + tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE> { if (tensorType != null) { profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorType, tensorString))); diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 819e6b79fbb..3e751a379d4 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -52,6 +52,7 @@ ParallelInstances = element parallel { Upgrade = element upgrade { attribute policy { xsd:string }? & + attribute revision { xsd:string }? & attribute rollout { xsd:string }? } diff --git a/config-model/src/main/resources/schema/legacygenericcluster.rnc b/config-model/src/main/resources/schema/legacygenericcluster.rnc deleted file mode 100644 index b00d4ff097b..00000000000 --- a/config-model/src/main/resources/schema/legacygenericcluster.rnc +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Generic, application-specific service cluster -# -include "legacygenericmodule.rnc" - -LegacyGenericCluster = element cluster { - attribute name { text } & - attribute command { text } & - attribute hostservice { text }? & - attribute num-hosts { text }? & - GenericConfig* & - LegacyGenericModule* & - element node { - service.attlist & - attribute name { text }? & - LegacyGenericModule* & - GenericConfig* - }* -} - diff --git a/config-model/src/main/resources/schema/services.rnc b/config-model/src/main/resources/schema/services.rnc index c8467898639..3a8ffe30563 100644 --- a/config-model/src/main/resources/schema/services.rnc +++ b/config-model/src/main/resources/schema/services.rnc @@ -7,13 +7,11 @@ include "docproc.rnc" include "routing.rnc" include "containercluster.rnc" include "genericcluster.rnc" -include "legacygenericcluster.rnc" start = element services { attribute version { "1.0" }? & attribute application-type { "hosted-infrastructure" }? & element legacy { element v7-geo-positions { xsd:boolean } }? & - LegacyGenericCluster* & GenericCluster* & GenericConfig* & Admin? & diff --git a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF index 760a9ecf00f..760a9ecf00f 100644 --- a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF +++ b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF diff --git a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar b/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar Binary files differdeleted file mode 100644 index 84781c4802e..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar b/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar Binary files differdeleted file mode 100644 index f4f7dd4e127..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok.jar b/config-model/src/test/cfg/application/validation/testjars/ok.jar Binary files differdeleted file mode 100644 index fce043c6ff7..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/ok.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..53773b8b1cc --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF @@ -0,0 +1,7 @@ +Manifest-Version: 1.0 +Created-By: 1.6.0_20 (Apple Inc.) +Bundle-ManifestVersion: 2 +Bundle-Name: ok +Bundle-SymbolicName: ok +Bundle-Version: 0 + diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar Binary files differdeleted file mode 100644 index a395a52d17d..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..21c58490c13 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Archiver-Version: Plexus Archiver +Created-By: vespa container maven plugin +Built-By: tonyv +Build-Jdk: 1.6.0_26 +Bundle-Vendor: Yahoo! +Bundle-ClassPath: .,dependencies/jrt-5.1-SNAPSHOT.jar +Bundle-Version: 5.1.0.SNAPSHOT +Bundle-Name: container maven plugin test +Bundle-ManifestVersion: 2 +Bundle-SymbolicName: TestBundle + diff --git a/config-model/src/test/cfg/application/validation/testjars/test.jar b/config-model/src/test/cfg/application/validation/testjars/test.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/test.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar Binary files differdeleted file mode 100644 index 31266f1e8f2..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar +++ /dev/null diff --git a/config-model/src/test/cfg/routing/content_two_clusters/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/content_two_clusters/document-protocol-policies.cfg new file mode 100644 index 00000000000..2dcc2cde532 --- /dev/null +++ b/config-model/src/test/cfg/routing/content_two_clusters/document-protocol-policies.cfg @@ -0,0 +1,16 @@ +cluster{mobile}.defaultRoute "mobile-direct" +cluster{mobile}.route[0].name "mobile-index" +cluster{mobile}.route[0].messageType 100004 +cluster{mobile}.route[1].name "mobile-index" +cluster{mobile}.route[1].messageType 100005 +cluster{mobile}.route[2].name "mobile-index" +cluster{mobile}.route[2].messageType 100006 +cluster{mobile}.selector "(mobile)" +cluster{music}.defaultRoute "music-direct" +cluster{music}.route[0].name "music-index" +cluster{music}.route[0].messageType 100004 +cluster{music}.route[1].name "music-index" +cluster{music}.route[1].messageType 100005 +cluster{music}.route[2].name "music-index" +cluster{music}.route[2].messageType 100006 +cluster{music}.selector "(music)" diff --git a/config-model/src/test/cfg/routing/content_two_clusters/documentrouteselectorpolicy.cfg b/config-model/src/test/cfg/routing/content_two_clusters/documentrouteselectorpolicy.cfg index 657b05d63b7..fce2a6ef2ac 100755 --- a/config-model/src/test/cfg/routing/content_two_clusters/documentrouteselectorpolicy.cfg +++ b/config-model/src/test/cfg/routing/content_two_clusters/documentrouteselectorpolicy.cfg @@ -1,6 +1,6 @@ -route[0].name "content/mobile" -route[0].selector "mobile or mobile_search" +route[0].name "mobile" +route[0].selector "(mobile)" route[0].feed "" -route[1].name "content/music" -route[1].selector "music or music_search" +route[1].name "music" +route[1].selector "(music)" route[1].feed "" diff --git a/config-model/src/test/cfg/routing/content_two_clusters/messagebus.cfg b/config-model/src/test/cfg/routing/content_two_clusters/messagebus.cfg index 3860bf71431..bff785cdf04 100755 --- a/config-model/src/test/cfg/routing/content_two_clusters/messagebus.cfg +++ b/config-model/src/test/cfg/routing/content_two_clusters/messagebus.cfg @@ -1,44 +1,34 @@ routingtable[0].protocol "document" -routingtable[0].hop[0].name "docproc/cluster.mobile.indexing/chain.mobile.indexing" -routingtable[0].hop[0].selector "[LoadBalancer:cluster=docproc/cluster.mobile.indexing;session=chain.mobile.indexing]" +routingtable[0].hop[0].name "docproc/cluster.mobile.indexing/chain.indexing" +routingtable[0].hop[0].selector "[LoadBalancer:cluster=docproc/cluster.mobile.indexing;session=chain.indexing]" routingtable[0].hop[0].ignoreresult false -routingtable[0].hop[1].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[1].selector "[LoadBalancer:cluster=docproc/cluster.music.indexing;session=chain.music.indexing]" +routingtable[0].hop[1].name "docproc/cluster.music.indexing/chain.indexing" +routingtable[0].hop[1].selector "[LoadBalancer:cluster=docproc/cluster.music.indexing;session=chain.indexing]" routingtable[0].hop[1].ignoreresult false routingtable[0].hop[2].name "indexing" routingtable[0].hop[2].selector "[DocumentRouteSelector]" -routingtable[0].hop[2].recipient[0] "content/mobile" -routingtable[0].hop[2].recipient[1] "content/music" +routingtable[0].hop[2].recipient[0] "mobile" +routingtable[0].hop[2].recipient[1] "music" routingtable[0].hop[2].ignoreresult false -routingtable[0].route[0].name "content/mobile" -routingtable[0].route[0].hop[0] "[MessageType:content/mobile]" -routingtable[0].route[1].name "content/mobile-direct" -routingtable[0].route[1].hop[0] "[Content:cluster=mobile]" -routingtable[0].route[2].name "content/mobile-index" -routingtable[0].route[2].hop[0] "docproc/cluster.mobile.indexing/chain.mobile.indexing" -routingtable[0].route[2].hop[1] "[Content:cluster=mobile]" -routingtable[0].route[3].name "content/music" -routingtable[0].route[3].hop[0] "[MessageType:content/music]" -routingtable[0].route[4].name "content/music-direct" -routingtable[0].route[4].hop[0] "[Content:cluster=music]" -routingtable[0].route[5].name "content/music-index" -routingtable[0].route[5].hop[0] "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].route[5].hop[1] "[Content:cluster=music]" -routingtable[0].route[6].name "default" -routingtable[0].route[6].hop[0] "indexing" -routingtable[0].route[7].name "mobile" -routingtable[0].route[7].hop[0] "route:content/mobile" -routingtable[0].route[8].name "mobile-direct" -routingtable[0].route[8].hop[0] "route:content/mobile-direct" -routingtable[0].route[9].name "mobile-index" -routingtable[0].route[9].hop[0] "route:content/mobile-index" -routingtable[0].route[10].name "music" -routingtable[0].route[10].hop[0] "route:content/music" -routingtable[0].route[11].name "music-direct" -routingtable[0].route[11].hop[0] "route:content/music-direct" -routingtable[0].route[12].name "music-index" -routingtable[0].route[12].hop[0] "route:content/music-index" -routingtable[0].route[13].name "storage/cluster.mobile" -routingtable[0].route[13].hop[0] "route:content/mobile" -routingtable[0].route[14].name "storage/cluster.music" -routingtable[0].route[14].hop[0] "route:content/music" +routingtable[0].route[0].name "default" +routingtable[0].route[0].hop[0] "indexing" +routingtable[0].route[1].name "default-get" +routingtable[0].route[1].hop[0] "indexing" +routingtable[0].route[2].name "mobile" +routingtable[0].route[2].hop[0] "[MessageType:mobile]" +routingtable[0].route[3].name "mobile-direct" +routingtable[0].route[3].hop[0] "[Content:cluster=mobile]" +routingtable[0].route[4].name "mobile-index" +routingtable[0].route[4].hop[0] "docproc/cluster.mobile.indexing/chain.indexing" +routingtable[0].route[4].hop[1] "[Content:cluster=mobile]" +routingtable[0].route[5].name "music" +routingtable[0].route[5].hop[0] "[MessageType:music]" +routingtable[0].route[6].name "music-direct" +routingtable[0].route[6].hop[0] "[Content:cluster=music]" +routingtable[0].route[7].name "music-index" +routingtable[0].route[7].hop[0] "docproc/cluster.music.indexing/chain.indexing" +routingtable[0].route[7].hop[1] "[Content:cluster=music]" +routingtable[0].route[8].name "storage/cluster.mobile" +routingtable[0].route[8].hop[0] "route:mobile" +routingtable[0].route[9].name "storage/cluster.music" +routingtable[0].route[9].hop[0] "route:music" diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/contentsimpleconfig/document-protocol-policies.cfg new file mode 100644 index 00000000000..438969400fe --- /dev/null +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/document-protocol-policies.cfg @@ -0,0 +1,8 @@ +cluster{c}.defaultRoute "c-direct" +cluster{c}.route[0].name "c-index" +cluster{c}.route[0].messageType 100004 +cluster{c}.route[1].name "c-index" +cluster{c}.route[1].messageType 100005 +cluster{c}.route[2].name "c-index" +cluster{c}.route[2].messageType 100006 +cluster{c}.selector "(music)" diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/documentrouteselectorpolicy.cfg b/config-model/src/test/cfg/routing/contentsimpleconfig/documentrouteselectorpolicy.cfg index 7e7a546235c..041c6ac08d9 100755 --- a/config-model/src/test/cfg/routing/contentsimpleconfig/documentrouteselectorpolicy.cfg +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/documentrouteselectorpolicy.cfg @@ -1,3 +1,3 @@ -route[0].name "content/music" -route[0].selector "music or music_search" +route[0].name "c" +route[0].selector "(music)" route[0].feed "" diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/messagebus.cfg b/config-model/src/test/cfg/routing/contentsimpleconfig/messagebus.cfg index e1c87a4274d..b56b2316440 100755 --- a/config-model/src/test/cfg/routing/contentsimpleconfig/messagebus.cfg +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/messagebus.cfg @@ -1,25 +1,21 @@ routingtable[0].protocol "document" -routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[0].selector "[LoadBalancer:cluster=docproc/cluster.music.indexing;session=chain.music.indexing]" +routingtable[0].hop[0].name "docproc/cluster.c.indexing/chain.indexing" +routingtable[0].hop[0].selector "[LoadBalancer:cluster=docproc/cluster.c.indexing;session=chain.indexing]" routingtable[0].hop[0].ignoreresult false routingtable[0].hop[1].name "indexing" routingtable[0].hop[1].selector "[DocumentRouteSelector]" -routingtable[0].hop[1].recipient[0] "content/music" +routingtable[0].hop[1].recipient[0] "c" routingtable[0].hop[1].ignoreresult false -routingtable[0].route[0].name "content/music" -routingtable[0].route[0].hop[0] "[MessageType:content/music]" -routingtable[0].route[1].name "content/music-direct" -routingtable[0].route[1].hop[0] "[Content:cluster=music]" -routingtable[0].route[2].name "content/music-index" -routingtable[0].route[2].hop[0] "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].route[2].hop[1] "[Content:cluster=music]" +routingtable[0].route[0].name "c" +routingtable[0].route[0].hop[0] "[MessageType:c]" +routingtable[0].route[1].name "c-direct" +routingtable[0].route[1].hop[0] "[Content:cluster=c]" +routingtable[0].route[2].name "c-index" +routingtable[0].route[2].hop[0] "docproc/cluster.c.indexing/chain.indexing" +routingtable[0].route[2].hop[1] "[Content:cluster=c]" routingtable[0].route[3].name "default" routingtable[0].route[3].hop[0] "indexing" -routingtable[0].route[4].name "music" -routingtable[0].route[4].hop[0] "route:content/music" -routingtable[0].route[5].name "music-direct" -routingtable[0].route[5].hop[0] "route:content/music-direct" -routingtable[0].route[6].name "music-index" -routingtable[0].route[6].hop[0] "route:content/music-index" -routingtable[0].route[7].name "storage/cluster.music" -routingtable[0].route[7].hop[0] "route:content/music" +routingtable[0].route[4].name "default-get" +routingtable[0].route[4].hop[0] "[Content:cluster=c]" +routingtable[0].route[5].name "storage/cluster.c" +routingtable[0].route[5].hop[0] "route:c" diff --git a/config-model/src/test/derived/advanced/advanced.sd b/config-model/src/test/derived/advanced/advanced.sd index b03db73d45d..928e74160b9 100644 --- a/config-model/src/test/derived/advanced/advanced.sd +++ b/config-model/src/test/derived/advanced/advanced.sd @@ -99,6 +99,7 @@ search advanced { fieldset titleabstract { fields: title } + raw-as-base64-in-summary : true fieldset default { fields: title } diff --git a/config-model/src/test/derived/attributerank/attributerank.sd b/config-model/src/test/derived/attributerank/attributerank.sd index 86f96e4817c..4989e1795cd 100644 --- a/config-model/src/test/derived/attributerank/attributerank.sd +++ b/config-model/src/test/derived/attributerank/attributerank.sd @@ -38,4 +38,5 @@ search attributerank { rank-type singledouble: identity rank-type singlestring: identity } + } diff --git a/config-model/src/test/derived/complex/complex.sd b/config-model/src/test/derived/complex/complex.sd index fffa9b02212..844d09a529d 100644 --- a/config-model/src/test/derived/complex/complex.sd +++ b/config-model/src/test/derived/complex/complex.sd @@ -140,6 +140,7 @@ search complex { fieldset special { fields: special1, special2, special3 } + raw-as-base64-in-summary : false fieldset all { fields: combineda, combinedb } diff --git a/config-model/src/test/derived/indexschema/vsmfields.cfg b/config-model/src/test/derived/indexschema/vsmfields.cfg index 9dcffd30313..31db622183e 100644 --- a/config-model/src/test/derived/indexschema/vsmfields.cfg +++ b/config-model/src/test/derived/indexschema/vsmfields.cfg @@ -20,6 +20,11 @@ fieldspec[].searchmethod AUTOUTF8 fieldspec[].arg1 "" fieldspec[].maxlength 1048576 fieldspec[].fieldtype INDEX +fieldspec[].name "pos" +fieldspec[].searchmethod GEOPOS +fieldspec[].arg1 "" +fieldspec[].maxlength 1048576 +fieldspec[].fieldtype INDEX fieldspec[].name "se" fieldspec[].searchmethod AUTOUTF8 fieldspec[].arg1 "word" @@ -124,6 +129,8 @@ documenttype[].index[].name "sc" documenttype[].index[].field[].name "sc" documenttype[].index[].name "sd" documenttype[].index[].field[].name "sd" +documenttype[].index[].name "pos" +documenttype[].index[].field[].name "pos" documenttype[].index[].name "se" documenttype[].index[].field[].name "se" documenttype[].index[].name "sf" diff --git a/config-model/src/test/derived/multiplesummaries/attributes.cfg b/config-model/src/test/derived/multiplesummaries/attributes.cfg index d5db9a8fe9d..44c94c73781 100644 --- a/config-model/src/test/derived/multiplesummaries/attributes.cfg +++ b/config-model/src/test/derived/multiplesummaries/attributes.cfg @@ -1,15 +1,160 @@ -attribute[a].collectiontype SINGLE -attribute[a].datatype STRING -attribute[a].name "a" -attribute[abolded].collectiontype SINGLE -attribute[abolded].datatype STRING -attribute[abolded].name "abolded" -attribute[adynamic].collectiontype SINGLE -attribute[adynamic].datatype STRING -attribute[adynamic].name "adynamic" -attribute[c].collectiontype SINGLE -attribute[c].datatype STRING -attribute[c].name "c" -attribute[loc_pos_zcurve].collectiontype SINGLE -attribute[loc_pos_zcurve].datatype INT64 -attribute[loc_pos_zcurve].name "loc_pos_zcurve" +attribute[].name "a" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "adynamic" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "abolded" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "c" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "loc_pos_zcurve" +attribute[].datatype INT64 +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index fbb546179e1..5434b0770f7 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -1,13 +1,27 @@ -ilscript[multiplesummaries].doctype "multiplesummaries" -ilscript[multiplesummaries].name "multiplesummaries" -ilscript[multiplesummaries].content[] "clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a }" -ilscript[multiplesummaries].content[] "clear_state | guard { input adynamic | tokenize normalize stem:\"SHORTEST\" | summary adynamic | attribute adynamic }" -ilscript[multiplesummaries].content[] "clear_state | guard { input abolded | tokenize normalize stem:\"SHORTEST\" | summary abolded | attribute abolded }" -ilscript[multiplesummaries].content[] "clear_state | guard { input b | summary b }" -ilscript[multiplesummaries].content[] "clear_state | guard { input c | summary c | attribute c }" -ilscript[multiplesummaries].content[] "clear_state | guard { input d | tokenize normalize stem:\"SHORTEST\" | summary d }" -ilscript[multiplesummaries].content[] "clear_state | guard { input e | tokenize normalize stem:\"SHORTEST\" | summary dynamice }" -ilscript[multiplesummaries].content[] "clear_state | guard { input f | summary f }" -ilscript[multiplesummaries].content[] "clear_state | guard { input g | summary g }" -ilscript[multiplesummaries].content[] "clear_state | guard { input h | summary h }" -ilscript[multiplesummaries].content[] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve }" +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "multiplesummaries" +ilscript[].docfield[0] "a" +ilscript[].docfield[1] "adynamic" +ilscript[].docfield[2] "abolded" +ilscript[].docfield[3] "b" +ilscript[].docfield[4] "c" +ilscript[].docfield[5] "d" +ilscript[].docfield[6] "e" +ilscript[].docfield[7] "f" +ilscript[].docfield[8] "g" +ilscript[].docfield[9] "h" +ilscript[].docfield[10] "loc" +ilscript[].docfield[11] "mytags" +ilscript[].content[0] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }" +ilscript[].content[1] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }" +ilscript[].content[2] "clear_state | guard { input adynamic | tokenize normalize stem:\"BEST\" | summary adynamic | attribute adynamic; }" +ilscript[].content[3] "clear_state | guard { input abolded | tokenize normalize stem:\"BEST\" | summary abolded | attribute abolded; }" +ilscript[].content[4] "clear_state | guard { input b | summary anotherb | summary b; }" +ilscript[].content[5] "clear_state | guard { input c | summary c | attribute c; }" +ilscript[].content[6] "clear_state | guard { input d | tokenize normalize stem:\"BEST\" | summary d; }" +ilscript[].content[7] "clear_state | guard { input e | tokenize normalize stem:\"BEST\" | summary dynamice | summary e; }" +ilscript[].content[8] "clear_state | guard { input f | summary f; }" +ilscript[].content[9] "clear_state | guard { input g | summary g; }" +ilscript[].content[10] "clear_state | guard { input h | summary h; }" +ilscript[].content[11] "clear_state | guard { input mytags | for_each { tokenize normalize stem:\"BEST\" } | index mytags; }" diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index ec2d3f06e46..d5002535761 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -1,59 +1,145 @@ -indexinfo[multiplesummaries].name "multiplesummaries" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "sddocname" -indexinfo[multiplesummaries].command[].command "word" -indexinfo[multiplesummaries].command[].indexname "sddocname" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "a" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "a" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "adynamic" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "adynamic" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "abolded" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "abolded" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "b" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "c" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "c" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "d" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "e" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "f" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "g" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "h" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "loc" -indexinfo[multiplesummaries].command[].command "default-position" -indexinfo[multiplesummaries].command[].indexname "loc_pos" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "loc_pos" -indexinfo[multiplesummaries].command[].command "index" -indexinfo[multiplesummaries].command[].indexname "loc_pos_zcurve" -indexinfo[multiplesummaries].command[].command "attribute" -indexinfo[multiplesummaries].command[].indexname "loc_pos_zcurve" -indexinfo[multiplesummaries].command[].command "dynteaser" -indexinfo[multiplesummaries].command[].indexname "adynamic" -indexinfo[multiplesummaries].command[].command "highlight" -indexinfo[multiplesummaries].command[].indexname "d" -indexinfo[multiplesummaries].command[].command "dynteaser" -indexinfo[multiplesummaries].command[].indexname "adynamic2" -indexinfo[multiplesummaries].command[].command "highlight" -indexinfo[multiplesummaries].command[].indexname "abolded2" -indexinfo[multiplesummaries].command[].command "dynteaser" -indexinfo[multiplesummaries].command[].indexname "aboldeddynamic" -indexinfo[multiplesummaries].command[].command "highlight" -indexinfo[multiplesummaries].command[].indexname "aboldeddynamic" -indexinfo[multiplesummaries].command[].command "highlight" -indexinfo[multiplesummaries].command[].indexname "abolded" -indexinfo[multiplesummaries].command[].command "dynteaser" -indexinfo[multiplesummaries].command[].indexname "dynamice" +indexinfo[].name "multiplesummaries" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "a" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "a" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "a" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "a" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "b" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "b" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "c" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "c" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "c" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "c" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "d" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "d" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "e" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "e" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "f" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "f" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "f" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "g" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "g" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "g" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "g" +indexinfo[].command[].command "type Array<int>" +indexinfo[].command[].indexname "h" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "h" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "h" +indexinfo[].command[].command "type WeightedSet<string>" +indexinfo[].command[].indexname "loc" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "loc" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mytags" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "abolded2" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "abolded2" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "aboldeddynamic" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "aboldeddynamic" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "adynamic2" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "adynamic2" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "alltags" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "anotherb" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "anotherb" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "dynamice" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "dynamice" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "loc_pos" +indexinfo[].command[].command "default-position" +indexinfo[].command[].indexname "loc_pos" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "loc_pos" +indexinfo[].command[].command "type position" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "loc_pos_zcurve" +indexinfo[].command[].command "type long" +indexinfo[].command[].indexname "adynamic" +indexinfo[].command[].command "dynteaser" +indexinfo[].command[].indexname "d" +indexinfo[].command[].command "highlight" +indexinfo[].command[].indexname "adynamic2" +indexinfo[].command[].command "dynteaser" +indexinfo[].command[].indexname "abolded2" +indexinfo[].command[].command "highlight" +indexinfo[].command[].indexname "aboldeddynamic" +indexinfo[].command[].command "dynteaser" +indexinfo[].command[].indexname "aboldeddynamic" +indexinfo[].command[].command "highlight" +indexinfo[].command[].indexname "abolded" +indexinfo[].command[].command "highlight" +indexinfo[].command[].indexname "dynamice" +indexinfo[].command[].command "dynteaser" diff --git a/config-model/src/test/derived/multiplesummaries/juniperrc.cfg b/config-model/src/test/derived/multiplesummaries/juniperrc.cfg index 2151e5a59c4..9b5c6a5a7ba 100644 --- a/config-model/src/test/derived/multiplesummaries/juniperrc.cfg +++ b/config-model/src/test/derived/multiplesummaries/juniperrc.cfg @@ -1,16 +1,43 @@ +length 256 +max_matches 3 +min_length 128 prefix true -override[abolded].fieldname "abolded" -override[abolded].length 65536 -override[abolded].max_matches 1 -override[abolded].min_length 8192 -override[abolded].surround_max 65536 -override[abolded2].fieldname "abolded2" -override[abolded2].length 65536 -override[abolded2].max_matches 1 -override[abolded2].min_length 8192 -override[abolded2].surround_max 65536 -override[d].fieldname "d" -override[d].length 65536 -override[d].max_matches 1 -override[d].min_length 8192 -override[d].surround_max 65536 +surround_max 128 +winsize 200 +winsize_fallback_multiplier 10.0 +max_match_candidates 1000 +stem_min_length 5 +stem_max_extend 3 +override[].fieldname "d" +override[].length 65536 +override[].max_matches 1 +override[].min_length 8192 +override[].prefix true +override[].surround_max 65536 +override[].winsize 200 +override[].winsize_fallback_multiplier 10.0 +override[].max_match_candidates 1000 +override[].stem_min_length 5 +override[].stem_max_extend 3 +override[].fieldname "abolded2" +override[].length 65536 +override[].max_matches 1 +override[].min_length 8192 +override[].prefix true +override[].surround_max 65536 +override[].winsize 200 +override[].winsize_fallback_multiplier 10.0 +override[].max_match_candidates 1000 +override[].stem_min_length 5 +override[].stem_max_extend 3 +override[].fieldname "abolded" +override[].length 65536 +override[].max_matches 1 +override[].min_length 8192 +override[].prefix true +override[].surround_max 65536 +override[].winsize 200 +override[].winsize_fallback_multiplier 10.0 +override[].max_match_candidates 1000 +override[].stem_min_length 5 +override[].stem_max_extend 3 diff --git a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd index f4e131ee0bb..ae0e2fe92bc 100644 --- a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd +++ b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd @@ -67,6 +67,10 @@ search multiplesummaries { field loc type string { } + + field mytags type array<string> { + indexing: index + } } field loc_pos type position { @@ -120,7 +124,7 @@ search multiplesummaries { } # Since a here is a dynamic summary, it will be fetched from disk - document-summary notattributesonly2 { + document-summary anothernotattributesonly2 { summary adynamic2 type string { # Should still be dynamic here source: a @@ -130,6 +134,19 @@ search multiplesummaries { summary c type string { } + summary alltags type array<string> { + source: mytags + } + summary sometags type array<string> { + source: mytags + matched-elements-only + } + summary anothera type string { + source: a + } + summary anotherb type string { + source: b + } } # Not attributes only because d is bolded @@ -153,6 +170,7 @@ search multiplesummaries { } summary loc_position type long { + source: loc_pos_zcurve } } @@ -166,7 +184,7 @@ search multiplesummaries { source: a } - summary loc_position type long { + summary loc_pos_zcurve type long { } } diff --git a/config-model/src/test/derived/multiplesummaries/summary.cfg b/config-model/src/test/derived/multiplesummaries/summary.cfg index 16d0024155b..1c8fc47878b 100644 --- a/config-model/src/test/derived/multiplesummaries/summary.cfg +++ b/config-model/src/test/derived/multiplesummaries/summary.cfg @@ -1,174 +1,200 @@ -defaultsummaryid 235127765 +defaultsummaryid 456145241 usev8geopositions false -classes[1156201411].id 1156201411 -classes[1156201411].name "attributeprefetch" -classes[1156201411].fields[a].name "a" -classes[1156201411].fields[a].type "longstring" -classes[1156201411].fields[c].name "c" -classes[1156201411].fields[c].type "longstring" -classes[1156201411].fields[loc_pos_zcurve].name "loc_pos_zcurve" -classes[1156201411].fields[loc_pos_zcurve].type "int64" -classes[1156201411].fields[rankfeatures].name "rankfeatures" -classes[1156201411].fields[rankfeatures].type "longstring" -classes[1156201411].fields[summaryfeatures].name "summaryfeatures" -classes[1156201411].fields[summaryfeatures].type "longstring" -classes[1277791169].id 1277791169 -classes[1277791169].name "notattributesonly2" -classes[1277791169].fields[adynamic2].name "adynamic2" -classes[1277791169].fields[adynamic2].type "longstring" -classes[1277791169].fields[c].name "c" -classes[1277791169].fields[c].type "longstring" -classes[1277791169].fields[rankfeatures].name "rankfeatures" -classes[1277791169].fields[rankfeatures].type "longstring" -classes[1277791169].fields[summaryfeatures].name "summaryfeatures" -classes[1277791169].fields[summaryfeatures].type "longstring" -classes[1280967808].id 1280967808 -classes[1280967808].name "attributesonly2" -classes[1280967808].fields[anotdynamic].name "anotdynamic" -classes[1280967808].fields[anotdynamic].type "longstring" -classes[1280967808].fields[c].name "c" -classes[1280967808].fields[c].type "longstring" -classes[1280967808].fields[loc_position].name "loc_position" -classes[1280967808].fields[loc_position].type "int64" -classes[1280967808].fields[rankfeatures].name "rankfeatures" -classes[1280967808].fields[rankfeatures].type "longstring" -classes[1280967808].fields[summaryfeatures].name "summaryfeatures" -classes[1280967808].fields[summaryfeatures].type "longstring" -classes[1334083320].id 1334083320 -classes[1334083320].name "third" -classes[1334083320].fields[a].name "a" -classes[1334083320].fields[a].type "longstring" -classes[1334083320].fields[adynamic].name "adynamic" -classes[1334083320].fields[adynamic].type "longstring" -classes[1334083320].fields[d].name "d" -classes[1334083320].fields[d].type "longstring" -classes[1334083320].fields[e].name "e" -classes[1334083320].fields[e].type "longstring" -classes[1334083320].fields[f].name "f" -classes[1334083320].fields[f].type "jsonstring" -classes[1334083320].fields[g].name "g" -classes[1334083320].fields[g].type "jsonstring" -classes[1334083320].fields[h].name "h" -classes[1334083320].fields[h].type "jsonstring" -classes[1334083320].fields[rankfeatures].name "rankfeatures" -classes[1334083320].fields[rankfeatures].type "longstring" -classes[1334083320].fields[summaryfeatures].name "summaryfeatures" -classes[1334083320].fields[summaryfeatures].type "longstring" -classes[1439192258].id 1439192258 -classes[1439192258].name "second" -classes[1439192258].fields[a].name "a" -classes[1439192258].fields[a].type "longstring" -classes[1439192258].fields[adynamic].name "adynamic" -classes[1439192258].fields[adynamic].type "longstring" -classes[1439192258].fields[c].name "c" -classes[1439192258].fields[c].type "longstring" -classes[1439192258].fields[f].name "f" -classes[1439192258].fields[f].type "jsonstring" -classes[1439192258].fields[rankfeatures].name "rankfeatures" -classes[1439192258].fields[rankfeatures].type "longstring" -classes[1439192258].fields[summaryfeatures].name "summaryfeatures" -classes[1439192258].fields[summaryfeatures].type "longstring" -classes[1653275739].id 1653275739 -classes[1653275739].name "attributesonly3" -classes[1653275739].fields[a].name "a" -classes[1653275739].fields[a].type "longstring" -classes[1653275739].fields[anotbolded].name "anotbolded" -classes[1653275739].fields[anotbolded].type "longstring" -classes[1653275739].fields[loc_position].name "loc_position" -classes[1653275739].fields[loc_position].type "int64" -classes[1653275739].fields[rankfeatures].name "rankfeatures" -classes[1653275739].fields[rankfeatures].type "longstring" -classes[1653275739].fields[summaryfeatures].name "summaryfeatures" -classes[1653275739].fields[summaryfeatures].type "longstring" -classes[182001096].id 182001096 -classes[182001096].name "notattributesonly1" -classes[182001096].fields[adynamic].name "adynamic" -classes[182001096].fields[adynamic].type "longstring" -classes[182001096].fields[c].name "c" -classes[182001096].fields[c].type "longstring" -classes[182001096].fields[rankfeatures].name "rankfeatures" -classes[182001096].fields[rankfeatures].type "longstring" -classes[182001096].fields[summaryfeatures].name "summaryfeatures" -classes[182001096].fields[summaryfeatures].type "longstring" -classes[1881063334].id 1881063334 -classes[1881063334].name "notattributesonly4" -classes[1881063334].fields[abolded2].name "abolded2" -classes[1881063334].fields[abolded2].type "longstring" -classes[1881063334].fields[c].name "c" -classes[1881063334].fields[c].type "longstring" -classes[1881063334].fields[rankfeatures].name "rankfeatures" -classes[1881063334].fields[rankfeatures].type "longstring" -classes[1881063334].fields[summaryfeatures].name "summaryfeatures" -classes[1881063334].fields[summaryfeatures].type "longstring" -classes[1988966242].id 1988966242 -classes[1988966242].name "attributesonly1" -classes[1988966242].fields[a].name "a" -classes[1988966242].fields[a].type "longstring" -classes[1988966242].fields[c].name "c" -classes[1988966242].fields[c].type "longstring" -classes[1988966242].fields[rankfeatures].name "rankfeatures" -classes[1988966242].fields[rankfeatures].type "longstring" -classes[1988966242].fields[summaryfeatures].name "summaryfeatures" -classes[1988966242].fields[summaryfeatures].type "longstring" -classes[235127765].id 235127765 -classes[235127765].name "multiplesummaries" -classes[235127765].fields[a].name "a" -classes[235127765].fields[a].type "longstring" -classes[235127765].fields[abolded].name "abolded" -classes[235127765].fields[abolded].type "longstring" -classes[235127765].fields[abolded2].name "abolded2" -classes[235127765].fields[abolded2].type "longstring" -classes[235127765].fields[aboldeddynamic].name "aboldeddynamic" -classes[235127765].fields[aboldeddynamic].type "longstring" -classes[235127765].fields[adynamic].name "adynamic" -classes[235127765].fields[adynamic].type "longstring" -classes[235127765].fields[adynamic2].name "adynamic2" -classes[235127765].fields[adynamic2].type "longstring" -classes[235127765].fields[b].name "b" -classes[235127765].fields[b].type "longstring" -classes[235127765].fields[c].name "c" -classes[235127765].fields[c].type "longstring" -classes[235127765].fields[d].name "d" -classes[235127765].fields[d].type "longstring" -classes[235127765].fields[documentid].name "documentid" -classes[235127765].fields[documentid].type "longstring" -classes[235127765].fields[dynamice].name "dynamice" -classes[235127765].fields[dynamice].type "longstring" -classes[235127765].fields[e].name "e" -classes[235127765].fields[e].type "longstring" -classes[235127765].fields[f].name "f" -classes[235127765].fields[f].type "jsonstring" -classes[235127765].fields[g].name "g" -classes[235127765].fields[g].type "jsonstring" -classes[235127765].fields[h].name "h" -classes[235127765].fields[h].type "jsonstring" -classes[235127765].fields[loc_pos.distance].name "loc_pos.distance" -classes[235127765].fields[loc_pos.distance].type "integer" -classes[235127765].fields[loc_pos.position].name "loc_pos.position" -classes[235127765].fields[loc_pos.position].type "xmlstring" -classes[235127765].fields[loc_position].name "loc_position" -classes[235127765].fields[loc_position].type "int64" -classes[235127765].fields[rankfeatures].name "rankfeatures" -classes[235127765].fields[rankfeatures].type "longstring" -classes[235127765].fields[summaryfeatures].name "summaryfeatures" -classes[235127765].fields[summaryfeatures].type "longstring" -classes[803323247].id 803323247 -classes[803323247].name "notattributesonly3" -classes[803323247].fields[a].name "a" -classes[803323247].fields[a].type "longstring" -classes[803323247].fields[d].name "d" -classes[803323247].fields[d].type "longstring" -classes[803323247].fields[rankfeatures].name "rankfeatures" -classes[803323247].fields[rankfeatures].type "longstring" -classes[803323247].fields[summaryfeatures].name "summaryfeatures" -classes[803323247].fields[summaryfeatures].type "longstring" -classes[937467944].id 937467944 -classes[937467944].name "notattributesonly5" -classes[937467944].fields[aboldeddynamic].name "aboldeddynamic" -classes[937467944].fields[aboldeddynamic].type "longstring" -classes[937467944].fields[c].name "c" -classes[937467944].fields[c].type "longstring" -classes[937467944].fields[rankfeatures].name "rankfeatures" -classes[937467944].fields[rankfeatures].type "longstring" -classes[937467944].fields[summaryfeatures].name "summaryfeatures" -classes[937467944].fields[summaryfeatures].type "longstring" +classes[].id 456145241 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "loc_pos" +classes[].fields[].type "jsonstring" +classes[].fields[].name "loc_pos.position" +classes[].fields[].type "xmlstring" +classes[].fields[].name "loc_pos.distance" +classes[].fields[].type "integer" +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "adynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "abolded" +classes[].fields[].type "longstring" +classes[].fields[].name "b" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "d" +classes[].fields[].type "longstring" +classes[].fields[].name "dynamice" +classes[].fields[].type "longstring" +classes[].fields[].name "f" +classes[].fields[].type "jsonstring" +classes[].fields[].name "g" +classes[].fields[].type "jsonstring" +classes[].fields[].name "h" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "e" +classes[].fields[].type "longstring" +classes[].fields[].name "adynamic2" +classes[].fields[].type "longstring" +classes[].fields[].name "alltags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "sometags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "anotherb" +classes[].fields[].type "longstring" +classes[].fields[].name "abolded2" +classes[].fields[].type "longstring" +classes[].fields[].name "aboldeddynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 783153771 +classes[].name "third" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "adynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "d" +classes[].fields[].type "longstring" +classes[].fields[].name "e" +classes[].fields[].type "longstring" +classes[].fields[].name "f" +classes[].fields[].type "jsonstring" +classes[].fields[].name "g" +classes[].fields[].type "jsonstring" +classes[].fields[].name "h" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 815922035 +classes[].name "attributesonly1" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1308077923 +classes[].name "notattributesonly1" +classes[].omitsummaryfeatures false +classes[].fields[].name "adynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1609068631 +classes[].name "anothernotattributesonly2" +classes[].omitsummaryfeatures false +classes[].fields[].name "adynamic2" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "alltags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "sometags" +classes[].fields[].type "jsonstring" +classes[].fields[].name "anothera" +classes[].fields[].type "longstring" +classes[].fields[].name "anotherb" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 686755772 +classes[].name "notattributesonly3" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "d" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1711750363 +classes[].name "attributesonly2" +classes[].omitsummaryfeatures false +classes[].fields[].name "anotdynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "loc_position" +classes[].fields[].type "int64" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1510953467 +classes[].name "attributesonly3" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "anotbolded" +classes[].fields[].type "longstring" +classes[].fields[].name "loc_pos_zcurve" +classes[].fields[].type "int64" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 923824943 +classes[].name "notattributesonly4" +classes[].omitsummaryfeatures false +classes[].fields[].name "abolded2" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 552611075 +classes[].name "notattributesonly5" +classes[].omitsummaryfeatures false +classes[].fields[].name "aboldeddynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 146047714 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "loc_pos_zcurve" +classes[].fields[].type "int64" +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 324773027 +classes[].name "second" +classes[].omitsummaryfeatures false +classes[].fields[].name "a" +classes[].fields[].type "longstring" +classes[].fields[].name "adynamic" +classes[].fields[].type "longstring" +classes[].fields[].name "c" +classes[].fields[].type "longstring" +classes[].fields[].name "f" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/multiplesummaries/summarymap.cfg b/config-model/src/test/derived/multiplesummaries/summarymap.cfg index 94c86ea8c80..94adc250c54 100644 --- a/config-model/src/test/derived/multiplesummaries/summarymap.cfg +++ b/config-model/src/test/derived/multiplesummaries/summarymap.cfg @@ -1,49 +1,61 @@ defaultoutputclass -1 -override[a].arguments "a" -override[a].command "attribute" -override[a].field "a" -override[abolded].arguments "abolded" -override[abolded].command "dynamicteaser" -override[abolded].field "abolded" -override[abolded2].arguments "abolded2" -override[abolded2].command "dynamicteaser" -override[abolded2].field "abolded2" -override[aboldeddynamic].arguments "aboldeddynamic" -override[aboldeddynamic].command "dynamicteaser" -override[aboldeddynamic].field "aboldeddynamic" -override[adynamic].arguments "adynamic" -override[adynamic].command "dynamicteaser" -override[adynamic].field "adynamic" -override[adynamic2].arguments "adynamic2" -override[adynamic2].command "dynamicteaser" -override[adynamic2].field "adynamic2" -override[anotbolded].arguments "a" -override[anotbolded].command "attribute" -override[anotbolded].field "anotbolded" -override[anotdynamic].arguments "adynamic" -override[anotdynamic].command "attribute" -override[anotdynamic].field "anotdynamic" -override[c].arguments "c" -override[c].command "attribute" -override[c].field "c" -override[d].arguments "d" -override[d].command "dynamicteaser" -override[d].field "d" -override[dynamice].arguments "dynamice" -override[dynamice].command "dynamicteaser" -override[dynamice].field "dynamice" -override[loc_pos.distance].arguments "loc_pos_zcurve" -override[loc_pos.distance].command "absdist" -override[loc_pos.distance].field "loc_pos.distance" -override[loc_pos.position].arguments "loc_pos_zcurve" -override[loc_pos.position].command "positions" -override[loc_pos.position].field "loc_pos.position" -override[loc_pos_zcurve].arguments "loc_pos_zcurve" -override[loc_pos_zcurve].command "attribute" -override[loc_pos_zcurve].field "loc_pos_zcurve" -override[rankfeatures].arguments "" -override[rankfeatures].command "rankfeatures" -override[rankfeatures].field "rankfeatures" -override[summaryfeatures].arguments "" -override[summaryfeatures].command "summaryfeatures" -override[summaryfeatures].field "summaryfeatures" +override[].field "a" +override[].command "attribute" +override[].arguments "a" +override[].field "adynamic" +override[].command "dynamicteaser" +override[].arguments "adynamic" +override[].field "d" +override[].command "dynamicteaser" +override[].arguments "d" +override[].field "rankfeatures" +override[].command "rankfeatures" +override[].arguments "" +override[].field "summaryfeatures" +override[].command "summaryfeatures" +override[].arguments "" +override[].field "c" +override[].command "attribute" +override[].arguments "c" +override[].field "adynamic2" +override[].command "dynamicteaser" +override[].arguments "adynamic2" +override[].field "sometags" +override[].command "matchedelementsfilter" +override[].arguments "mytags" +override[].field "anothera" +override[].command "attribute" +override[].arguments "a" +override[].field "anotdynamic" +override[].command "attribute" +override[].arguments "adynamic" +override[].field "loc_position" +override[].command "attribute" +override[].arguments "loc_pos_zcurve" +override[].field "anotbolded" +override[].command "attribute" +override[].arguments "a" +override[].field "loc_pos_zcurve" +override[].command "attribute" +override[].arguments "loc_pos_zcurve" +override[].field "abolded2" +override[].command "dynamicteaser" +override[].arguments "abolded2" +override[].field "aboldeddynamic" +override[].command "dynamicteaser" +override[].arguments "aboldeddynamic" +override[].field "loc_pos" +override[].command "geopos" +override[].arguments "loc_pos_zcurve" +override[].field "loc_pos.position" +override[].command "positions" +override[].arguments "loc_pos_zcurve" +override[].field "loc_pos.distance" +override[].command "absdist" +override[].arguments "loc_pos_zcurve" +override[].field "abolded" +override[].command "dynamicteaser" +override[].arguments "abolded" +override[].field "dynamice" +override[].command "dynamicteaser" +override[].arguments "dynamice" diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg index cd7c295ab11..2c46031bdad 100644 --- a/config-model/src/test/derived/position_nosummary/summary.cfg +++ b/config-model/src/test/derived/position_nosummary/summary.cfg @@ -1,12 +1,8 @@ -defaultsummaryid 1727020212 -usev8geopositions false -classes[].id 1727020212 +defaultsummaryid 1151071433 +usev8geopositions true +classes[].id 1151071433 classes[].name "default" classes[].omitsummaryfeatures false -classes[].fields[].name "pos.position" -classes[].fields[].type "xmlstring" -classes[].fields[].name "pos.distance" -classes[].fields[].type "integer" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/position_nosummary/summarymap.cfg b/config-model/src/test/derived/position_nosummary/summarymap.cfg index cc1c14a6614..0a36f98c6ca 100644 --- a/config-model/src/test/derived/position_nosummary/summarymap.cfg +++ b/config-model/src/test/derived/position_nosummary/summarymap.cfg @@ -1,10 +1,4 @@ defaultoutputclass -1 -override[].field "pos.position" -override[].command "positions" -override[].arguments "pos_zcurve" -override[].field "pos.distance" -override[].command "absdist" -override[].arguments "pos_zcurve" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg index 7b8bf16287f..7fda1ca0c05 100644 --- a/config-model/src/test/derived/position_summary/summary.cfg +++ b/config-model/src/test/derived/position_summary/summary.cfg @@ -1,14 +1,10 @@ -defaultsummaryid 230670304 -usev8geopositions false -classes[].id 230670304 +defaultsummaryid 644855547 +usev8geopositions true +classes[].id 644855547 classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "pos" classes[].fields[].type "jsonstring" -classes[].fields[].name "pos.position" -classes[].fields[].type "xmlstring" -classes[].fields[].name "pos.distance" -classes[].fields[].type "integer" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/position_summary/summarymap.cfg b/config-model/src/test/derived/position_summary/summarymap.cfg index de48a19b6db..2223d3380a0 100644 --- a/config-model/src/test/derived/position_summary/summarymap.cfg +++ b/config-model/src/test/derived/position_summary/summarymap.cfg @@ -2,12 +2,6 @@ defaultoutputclass -1 override[].field "pos" override[].command "geopos" override[].arguments "pos_zcurve" -override[].field "pos.position" -override[].command "positions" -override[].arguments "pos_zcurve" -override[].field "pos.distance" -override[].command "absdist" -override[].arguments "pos_zcurve" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" diff --git a/config-model/src/test/derived/position_summary/vsmsummary.cfg b/config-model/src/test/derived/position_summary/vsmsummary.cfg index d7d45782843..56405069131 100644 --- a/config-model/src/test/derived/position_summary/vsmsummary.cfg +++ b/config-model/src/test/derived/position_summary/vsmsummary.cfg @@ -2,12 +2,6 @@ outputclass "" fieldmap[].summary "pos" fieldmap[].document[].field "pos" fieldmap[].command NONE -fieldmap[].summary "pos.position" -fieldmap[].document[].field "pos_zcurve" -fieldmap[].command NONE -fieldmap[].summary "pos.distance" -fieldmap[].document[].field "pos_zcurve" -fieldmap[].command NONE fieldmap[].summary "rankfeatures" fieldmap[].command NONE fieldmap[].summary "summaryfeatures" diff --git a/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg b/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg new file mode 100644 index 00000000000..3af23fd3770 --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg @@ -0,0 +1,48 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "fieldMatch(title).completeness" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].name "in_schema0" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "fieldMatch(title).completeness" +rankprofile[].name "in_schema1" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].name "in_schema2" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "rankingExpression(f2).rankingScript" +rankprofile[].fef.property[].value "fieldMatch(title) + rankingExpression(fo2)" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(f2)" +rankprofile[].name "in_schema3" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "rankingExpression(f2).rankingScript" +rankprofile[].fef.property[].value "fieldMatch(title) + rankingExpression(fo2)" +rankprofile[].fef.property[].name "rankingExpression(fo1).rankingScript" +rankprofile[].fef.property[].value "now" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(f2)" +rankprofile[].name "outside_schema1" +rankprofile[].fef.property[].name "rankingExpression(fo1).rankingScript" +rankprofile[].fef.property[].value "now" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].name "outside_schema2" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "fieldMatch(title).completeness" diff --git a/config-model/src/test/derived/rankprofilemodularity/test.sd b/config-model/src/test/derived/rankprofilemodularity/test.sd new file mode 100644 index 00000000000..34414414d6c --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/test.sd @@ -0,0 +1,49 @@ +schema test { + + document test { + + field title type string { + indexing: index + } + + } + + rank-profile default inherits outside_schema2 { + } + + rank-profile in_schema0 inherits outside_schema2 { + } + + rank-profile in_schema1 { + + first-phase { + expression: nativeRank + } + + } + + rank-profile in_schema2 inherits outside_schema2 { + + function f2() { + expression: fieldMatch(title) + fo2 + } + + first-phase { + expression: f2 + } + + } + + rank-profile in_schema3 inherits outside_schema1, outside_schema2 { + + function f2() { + expression: fieldMatch(title) + fo2 + } + + first-phase { + expression: f2 + } + + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile new file mode 100644 index 00000000000..a8092689b7f --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile @@ -0,0 +1,7 @@ +rank-profile outside_schema1 inherits in_schema1 { + + function fo1() { + expression: now + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/rankprofilemodularity/test/outside_schema2.profile b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema2.profile new file mode 100644 index 00000000000..8cf3fcfbb78 --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema2.profile @@ -0,0 +1,11 @@ +rank-profile outside_schema2 { + + function fo2() { + expression: random + } + + first-phase { + expression: fieldMatch(title).completeness + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/renamedfeatures/foo.sd b/config-model/src/test/derived/renamedfeatures/foo.sd index f7884dd937d..462eb0da44e 100644 --- a/config-model/src/test/derived/renamedfeatures/foo.sd +++ b/config-model/src/test/derived/renamedfeatures/foo.sd @@ -1,6 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search foo { - + document foo { field title type string { indexing: summary | index diff --git a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg index b78866fdf05..0a327575a48 100644 --- a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg +++ b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg @@ -79,4 +79,4 @@ rankprofile[3].fef.property[10].value "rankingExpression(mymul)" rankprofile[3].fef.property[11].name "vespa.feature.rename" rankprofile[3].fef.property[11].value "mymul" rankprofile[3].fef.property[12].name "vespa.type.attribute.t1" -rankprofile[3].fef.property[12].value "tensor(m{},v[3])"
\ No newline at end of file +rankprofile[3].fef.property[12].value "tensor(m{},v[3])" diff --git a/config-model/src/test/derived/slice/query-profiles/default.xml b/config-model/src/test/derived/slice/query-profiles/default.xml new file mode 100644 index 00000000000..2535ca895ed --- /dev/null +++ b/config-model/src/test/derived/slice/query-profiles/default.xml @@ -0,0 +1,3 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default" type="DefaultQueryProfileType"> +</query-profile> diff --git a/config-model/src/test/derived/slice/query-profiles/types/DefaultQueryProfileType.xml b/config-model/src/test/derived/slice/query-profiles/types/DefaultQueryProfileType.xml new file mode 100644 index 00000000000..50970d8743f --- /dev/null +++ b/config-model/src/test/derived/slice/query-profiles/types/DefaultQueryProfileType.xml @@ -0,0 +1,4 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="DefaultQueryProfileType" inherits="native"> + <field name="ranking.features.query(myTensor)" type="tensor<float>(key{})" /> +</query-profile-type> diff --git a/config-model/src/test/derived/slice/rank-profiles.cfg b/config-model/src/test/derived/slice/rank-profiles.cfg new file mode 100644 index 00000000000..75725b81ecf --- /dev/null +++ b/config-model/src/test/derived/slice/rank-profiles.cfg @@ -0,0 +1,25 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "vespa.type.query.myTensor" +rankprofile[].fef.property[].value "tensor<float>(key{})" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.query.myTensor" +rankprofile[].fef.property[].value "tensor<float>(key{})" +rankprofile[].name "parent" +rankprofile[].fef.property[].name "rankingExpression(mySlice@77dee0712164ce73).rankingScript" +rankprofile[].fef.property[].value "query(myTensor){key:MY_KEY2}" +rankprofile[].fef.property[].name "rankingExpression(myFunction).rankingScript" +rankprofile[].fef.property[].value "4 * query(myTensor){key:MY_KEY1} * rankingExpression(mySlice@77dee0712164ce73)" +rankprofile[].fef.property[].name "rankingExpression(myValue).rankingScript" +rankprofile[].fef.property[].value "4" +rankprofile[].fef.property[].name "rankingExpression(mySlice).rankingScript" +rankprofile[].fef.property[].value "myTensor{key:MY_KEY2}" +rankprofile[].fef.property[].name "vespa.type.query.myTensor" +rankprofile[].fef.property[].value "tensor<float>(key{})" diff --git a/config-model/src/test/derived/slice/test.sd b/config-model/src/test/derived/slice/test.sd new file mode 100644 index 00000000000..c2060300785 --- /dev/null +++ b/config-model/src/test/derived/slice/test.sd @@ -0,0 +1,23 @@ +search test { + + document test { + } + + rank-profile parent { + + function inline myFunction() { + expression: myValue * query(myTensor){MY_KEY1} * mySlice(query(myTensor)) + } + + function inline myValue() { + expression: 4 + } + + function inline mySlice(myTensor) { + # TODO: We are missing type resolving across function calls in serialization, + # so using the short form (without 'key') here will fail + expression: myTensor{key:MY_KEY2} + } + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/twostreamingstructs/whatever.sd b/config-model/src/test/derived/twostreamingstructs/whatever.sd index 34b0261326f..17ac30102a8 100644 --- a/config-model/src/test/derived/twostreamingstructs/whatever.sd +++ b/config-model/src/test/derived/twostreamingstructs/whatever.sd @@ -9,7 +9,6 @@ search whatever { field f1 type pair { indexing: summary - body } } diff --git a/config-model/src/test/examples/header_body.sd b/config-model/src/test/examples/header_body.sd deleted file mode 100644 index 61ddbaa7238..00000000000 --- a/config-model/src/test/examples/header_body.sd +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Search definition with (ignored) header and body statements -# TODO: Remove in Vespa 8 -search header_body { - - document header_body { - - field title type string { - indexing: summary | index - header - } - - field description type string { - indexing: summary | index - body - } - } -} diff --git a/config-model/src/test/examples/nextgen/summaryfield.sd b/config-model/src/test/examples/nextgen/summaryfield.sd index 9b3cc6862b9..99c73d1be53 100644 --- a/config-model/src/test/examples/nextgen/summaryfield.sd +++ b/config-model/src/test/examples/nextgen/summaryfield.sd @@ -5,11 +5,21 @@ search summaryfield { indexing: index | summary summary bar: full } + field mytags type array<string> { + indexing: index + } } document-summary baz { summary cox type string { source: bar } + summary alltags type array<string> { + source: mytags + } + summary sometags type array<string> { + source: mytags + matched-elements-only + } } } diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 0ccbec529ed..3acc44113d0 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -18,7 +18,6 @@ import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.NamedSchema; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -55,19 +54,18 @@ public class ApplicationDeployTest { public void testVespaModel() throws SAXException, IOException { ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "app1"); new VespaModel(tester.app()); - List<NamedSchema> schemas = tester.getSchemas(); + List<Schema> schemas = tester.getSchemas(); assertEquals(schemas.size(), 5); - for (NamedSchema searchDefinition : schemas) { - Schema s = searchDefinition.getSearch(); - switch (s.getName()) { + for (Schema schema : schemas) { + switch (schema.getName()) { case "music": case "laptop": case "pc": case "sock": break; case "product": - assertTrue(s instanceof DocumentOnlySchema); - assertEquals(DataType.STRING, s.getDocument().getField("title").getDataType()); + assertTrue(schema instanceof DocumentOnlySchema); + assertEquals(DataType.STRING, schema.getDocument().getField("title").getDataType()); break; default: fail(); @@ -95,8 +93,8 @@ public class ApplicationDeployTest { // Check that getFilename works ArrayList<String> sdFileNames = new ArrayList<>(); - for (NamedSchema sd : schemas) - sdFileNames.add(sd.getFilename()); + for (Schema schema : schemas) + sdFileNames.add(schema.getName() + ApplicationPackage.SD_NAME_SUFFIX); Collections.sort(sdFileNames); assertEquals("laptop.sd", sdFileNames.get(0)); assertEquals("music.sd", sdFileNames.get(1)); diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java index 3d799375420..96186150108 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.search.NamedSchema; +import com.yahoo.searchdefinition.Schema; import java.io.File; import java.io.IOException; @@ -39,7 +39,7 @@ public class ApplicationPackageTester { public FilesApplicationPackage app() { return applicationPackage; } - public List<NamedSchema> getSchemas() { + public List<Schema> getSchemas() { return new DeployState.Builder().applicationPackage(app()).build().getSchemas(); } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 10f883bdc75..ba70b7493a2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -2054,7 +2054,7 @@ public class ModelProvisioningTest { assertTrue("Initial servers are not joining", config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining)); } { - VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, false, 0, Optional.of(model), new DeployState.Builder()); + VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, 0, Optional.of(model), new DeployState.Builder(), "node-1-3-10-04", "node-1-3-10-03"); ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk"); ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder(); cluster.getContainers().forEach(c -> c.getConfig(config)); @@ -2067,6 +2067,14 @@ public class ModelProvisioningTest { 4, true), config.build().server().stream().collect(Collectors.toMap(ZookeeperServerConfig.Server::id, ZookeeperServerConfig.Server::joining))); + assertEquals("Retired nodes are retired", + Map.of(0, false, + 1, true, + 2, true, + 3, false, + 4, false), + config.build().server().stream().collect(Collectors.toMap(ZookeeperServerConfig.Server::id, + ZookeeperServerConfig.Server::retired))); } } diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java index f1bebdb0a29..d4ebbe62507 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java @@ -5,12 +5,10 @@ import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.ImportedFieldsResolver; -import com.yahoo.searchdefinition.processing.OnnxModelTypeResolver; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; @@ -122,8 +120,8 @@ public class SDDocumentTypeTestCase extends AbstractSchemaTestCase { " }" + "}"); - SchemaBuilder builder = new SchemaBuilder(new DeployLoggerStub()); - builder.importString(schemaLines); + ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); + builder.addSchema(schemaLines); builder.build(true); var application = builder.application(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java index aaf75f0c852..2ed1ab6892a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java @@ -22,7 +22,7 @@ public class ArraysTestCase extends AbstractSchemaTestCase { @Test public void testArrayImporting() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/arrays.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arrays.sd"); SDField tags = (SDField) schema.getDocument().getField("tags"); assertEquals(DataType.STRING, ((CollectionDataType)tags.getDataType()).getNestedType()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java index 1dd2c987d90..9c70fcf79f7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertTrue; public class ArraysWeightedSetsTestCase extends AbstractSchemaTestCase { @Test public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); SDField tags = (SDField) schema.getDocument().getField("tags"); assertTrue(tags.getDataType() instanceof ArrayDataType); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index e4de0aef622..7b2ed2d7a7f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -30,7 +30,7 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void testAttributeSettings() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/attributesettings.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); SDField f1=(SDField) schema.getDocument().getField("f1"); assertEquals(1, f1.getAttributes().size()); @@ -91,22 +91,22 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireThatFastAccessCanBeSet() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/attributesettings.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); SDField field = (SDField) schema.getDocument().getField("fast_access"); assertEquals(1, field.getAttributes().size()); Attribute attr = field.getAttributes().get(field.getName()); assertTrue(attr.isFastAccess()); } - private Schema getSearch(String sd) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(sd); - builder.build(); + private Schema getSchema(String sd) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(sd); + builder.build(true); return builder.getSchema(); } private Attribute getAttributeF(String sd) throws ParseException { - Schema schema = getSearch(sd); + Schema schema = getSchema(sd); SDField field = (SDField) schema.getDocument().getField("f"); return field.getAttributes().get(field.getName()); } @@ -153,14 +153,14 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireThatMutableCanNotbeSetInDocument() throws ParseException { try { - getSearch("search test {\n" + - " document test {\n" + - " field f type int {\n" + - " indexing: attribute\n" + - " attribute: mutable\n" + - " }\n" + - " }\n" + - "}\n"); + getSchema("search test {\n" + + " document test {\n" + + " field f type int {\n" + + " indexing: attribute\n" + + " attribute: mutable\n" + + " }\n" + + " }\n" + + "}\n"); fail(); } catch (IllegalArgumentException e) { assertEquals("Field 'f' in 'test' can not be marked mutable as it is inside the document clause.", e.getMessage()); @@ -185,7 +185,7 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { } private Schema getSearchWithMutables() throws ParseException { - return getSearch( + return getSchema( "search test {\n" + " document test { \n" + " field a type int { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java index 8b7f0885907..3e92c1df9e7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java @@ -18,7 +18,7 @@ public class CommentTestCase extends AbstractSchemaTestCase { @Test public void testComments() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/comment.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/comment.sd"); SDField field = schema.getConcreteField("a"); assertEquals("{ input a | tokenize normalize stem:\"BEST\" | summary a | index a; }", field.getIndexingScript().toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java index 7c5c5c6fccc..ca7313a5c3f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java @@ -15,8 +15,8 @@ public class DiversityTestCase { @Test public void testDiversity() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type int { \n" + @@ -42,7 +42,7 @@ public class DiversityTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); @@ -60,10 +60,10 @@ public class DiversityTestCase { } @Test public void requireSingleNumericOrString() throws ParseException { - SchemaBuilder builder = getSearchBuilder("field b type predicate { indexing: attribute }"); + ApplicationBuilder builder = getSearchBuilder("field b type predicate { indexing: attribute }"); try { - builder.build(); + builder.build(true); fail("Should throw."); } catch (IllegalArgumentException e) { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'predicate'", e.getMessage()); @@ -72,19 +72,19 @@ public class DiversityTestCase { @Test public void requireSingle() throws ParseException { - SchemaBuilder builder = getSearchBuilder("field b type array<int> { indexing: attribute }"); + ApplicationBuilder builder = getSearchBuilder("field b type array<int> { indexing: attribute }"); try { - builder.build(); + builder.build(true); fail("Should throw."); } catch (IllegalArgumentException e) { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array<int>'", e.getMessage()); } } - private SchemaBuilder getSearchBuilder(String diversity) throws ParseException { + private ApplicationBuilder getSearchBuilder(String diversity) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type int { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java index fb552f5cfb0..81a44261daf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -143,7 +144,7 @@ public class DocumentGraphValidatorTest { } private static Schema createSearchWithName(String name, Schema... parents) { - Schema campaignSchema = new Schema(name); + Schema campaignSchema = new Schema(name, MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType(name); campaignSchema.addDocument(document); document.setDocumentReferences(new DocumentReferences(Collections.emptyMap())); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index cc45a2aaec0..66f1850bd10 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TemporaryStructuredDataType; @@ -32,7 +33,7 @@ public class DocumentReferenceResolverTest { @Test public void reference_from_one_document_to_another_is_resolved() { // Create bar document with no fields - Schema barSchema = new Schema(BAR); + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); barSchema.addDocument(barDocument); @@ -41,7 +42,7 @@ public class DocumentReferenceResolverTest { ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); AttributeUtils.addAttributeAspect(fooRefToBarField); SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); - Schema fooSchema = new Schema(FOO); + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooDocument.addField(irrelevantField); @@ -64,7 +65,7 @@ public class DocumentReferenceResolverTest { SDField fooRefToBarField = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); AttributeUtils.addAttributeAspect(fooRefToBarField); - Schema fooSchema = new Schema(FOO); + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); @@ -80,14 +81,14 @@ public class DocumentReferenceResolverTest { @Test public void throws_exception_if_reference_is_not_an_attribute() { // Create bar document with no fields - Schema barSchema = new Schema(BAR); + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); SDDocumentType barDocument = new SDDocumentType("bar", barSchema); barSchema.addDocument(barDocument); // Create foo document with document reference to bar SDField fooRefToBarField = new SDField ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - Schema fooSchema = new Schema(FOO); + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java index 4057fc469bc..7d2386030da 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.ReferenceDataType; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -19,7 +20,7 @@ public class ImportedFieldsEnumeratorTest { @Test public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { String PARENT = "parent"; - Schema parentSchema = new Schema(PARENT); + Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); var parentField = new SDField("their_field", DataType.INT); AttributeUtils.addAttributeAspect(parentField); @@ -27,7 +28,7 @@ public class ImportedFieldsEnumeratorTest { parentSchema.addDocument(parentDocument); String FOO = "foo"; - Schema fooSchema = new Schema(FOO); + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDField fooRefToParent = new SDField( "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(fooRefToParent); @@ -38,7 +39,7 @@ public class ImportedFieldsEnumeratorTest { fooSchema.addDocument(fooDocument); String BAR = "bar"; - Schema barSchema = new Schema(BAR); + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); SDField barRefToParent = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(barRefToParent); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 621e7061203..23be59e38f5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -21,9 +21,9 @@ public class IncorrectRankingExpressionFileRefTestCase extends AbstractSchemaTes public void testIncorrectRef() throws IOException, ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", - registry, - new QueryProfileRegistry()); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", + registry, + new QueryProfileRegistry()); new DerivedConfiguration(schema, registry); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java index 55593ef64e4..d81d07bf6e2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java @@ -15,7 +15,7 @@ public class IncorrectSummaryTypesTestCase extends AbstractSchemaTestCase { @Test public void testImportingIncorrect() throws ParseException { try { - SchemaBuilder.createFromString( + ApplicationBuilder.createFromString( "search incorrectsummarytypes {\n" + " document incorrectsummarytypes {\n" + " field somestring type string {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java index f2e50439306..198f8b6c79e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java @@ -21,7 +21,7 @@ public class IndexSettingsTestCase extends AbstractSchemaTestCase { @Test public void testStemmingSettings() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/indexsettings.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/indexsettings.sd"); SDField usingDefault=(SDField) schema.getDocument().getField("usingdefault"); assertEquals(Stemming.SHORTEST,usingDefault.getStemming(schema)); @@ -38,7 +38,7 @@ public class IndexSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireThatInterlavedFeaturesAreSetOnExtraField() throws ParseException { - SchemaBuilder builder = SchemaBuilder.createFromString(joinLines( + ApplicationBuilder builder = ApplicationBuilder.createFromString(joinLines( "search test {", " document test {", " field content type string {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java index 60728ab7d99..8ba7dfdb14f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java @@ -15,13 +15,13 @@ public class IndexingParsingTestCase extends AbstractSchemaTestCase { @Test public void requireThatIndexingExpressionsCanBeParsed() throws Exception { - assertNotNull(SchemaBuilder.buildFromFile("src/test/examples/indexing.sd")); + assertNotNull(ApplicationBuilder.buildFromFile("src/test/examples/indexing.sd")); } @Test public void requireThatParseExceptionPositionIsCorrect() throws Exception { try { - SchemaBuilder.buildFromFile("src/test/examples/indexing_invalid_expression.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/indexing_invalid_expression.sd"); } catch (ParseException e) { if (!e.getMessage().contains("at line 5, column 57.")) { throw e; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java index 41321b5c81e..0fcc6e09b02 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java @@ -14,6 +14,6 @@ import java.io.IOException; public class MultipleSummariesTestCase extends AbstractSchemaTestCase { @Test public void testArrayImporting() throws IOException, ParseException { - SchemaBuilder.buildFromFile("src/test/examples/multiplesummaries.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/multiplesummaries.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java index fc94bbe8731..1e766b7793d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java @@ -19,7 +19,7 @@ public class NameFieldCheckTestCase extends AbstractSchemaTestCase { @Test public void testNameField() { try { - SchemaBuilder.createFromString( + ApplicationBuilder.createFromString( "search simple {\n" + " document name-check {\n" + " field title type string {\n" + @@ -40,7 +40,7 @@ public class NameFieldCheckTestCase extends AbstractSchemaTestCase { @Test public void testDuplicateNamesInSearchDifferentType() { try { - SchemaBuilder.createFromString( + ApplicationBuilder.createFromString( "search duplicatenamesinsearch {\n" + " document {\n" + " field grpphotoids64 type string { }\n" + @@ -60,7 +60,7 @@ public class NameFieldCheckTestCase extends AbstractSchemaTestCase { @Test public void testDuplicateNamesInDoc() { try { - SchemaBuilder.createFromString( + ApplicationBuilder.createFromString( "search duplicatenamesindoc {\n" + " document {\n" + " field foo type int {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java index 5663473481b..6563fb8d3f1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java @@ -17,7 +17,7 @@ public class OutsideTestCase extends AbstractSchemaTestCase { @Test public void testOutsideIndex() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/outsidedoc.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/outsidedoc.sd"); Index defaultIndex= schema.getIndex("default"); assertTrue(defaultIndex.isPrefix()); @@ -26,7 +26,7 @@ public class OutsideTestCase extends AbstractSchemaTestCase { @Test public void testOutsideSummary() throws IOException, ParseException { - SchemaBuilder.buildFromFile("src/test/examples/outsidesummary.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/outsidesummary.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java index 456a236f293..2b49f3d67a6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java @@ -68,7 +68,7 @@ public class PredicateDataTypeTestCase { lowerBoundParameter(lowerBound) + upperBoundParameter(upperBound)))); - SchemaBuilder sb = SchemaBuilder.createFromString(sd); + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { @@ -95,7 +95,7 @@ public class PredicateDataTypeTestCase { "lower-bound: -100000000000000000L\n" + // +'L' upperBoundParameter(upperBound)))); - SchemaBuilder sb = SchemaBuilder.createFromString(sd); + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { @@ -113,7 +113,7 @@ public class PredicateDataTypeTestCase { predicateFieldSd( attributeFieldSd( arityParameter(2)))); - SchemaBuilder sb = SchemaBuilder.createFromString(sd); + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { @@ -131,7 +131,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Missing arity value in predicate field."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); fail(); } @@ -141,7 +141,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("For schema 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @Test @@ -150,7 +150,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("For schema 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @@ -160,7 +160,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Invalid arity value in predicate field, must be greater than 1."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @Test @@ -169,7 +169,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Arity parameter is used only for predicate type fields."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @Test @@ -181,7 +181,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Parameters lower-bound and upper-bound are used only for predicate type fields."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } @Test @@ -193,7 +193,7 @@ public class PredicateDataTypeTestCase { exception.expect(IllegalArgumentException.class); exception.expectMessage("Collections of predicates are not allowed."); - SchemaBuilder.createFromString(sd); + ApplicationBuilder.createFromString(sd); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java index d2486395340..d07f4513c3c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -33,7 +34,7 @@ public class RankProfileRegistryTest { @Test(expected = IllegalArgumentException.class) public void testRankProfileDuplicateNameIsIllegal() { - Schema schema = new Schema("foo"); + Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); RankProfile barRankProfile = new RankProfile("bar", schema, rankProfileRegistry, schema.rankingConstants()); rankProfileRegistry.add(barRankProfile); @@ -42,7 +43,7 @@ public class RankProfileRegistryTest { @Test public void testRankProfileDuplicateNameLegalForOverridableRankProfiles() { - Schema schema = new Schema("foo"); + Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 3fa129eae39..d77c6d822e3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; @@ -43,7 +44,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void testRankProfileInheritance() { - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("test"); SDField a = document.addField("a", DataType.STRING); @@ -51,7 +52,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { document.addField("b", DataType.STRING); schema.addDocument(document); RankProfile child = new RankProfile("child", schema, rankProfileRegistry, schema.rankingConstants()); - child.setInherited("default"); + child.inherit("default"); rankProfileRegistry.add(child); Iterator<RankProfile.RankSetting> i = child.rankSettingIterator(); @@ -69,8 +70,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { public void requireThatIllegalInheritanceIsChecked() throws ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "search test {", " document test { } ", " rank-profile p1 inherits notexist {}", @@ -78,7 +79,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { builder.build(true); fail(); } catch (IllegalArgumentException e) { - assertEquals("rank-profile 'p1' inherits 'notexist', but it does not exist anywhere in the inheritance of search 'test'.", e.getMessage()); + assertEquals("rank-profile 'p1' inherits 'notexist', but this is not found in schema 'test'", e.getMessage()); } } @@ -86,8 +87,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { public void requireThatSelfInheritanceIsIllegal() throws ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "schema test {", " document test { } ", " rank-profile self inherits self {}", @@ -102,13 +103,13 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatSelfInheritanceIsLegalWhenOverloading() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "schema base {", " document base { } ", " rank-profile self inherits default {}", "}")); - builder.importString(joinLines( + builder.addSchema(joinLines( "schema test {", " document test inherits base { } ", " rank-profile self inherits self {}", @@ -119,8 +120,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatSidewaysInheritanceIsImpossible() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "schema child1 {", " document child1 {", " field field1 type int {", @@ -141,7 +142,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { " }", " }", "}\n")); - builder.importString(joinLines( + builder.addSchema(joinLines( "schema child2 {", " document child2 {", " field field1 type int {", @@ -165,15 +166,15 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { builder.build(true); fail("Sideways inheritance should have been enforced"); } catch (IllegalArgumentException e) { - assertEquals("rank-profile 'child' inherits 'parent', but it does not exist anywhere in the inheritance of search 'child1'.", e.getMessage()); + assertEquals("rank-profile 'child' inherits 'parent', but this is not found in schema 'child1'", e.getMessage()); } } @Test - public void requireThatDefaultCanAlwaysBeInherited() throws ParseException { + public void requireThatDefaultInheritingDefaultIsIgnored() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "schema test {", " document test { } ", " rank-profile default inherits default {}", @@ -185,8 +186,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { public void requireThatCyclicInheritanceIsIllegal() throws ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "search test {", " document test { } ", " rank-profile a inherits b {}", @@ -204,8 +205,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { public void requireThatRankProfilesCanInheritNotYetSeenProfiles() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "search test {", " document test { } ", " rank-profile p1 inherits not_yet_defined {}", @@ -246,9 +247,9 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { private void verifyTermwiseLimitAndSomeMoreIncludingInheritance(ModelContext.Properties deployProperties, String sd, Double termwiseLimit) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString(sd); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(sd); + builder.build(true); Schema schema = builder.getSchema(); AttributeFields attributeFields = new AttributeFields(schema); verifyRankProfile(rankProfileRegistry.get(schema, "parent"), attributeFields, deployProperties, termwiseLimit); @@ -279,8 +280,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatConfigIsDerivedForAttributeTypeSettings() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10]) { indexing: attribute }", @@ -290,7 +291,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { " rank-profile p1 {}", " rank-profile p2 {}", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); assertEquals(4, registry.all().size()); @@ -303,14 +304,14 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatDenseDimensionsMustBeBound() throws ParseException { try { - SchemaBuilder builder = new SchemaBuilder(new RankProfileRegistry()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[]) { indexing: attribute }", " }", "}")); - builder.build(); + builder.build(true); } catch (IllegalArgumentException e) { assertEquals("Illegal type in field a type tensor(x[]): Dense tensor dimensions must have a size", @@ -332,8 +333,8 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { @Test public void requireThatConfigIsDerivedForQueryFeatureTypeSettings() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(registry, setupQueryProfileTypes()); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( "search test {", " document test { } ", " rank-profile p1 {}", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java index d6065e74af5..6de890549c1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java @@ -23,8 +23,8 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { @Test public void testRankPropertyInheritance() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( "search test {", " document test {", " field a type string { ", @@ -48,7 +48,7 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { " }", " }", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); AttributeFields attributeFields = new AttributeFields(schema); @@ -80,8 +80,8 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { @Test public void testRankProfileMutate() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( "search test {", " document test {", " field a type int { ", @@ -128,7 +128,7 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { " }", " }", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); RankProfile a = rankProfileRegistry.get(schema, "a"); List<RankProfile.MutateOperation> operations = a.getMutateOperations(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java index 1925b7e77b9..6eb74dede62 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java @@ -28,8 +28,8 @@ public class RankingConstantTest { final String TENSOR_FILE = "path/my-tensor-file.json"; final String TENSOR_TYPE = "tensor(x{})"; RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " rank-profile my_rank_profile {", @@ -43,7 +43,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); Iterator<RankingConstant> constantIterator = schema.rankingConstants().asMap().values().iterator(); @@ -59,10 +59,10 @@ public class RankingConstantTest { @Test public void tensor_constant_must_have_a_type() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("must have a type"); - schemaBuilder.importString(joinLines( + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -75,10 +75,10 @@ public class RankingConstantTest { @Test public void tensor_constant_must_have_a_file() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("must have a file"); - schemaBuilder.importString(joinLines( + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -91,8 +91,8 @@ public class RankingConstantTest { @Test public void constant_file_does_not_need_path_or_ending() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -101,7 +101,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals("simplename", constant.getFileName()); @@ -110,8 +110,8 @@ public class RankingConstantTest { @Test public void constant_uri_is_allowed() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -120,7 +120,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -130,8 +130,8 @@ public class RankingConstantTest { @Test public void constant_https_uri_is_allowed() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -140,7 +140,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -150,8 +150,8 @@ public class RankingConstantTest { @Test public void constant_uri_with_port_is_allowed() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -160,7 +160,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -170,8 +170,8 @@ public class RankingConstantTest { @Test public void constant_uri_no_dual_slashes_is_allowed() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", @@ -180,7 +180,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -190,12 +190,12 @@ public class RankingConstantTest { @Test public void constant_uri_only_supports_http_and_https() { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder schemaBuilder = new SchemaBuilder(rankProfileRegistry); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); String expectedMessage = "Encountered \" <IDENTIFIER> \"ftp\"\" at line 5, column 10.\n\n" + "Was expecting:\n\n" + "<URI_PATH> ..."; try { - schemaBuilder.importString(joinLines( + schemaBuilder.addSchema(joinLines( "search test {", " document test { }", " constant foo {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 87b406c39fa..4646ff2f394 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -25,8 +25,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { public void testConstants() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -66,7 +66,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(queryProfileRegistry, new ImportedMlModels()); assertEquals("0.0", parent.getFirstPhaseRanking().getRoot().toString()); @@ -91,8 +91,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void testNameCollision() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -110,7 +110,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); try { rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -125,8 +125,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void testNegativeLiteralArgument() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -141,7 +141,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); @@ -150,8 +150,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void testNegativeConstantArgument() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -169,7 +169,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); @@ -180,8 +180,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void testConstantDivisorInFunction() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " }\n" + @@ -193,7 +193,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 1.0E8", @@ -203,8 +203,8 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { @Test public void test3() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field rating_yelp type int {" + @@ -219,7 +219,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index 0598fee538a..94d25deb16a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -28,8 +28,8 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { @Test public void testFunctionInliningPreserveArithmeticOrdering() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type double { \n" + @@ -65,7 +65,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -79,8 +79,8 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { @Test public void testConstants() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -125,7 +125,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -150,8 +150,8 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { @Test public void testNonTopLevelInlining() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type double { \n" + @@ -181,7 +181,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -194,13 +194,13 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { public void testFunctionInliningWithReplacement() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); MockDeployLogger deployLogger = new MockDeployLogger(); - SchemaBuilder builder = new SchemaBuilder(MockApplicationPackage.createEmpty(), - new MockFileRegistry(), - deployLogger, - new TestProperties(), - rankProfileRegistry, - new QueryProfileRegistry()); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + deployLogger, + new TestProperties(), + rankProfileRegistry, + new QueryProfileRegistry()); + builder.addSchema( "search test {\n" + " document test { }\n" + " rank-profile test {\n" + @@ -215,7 +215,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); assertEquals("foo(2)", test.getFirstPhaseRanking().getRoot().toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java index 689b75690ae..ff8a9df4076 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java @@ -16,8 +16,8 @@ public class RankingExpressionLoopDetectionTestCase { @Test public void testSelfLoop() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -36,7 +36,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -48,8 +48,8 @@ public class RankingExpressionLoopDetectionTestCase { @Test public void testNestedLoop() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -71,7 +71,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -83,8 +83,8 @@ public class RankingExpressionLoopDetectionTestCase { @Test public void testSelfArgumentLoop() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -106,7 +106,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -118,8 +118,8 @@ public class RankingExpressionLoopDetectionTestCase { @Test public void testNoLoopWithSameLocalArgument() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -140,14 +140,14 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); } @Test public void testNoLoopWithMultipleInvocations() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -168,14 +168,14 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); } @Test public void testNoLoopWithBoundIdentifiers() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " }\n" + @@ -191,14 +191,14 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } @Test public void testNoLoopWithTheSameNestedIdentifierWhichIsUnbound() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " }\n" + @@ -214,14 +214,14 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } @Test public void testNoLoopWithTheSameAlternatingNestedIdentifierWhichIsUnbound() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " }\n" + @@ -240,7 +240,7 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index ff146fedd88..8354eb59416 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -26,8 +26,8 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { @Test public void testBasicFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -45,7 +45,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List<Pair<String, String>> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -61,8 +61,8 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { @Test public void testMultiLevelFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -86,7 +86,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List<Pair<String, String>> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -113,8 +113,8 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { @Test public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -132,7 +132,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List<Pair<String, String>> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -153,8 +153,8 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { // Note: the type assigned to query profile and constant tensors here is not the correct type RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(input[1])"); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry, queryProfiles); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + @@ -193,7 +193,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " file: ignored.json\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels()); List<Pair<String, String>> testRankProperties = createRawRankProfile(test, queryProfiles, s).configProperties(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index 299724ea39e..ee3b4ab25e0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -32,20 +32,20 @@ public class RankingExpressionValidationTestCase extends AbstractSchemaTestCase } private Schema importWithExpression(String expression, RankProfileRegistry registry) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(registry); - builder.importString("search test {" + - " document test { " + - " field a type string { " + - " indexing: index " + - " }" + - " }" + - " rank-profile default {" + - " first-phase {" + - " expression: " + expression + - " }" + - " }" + - "}"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(registry); + builder.addSchema("search test {" + + " document test { " + + " field a type string { " + + " indexing: index " + + " }" + + " }" + + " rank-profile default {" + + " first-phase {" + + " expression: " + expression + + " }" + + " }" + + "}"); + builder.build(true); return builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java index a6f76897c0b..a8aaee83938 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java @@ -15,7 +15,7 @@ public class ReservedWordsAsFieldNamesTestCase extends AbstractSchemaTestCase { @Test public void testIt() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/reserved_words_as_field_names.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/reserved_words_as_field_names.sd"); assertNotNull(schema.getDocument().getField("inline")); assertNotNull(schema.getDocument().getField("constants")); assertNotNull(schema.getDocument().getField("reference")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java index c2816336076..9c895b452d5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java @@ -18,6 +18,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Iterator; +import static com.google.common.collect.testing.Helpers.assertEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -36,9 +37,9 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { @SuppressWarnings("deprecation") public void testSimpleImporting() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder sb = new SchemaBuilder(rankProfileRegistry, new QueryProfileRegistry()); - sb.importFile("src/test/examples/simple.sd"); - sb.build(); + ApplicationBuilder sb = new ApplicationBuilder(rankProfileRegistry, new QueryProfileRegistry()); + sb.addSchemaFile("src/test/examples/simple.sd"); + sb.build(true); Schema schema = sb.getSchema(); assertEquals("simple", schema.getName()); assertTrue(schema.hasDocument()); @@ -53,14 +54,14 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { new MakeAliases(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); // First field - field=(SDField) document.getField("title"); - assertEquals(DataType.STRING,field.getDataType()); + field = (SDField) document.getField("title"); + assertEquals(DataType.STRING, field.getDataType()); assertEquals("{ input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", field.getIndexingScript().toString()); assertFalse(schema.getIndex("default").isPrefix()); assertTrue(schema.getIndex("title").isPrefix()); - Iterator<String> titleAliases= schema.getIndex("title").aliasIterator(); - assertEquals("aliaz",titleAliases.next()); - assertEquals("analias.totitle",titleAliases.next()); + Iterator<String> titleAliases = schema.getIndex("title").aliasIterator(); + assertEquals("aliaz", titleAliases.next()); + assertEquals("analias.totitle", titleAliases.next()); assertEquals("analias.todefault", schema.getIndex("default").aliasIterator().next()); assertEquals(RankType.IDENTITY, field.getRankType()); @@ -70,7 +71,7 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { assertTrue(field.isHeader()); // Second field - field=(SDField) document.getField("description"); + field = (SDField) document.getField("description"); assertEquals(RankType.ABOUT, field.getRankType()); assertEquals(SummaryTransform.NONE, field.getSummaryField("description").getTransform()); @@ -81,30 +82,30 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { assertEquals("hallo", schema.getIndex("description").aliasIterator().next()); // Third field - field=(SDField) document.getField("chatter"); + field = (SDField) document.getField("chatter"); assertEquals(RankType.ABOUT, field.getRankType()); assertNull(field.getStemming()); assertTrue(field.getNormalizing().doRemoveAccents()); // Fourth field - field=(SDField) document.getField("category"); + field = (SDField) document.getField("category"); assertEquals(0, field.getAttributes().size()); assertEquals(Stemming.NONE, field.getStemming()); assertFalse(field.getNormalizing().doRemoveAccents()); // Fifth field - field=(SDField) document.getField("popularity"); + field = (SDField) document.getField("popularity"); assertEquals("{ input popularity | attribute popularity; }", field.getIndexingScript().toString()); // Sixth field - field=(SDField) document.getField("measurement"); - assertEquals(DataType.INT,field.getDataType()); + field = (SDField) document.getField("measurement"); + assertEquals(DataType.INT, field.getDataType()); assertEquals(RankType.EMPTY, field.getRankType()); assertEquals(1, field.getAttributes().size()); // Seventh field - field= schema.getConcreteField("categories"); + field = schema.getConcreteField("categories"); assertEquals("{ input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", field.getIndexingScript().toString()); assertTrue(field.isHeader()); @@ -139,27 +140,27 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); // Rank Profiles - RankProfile profile=rankProfileRegistry.get(schema, "default"); + RankProfile profile = rankProfileRegistry.get(schema, "default"); assertNotNull(profile); - assertNull(profile.getInheritedName()); + assertEmpty(profile.inheritedNames()); assertNull(profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE)); assertEquals(RankType.EMPTY, profile.getRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); - profile=rankProfileRegistry.get(schema, "experimental"); + profile = rankProfileRegistry.get(schema, "experimental"); assertNotNull(profile); - assertEquals("default",profile.getInheritedName()); + assertEquals("default", profile.inheritedNames().get(0)); assertEquals(RankType.IDENTITY, profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); - profile=rankProfileRegistry.get(schema, "other"); + profile = rankProfileRegistry.get(schema, "other"); assertNotNull(profile); - assertEquals("experimental",profile.getInheritedName()); + assertEquals("experimental", profile.inheritedNames().get(0)); // The extra-document field - SDField exact= schema.getConcreteField("exact"); - assertNotNull("Extra field was parsed",exact); - assertEquals("exact",exact.getName()); - assertEquals(Stemming.NONE,exact.getStemming()); + SDField exact = schema.getConcreteField("exact"); + assertNotNull("Extra field was parsed", exact); + assertEquals("exact", exact.getName()); + assertEquals(Stemming.NONE, exact.getStemming()); assertFalse(exact.getNormalizing().doRemoveAccents()); assertEquals("{ input title . \" \" . input category | tokenize | summary exact | index exact; }", exact.getIndexingScript().toString()); @@ -170,7 +171,7 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { public void testDocumentImporting() throws IOException, ParseException { try { // Having two documents in one sd-file is illegal. - SchemaBuilder.buildFromFile("src/test/examples/documents.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/documents.sd"); fail(); } catch (IllegalArgumentException e) { } @@ -178,11 +179,11 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { @Test public void testIdImporting() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/strange.sd"); - SDField idecidemyide=(SDField) schema.getDocument().getField("idecidemyide"); - assertEquals(5,idecidemyide.getId()); - SDField sodoi=(SDField) schema.getDocument().getField("sodoi"); - assertEquals(7,sodoi.getId()); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/strange.sd"); + SDField idecidemyide = (SDField)schema.getDocument().getField("idecidemyide"); + assertEquals(5, idecidemyide.getId()); + SDField sodoi = (SDField) schema.getDocument().getField("sodoi"); + assertEquals(7, sodoi.getId()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java index dfe60c5871e..6d619b76c18 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java @@ -19,13 +19,13 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void requireThatIndexingExpressionsCanBeParsed() throws Exception { - assertNotNull(SchemaBuilder.buildFromFile("src/test/examples/simple.sd")); + assertNotNull(ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); } @Test public void requireThatParseExceptionPositionIsCorrect() throws Exception { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_construct.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_construct.sd"); } catch (ParseException e) { if ( ! e.getMessage().contains("at line 5, column 36.")) { throw e; @@ -36,7 +36,7 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void requireThatParserHandlesLexicalError() throws Exception { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_lexical_error.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_lexical_error.sd"); } catch (ParseException e) { if (!e.getMessage().contains("at line 7, column 27.")) { throw e; @@ -47,7 +47,7 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void requireErrorWhenJunkAfterSearchBlock() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_junk_at_end.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_junk_at_end.sd"); fail("Illegal junk at end of SD passed"); } catch (ParseException e) { if (!e.getMessage().contains("at line 10, column 1")) { @@ -59,7 +59,7 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void requireErrorWhenMissingClosingSearchBracket() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_no_closing_bracket.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_no_closing_bracket.sd"); fail("SD without closing bracket passed"); } catch (ParseException e) { if (!e.getMessage().contains("Encountered \"<EOF>\" at line 8, column 1")) { @@ -71,7 +71,7 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { @Test public void illegalSearchDefinitionName() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid-name.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid-name.sd"); fail("Name with dash passed"); } catch (ParseException e) { if ( ! e.getMessage().contains("invalid-name")) { @@ -80,10 +80,4 @@ public class SchemaParsingTestCase extends AbstractSchemaTestCase { } } - // TODO: Remove in Vespa 8 - @Test - public void requireThatParserHandlesHeadAndBody() throws IOException, ParseException { - assertNotNull(SchemaBuilder.buildFromFile("src/test/examples/header_body.sd")); - } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java index d906685d502..112dbd2587e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java @@ -31,10 +31,10 @@ public class SchemaTestCase { " }" + "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromStrings(logger, schema); + ApplicationBuilder.createFromStrings(logger, schema); assertEquals("schema 'test' inherits 'nonesuch', but this schema does not exist", logger.entries.get(0).message); - fail("Expected failure"); + fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("schema 'test' inherits 'nonesuch', but this schema does not exist", e.getMessage()); @@ -60,7 +60,8 @@ public class SchemaTestCase { " }" + " }" + "}"); - SchemaBuilder.createFromStrings(new DeployLoggerStub(), parent, child); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), parent, child); + fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("schema 'child' inherits 'parent', " + @@ -166,12 +167,12 @@ public class SchemaTestCase { " import field parentschema_ref.name as child2_imported {}" + "}"); - SchemaBuilder builder = new SchemaBuilder(new DeployLoggerStub()); + ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere - builder.importString(parentLines); - builder.importString(child1Lines); - builder.importString(child2Lines); + builder.addSchema(parentLines); + builder.addSchema(child1Lines); + builder.addSchema(child2Lines); builder.build(true); var application = builder.application(); @@ -187,7 +188,7 @@ public class SchemaTestCase { assertNotNull(child1.getExtraField("child1_field")); assertNotNull(builder.getRankProfileRegistry().get(child1, "parent_profile")); assertNotNull(builder.getRankProfileRegistry().get(child1, "child1_profile")); - assertEquals("parent_profile", builder.getRankProfileRegistry().get(child1, "child1_profile").getInheritedName()); + assertEquals("parent_profile", builder.getRankProfileRegistry().get(child1, "child1_profile").inheritedNames().get(0)); assertNotNull(child1.rankingConstants().get("parent_constant")); assertNotNull(child1.rankingConstants().get("child1_constant")); assertTrue(child1.rankingConstants().asMap().containsKey("parent_constant")); @@ -222,7 +223,7 @@ public class SchemaTestCase { assertNotNull(child2.getExtraField("child2_field")); assertNotNull(builder.getRankProfileRegistry().get(child2, "parent_profile")); assertNotNull(builder.getRankProfileRegistry().get(child2, "child2_profile")); - assertEquals("parent_profile", builder.getRankProfileRegistry().get(child2, "child2_profile").getInheritedName()); + assertEquals("parent_profile", builder.getRankProfileRegistry().get(child2, "child2_profile").inheritedNames().get(0)); assertNotNull(child2.rankingConstants().get("parent_constant")); assertNotNull(child2.rankingConstants().get("child2_constant")); assertTrue(child2.rankingConstants().asMap().containsKey("parent_constant")); @@ -307,12 +308,12 @@ public class SchemaTestCase { " }" + "}"); - SchemaBuilder builder = new SchemaBuilder(new DeployLoggerStub()); + ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere - builder.importString(parentLines); - builder.importString(childLines); - builder.importString(grandchildLines); + builder.addSchema(parentLines); + builder.addSchema(childLines); + builder.addSchema(grandchildLines); builder.build(true); var application = builder.application(); @@ -320,6 +321,98 @@ public class SchemaTestCase { assertInheritedFromParent(application.schemas().get("grandchild"), application, builder.getRankProfileRegistry()); } + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsDisallowed1() throws ParseException { + try { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " }" + + " rank-profile r1 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Only one of the profiles inherited by rank profile 'r3' can contain first-phase expression, but it is present in all of [rank profile 'r1', rank profile 'r2']", + e.getMessage()); + } + } + + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsAllowedWhenDefinedInChild() throws ParseException { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " }" + + " rank-profile r1 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " first-phase {" + // Redefined here so this does not cause failure + " expression: nativeRank" + + " }" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + } + + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsDisallowed2() throws ParseException { + try { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " }" + + " rank-profile r1 {" + + " function f1() {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " function f1() {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("rank profile 'r3' inherits rank profile 'r2' which contains function 'f1', but this function is already defined in another profile this inherits", + e.getMessage()); + } + } + private void assertInheritedFromParent(Schema schema, Application application, RankProfileRegistry rankProfileRegistry) { assertEquals("pf1", schema.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); assertEquals(Stemming.NONE, schema.getStemming()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java index 75dcd6bc209..aec258f2df0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java @@ -20,7 +20,7 @@ public class StemmingSettingTestCase extends AbstractSchemaTestCase { @Test public void testStemmingSettings() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/stemmingsetting.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/stemmingsetting.sd"); SDField artist = (SDField) schema.getDocument().getField("artist"); assertEquals(Stemming.SHORTEST, artist.getStemming(schema)); @@ -43,7 +43,7 @@ public class StemmingSettingTestCase extends AbstractSchemaTestCase { @Test public void requireThatStemmingIsDefaultBest() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/stemmingdefault.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/stemmingdefault.sd"); assertNull(schema.getConcreteField("my_str").getStemming()); assertEquals(Stemming.BEST, schema.getConcreteField("my_str").getStemming(schema)); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java index 34093bf72ef..c27dd9dfdfb 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java @@ -25,7 +25,7 @@ public class StructTestCase extends AbstractSchemaTestCase { @Test public void testBadStruct() throws IOException { try { - SchemaBuilder.buildFromFile("src/test/examples/badstruct.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/badstruct.sd"); fail("Should throw exception."); } catch (ParseException expected) { // success @@ -46,7 +46,7 @@ public class StructTestCase extends AbstractSchemaTestCase { */ @Test(expected = IllegalArgumentException.class) public void testStructOutsideDocumentIllegal() throws IOException, ParseException { - SchemaBuilder.buildFromFile("src/test/examples/structoutsideofdocument.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/structoutsideofdocument.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java index 7c8e806d768..086e3485962 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java @@ -39,7 +39,7 @@ public class SummaryTestCase { " }", "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); } @@ -61,7 +61,7 @@ public class SummaryTestCase { " }", "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); assertEquals(1, logger.entries.size()); assertEquals(Level.WARNING, logger.entries.get(0).level); assertEquals("summary field 'foo2' in document summary 'foobar' references source field 'ondisk', " + @@ -89,7 +89,7 @@ public class SummaryTestCase { " }", "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); } @@ -120,7 +120,7 @@ public class SummaryTestCase { " }", "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); } @@ -156,7 +156,7 @@ public class SummaryTestCase { " }", "}"); var logger = new DeployLoggerStub(); - var search = SchemaBuilder.createFromString(sd, logger).getSchema(); + var search = ApplicationBuilder.createFromString(sd, logger).getSchema(); assertEquals(List.of(), logger.entries); var titleField = "title"; @@ -208,7 +208,7 @@ public class SummaryTestCase { "}"); var logger = new DeployLoggerStub(); try { - SchemaBuilder.createFromString(sd, logger); + ApplicationBuilder.createFromString(sd, logger); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("For schema 'music', summary class 'title2', summary field 'title': Can not use " + @@ -228,7 +228,7 @@ public class SummaryTestCase { " }" + "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromStrings(logger, schema); + ApplicationBuilder.createFromStrings(logger, schema); assertEquals("document summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", logger.entries.get(0).message); // fail("Expected failure"); @@ -264,7 +264,7 @@ public class SummaryTestCase { " }" + "}"); DeployLoggerStub logger = new DeployLoggerStub(); - SchemaBuilder.createFromStrings(logger, parent, child); + ApplicationBuilder.createFromStrings(logger, parent, child); logger.entries.forEach(e -> System.out.println(e)); //assertTrue(logger.entries.isEmpty()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java index 9fdeb9ece1d..f2d78796553 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java @@ -15,14 +15,14 @@ public class UrlFieldValidationTestCase { @Test public void requireThatInheritedRiseFieldsStillCanBeInConflictButDontThrowException() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString("search test {" + - " document test { " + - " field a type uri { indexing: attribute | summary }" + - " }" + - "}"); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema("search test {" + + " document test { " + + " field a type uri { indexing: attribute | summary }" + + " }" + + "}"); try { - builder.build(); + builder.build(true); fail("Should have caused an exception"); // success } catch (IllegalArgumentException e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 8b54455d176..456efdb08ae 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -8,7 +8,7 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; @@ -38,14 +38,14 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { toDir.mkdirs(); deleteContent(toDir); - SchemaBuilder builder = SchemaBuilder.createFromDirectory(searchDefRoot + dirName + "/", new MockFileRegistry(), logger, properties); + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory(searchDefRoot + dirName + "/", new MockFileRegistry(), logger, properties); return derive(dirName, searchDefinitionName, properties, builder, logger); } private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties, - SchemaBuilder builder, + ApplicationBuilder builder, DeployLogger logger) throws IOException { DerivedConfiguration config = new DerivedConfiguration(builder.getSchema(searchDefinitionName), logger, @@ -57,14 +57,14 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { return export(dirName, builder, config); } - DerivedConfiguration derive(String dirName, SchemaBuilder builder, Schema schema) throws IOException { + DerivedConfiguration derive(String dirName, ApplicationBuilder builder, Schema schema) throws IOException { DerivedConfiguration config = new DerivedConfiguration(schema, builder.getRankProfileRegistry(), builder.getQueryProfileRegistry()); return export(dirName, builder, config); } - private DerivedConfiguration export(String name, SchemaBuilder builder, DerivedConfiguration config) throws IOException { + private DerivedConfiguration export(String name, ApplicationBuilder builder, DerivedConfiguration config) throws IOException { String path = exportConfig(name, config); DerivedConfiguration.exportDocuments(new DocumentManager().useV8DocManagerCfg(useV8DocManagerCfg()) .produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); @@ -100,6 +100,11 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { DeployLogger logger) throws IOException, ParseException { return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties(), logger); } + protected DerivedConfiguration assertCorrectDeriving(String dirName, + TestProperties properties) throws IOException, ParseException + { + return assertCorrectDeriving(dirName, null, properties, new TestableDeployLogger()); + } protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, @@ -114,14 +119,14 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { * Asserts config is correctly derived given a builder. * This will fail if the builder contains multiple search definitions. */ - protected DerivedConfiguration assertCorrectDeriving(SchemaBuilder builder, String dirName, DeployLogger logger) throws IOException { - builder.build(); + protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, String dirName, DeployLogger logger) throws IOException { + builder.build(true); DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder, logger); assertCorrectConfigFiles(dirName); return derived; } - protected DerivedConfiguration assertCorrectDeriving(SchemaBuilder builder, Schema schema, String name) throws IOException { + protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, Schema schema, String name) throws IOException { DerivedConfiguration derived = derive(name, builder, schema); assertCorrectConfigFiles(name); return derived; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 58a4350b73b..a272ab14dad 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.parser.ParseException; @@ -25,7 +25,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { // Test attribute importing - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/simple.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); // Test attribute deriving AttributeFields attributeFields = new AttributeFields(schema); @@ -71,7 +71,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void fields_in_array_of_struct_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(schema).attributeIterator(); assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); @@ -81,7 +81,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void fields_in_map_of_struct_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(schema).attributeIterator(); assertAttribute("str_elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); @@ -101,7 +101,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void only_zcurve_attribute_is_derived_from_array_of_position_field() throws ParseException { - Schema schema = SchemaBuilder.createFromString( + Schema schema = ApplicationBuilder.createFromString( joinLines("search test {", " document test {", " field pos_array type array<position> {", @@ -117,7 +117,7 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { @Test public void fields_in_map_of_primitive_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/map_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_attribute/test.sd"); Iterator<Attribute> attributes = new AttributeFields(schema).attributeIterator(); assertAttribute("str_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java index 06d72bb6972..ee9a9eac02c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -20,7 +20,7 @@ public class CasingTestCase extends AbstractSchemaTestCase { @Test public void testCasing() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/casing.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/casing.sd"); assertEquals(schema.getIndex("color").getName(), "color"); assertEquals(schema.getIndex("Foo").getName(), "Foo"); assertEquals(schema.getIndex("Price").getName(), "Price"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 604082fb52e..237e6c8c992 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -20,7 +21,7 @@ public class EmptyRankProfileTestCase extends AbstractSchemaTestCase { @Test public void testDeriving() { - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType doc = new SDDocumentType("test"); schema.addDocument(doc); @@ -30,7 +31,7 @@ public class EmptyRankProfileTestCase extends AbstractSchemaTestCase { doc.addField(field); doc.addField(new SDField("c", DataType.STRING)); - schema = SchemaBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(schema, rankProfileRegistry); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 8cc8967269a..05f8f1a6b93 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -24,27 +24,32 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testPositionArray() throws IOException, ParseException { - assertCorrectDeriving("position_array"); + assertCorrectDeriving("position_array", + new TestProperties().setUseV8GeoPositions(true)); } @Test public void testPositionAttribute() throws IOException, ParseException { - assertCorrectDeriving("position_attribute"); + assertCorrectDeriving("position_attribute", + new TestProperties().setUseV8GeoPositions(true)); } @Test public void testPositionExtra() throws IOException, ParseException { - assertCorrectDeriving("position_extra"); + assertCorrectDeriving("position_extra", + new TestProperties().setUseV8GeoPositions(true)); } @Test public void testPositionNoSummary() throws IOException, ParseException { - assertCorrectDeriving("position_nosummary"); + assertCorrectDeriving("position_nosummary", + new TestProperties().setUseV8GeoPositions(true)); } @Test public void testPositionSummary() throws IOException, ParseException { - assertCorrectDeriving("position_summary"); + assertCorrectDeriving("position_summary", + new TestProperties().setUseV8GeoPositions(true)); } @Test @@ -152,10 +157,10 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testTensor2() throws IOException, ParseException { String dir = "src/test/derived/tensor2/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "first.sd"); - builder.importFile(dir + "second.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "first.sd"); + builder.addSchemaFile(dir + "second.sd"); + builder.build(true); derive("tensor2", builder, builder.getSchema("second")); assertCorrectConfigFiles("tensor2"); } @@ -177,4 +182,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertEquals(0, logger.warnings.size()); } + @Test + public void testRankProfileModularity() throws IOException, ParseException { + assertCorrectDeriving("rankprofilemodularity"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index b262ef92e1a..440f067dd00 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; @@ -25,7 +26,7 @@ public class IdTestCase extends AbstractExportingTestCase { @Test public void testExplicitUpperCaseIdField() { - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); SDField uri = new SDField("URI", DataType.URI); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java index f00072a5a19..bcf68387294 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; @@ -40,10 +41,10 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void requireThatIndexedStructFieldCanBeInherited() throws IOException, ParseException { String dir = "src/test/derived/inheritstruct/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "parent.sd"); - builder.importFile(dir + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); derive("inheritstruct", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritstruct"); } @@ -52,7 +53,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { public void requireThatInheritFromNullIsCaught() throws IOException, ParseException { try { assertCorrectDeriving("inheritfromnull"); - } catch (IllegalStateException e) { + } catch (IllegalArgumentException e) { assertEquals("Document type 'foo' not found", e.getMessage()); } } @@ -63,12 +64,12 @@ public class InheritanceTestCase extends AbstractExportingTestCase { List<String> files = Arrays.asList("grandparent.sd", "mother.sd", "father.sd", "child.sd"); File outDir = tmpDir.newFolder("out"); for (int startIdx = 0; startIdx < files.size(); ++startIdx) { - SchemaBuilder builder = new SchemaBuilder(); + ApplicationBuilder builder = new ApplicationBuilder(); for (int fileIdx = startIdx; fileIdx < startIdx + files.size(); ++fileIdx) { String fileName = files.get(fileIdx % files.size()); - builder.importFile(dir + fileName); + builder.addSchemaFile(dir + fileName); } - builder.build(); + builder.build(true); DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), b), outDir.getPath()); DocumentmanagerConfig dc = b.build(); @@ -110,10 +111,10 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void requireThatStructTypesAreInheritedFromParent() throws IOException, ParseException { String dir = "src/test/derived/inheritfromparent/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "parent.sd"); - builder.importFile(dir + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); derive("inheritfromparent", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritfromparent"); } @@ -121,11 +122,11 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void requireThatStructTypesAreInheritedFromGrandParent() throws IOException, ParseException { String dir = "src/test/derived/inheritfromgrandparent/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "grandparent.sd"); - builder.importFile(dir + "parent.sd"); - builder.importFile(dir + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "grandparent.sd"); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); derive("inheritfromgrandparent", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritfromgrandparent"); } @@ -133,12 +134,12 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void testInheritance() throws IOException, ParseException { String dir = "src/test/derived/inheritance/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "grandparent.sd"); - builder.importFile(dir + "father.sd"); - builder.importFile(dir + "mother.sd"); - builder.importFile(dir + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "grandparent.sd"); + builder.addSchemaFile(dir + "father.sd"); + builder.addSchemaFile(dir + "mother.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); derive("inheritance", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritance"); } @@ -146,7 +147,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test public void testIndexSettingInheritance() { SDDocumentType parent = new SDDocumentType("parent"); - Schema parentSchema = new Schema("parent"); + Schema parentSchema = new Schema("parent", MockApplicationPackage.createEmpty()); parentSchema.addDocument(parent); SDField prefixed = parent.addField("prefixed", DataType.STRING); prefixed.parseIndexingScript("{ index }"); @@ -154,7 +155,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { SDDocumentType child = new SDDocumentType("child"); child.inherit(parent); - Schema childSchema = new Schema("child"); + Schema childSchema = new Schema("child", MockApplicationPackage.createEmpty()); childSchema.addDocument(child); prefixed = (SDField)child.getField("prefixed"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index f8f1bf9e4f1..1e7dd3a2405 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -2,12 +2,13 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processing; @@ -30,7 +31,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { */ @Test public void testLiteralBoost() { - Schema schema = new Schema("literalboost"); + Schema schema = new Schema("literalboost", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("literalboost"); schema.addDocument(document); @@ -63,7 +64,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { */ @Test public void testNonDefaultRankLiteralBoost() { - Schema schema = new Schema("literalboost"); + Schema schema = new Schema("literalboost", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("literalboost"); schema.addDocument(document); @@ -73,7 +74,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - schema = SchemaBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); // Check il script addition @@ -89,7 +90,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { /** Tests literal boosts in two fields going to the same index */ @Test public void testTwoLiteralBoostFields() { - Schema schema = new Schema("msb"); + Schema schema = new Schema("msb", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("msb"); schema.addDocument(document); @@ -100,7 +101,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { field2.parseIndexingScript("{ summary | index }"); field2.setLiteralBoost(20); - schema = SchemaBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(schema, rankProfileRegistry); assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", "clear_state | guard { input body | tokenize normalize stem:\"BEST\" | summary body | index body; }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java index 9ad6dfbb972..53bf55fc73f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; import java.io.IOException; @@ -16,8 +16,8 @@ public class MailTestCase extends AbstractExportingTestCase { @Test public void testMail() throws IOException, ParseException { String dir = "src/test/derived/mail/"; - SchemaBuilder sb = new SchemaBuilder(); - sb.importFile(dir + "mail.sd"); + ApplicationBuilder sb = new ApplicationBuilder(); + sb.addSchemaFile(dir + "mail.sd"); assertCorrectDeriving(sb, dir, new TestableDeployLogger()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java index 51135ae1cb2..0e6d7b8442f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -14,7 +13,6 @@ import java.io.IOException; */ public class MultipleSummariesTestCase extends AbstractExportingTestCase { @Test - @Ignore public void testMultipleSummaries() throws IOException, ParseException { assertCorrectDeriving("multiplesummaries"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java index c7297c41c62..a458036a03f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -19,10 +19,10 @@ public class SchemaInheritanceTestCase extends AbstractExportingTestCase { @Test public void testIt() throws IOException, ParseException { try { - SchemaBuilder builder = SchemaBuilder.createFromDirectory("src/test/derived/schemainheritance/", - new MockFileRegistry(), - new TestableDeployLogger(), - new TestProperties()); + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory("src/test/derived/schemainheritance/", + new MockFileRegistry(), + new TestableDeployLogger(), + new TestProperties()); derive("schemainheritance", builder, builder.getSchema("child")); assertCorrectConfigFiles("schemainheritance"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java index b3f2fb62ac2..34d33a00d9e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.DocumentReference; @@ -62,7 +63,7 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { } private static Schema createSchema(String name, Map<String, Schema> schemas) { - Schema schema = new Schema(name); + Schema schema = new Schema(name, MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType(name); document.setDocumentReferences(new DocumentReferences(emptyMap())); schema.addDocument(document); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java index c0f2b6887d2..d89e5f2c957 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -19,10 +19,10 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase { String name = "emptychild"; final String expectedResultsDirName = "src/test/derived/" + name + "/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(expectedResultsDirName + "parent.sd"); - builder.importFile(expectedResultsDirName + "child.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(expectedResultsDirName + "parent.sd"); + builder.addSchemaFile(expectedResultsDirName + "child.sd"); + builder.build(true); Schema schema = builder.getSchema("child"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java new file mode 100644 index 00000000000..e6c7efd7052 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SliceTestCase extends AbstractExportingTestCase { + + @Test + public void testSlice() throws IOException, ParseException { + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("slice"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java index b2c2a54ce5e..1f6c70c9383 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Rule; @@ -31,8 +31,8 @@ public class StructInheritanceTestCase extends AbstractExportingTestCase { @Test public void requireThatStructCanInherit() throws IOException, ParseException { String dir = "src/test/derived/structinheritance/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "simple.sd"); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "simple.sd"); builder.build(false); derive("structinheritance", builder, builder.getSchema("simple")); assertCorrectConfigFiles("structinheritance"); @@ -43,9 +43,9 @@ public class StructInheritanceTestCase extends AbstractExportingTestCase { exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage("cannot inherit from base and redeclare field name"); String dir = "src/test/derived/structinheritance/"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(dir + "bad.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "bad.sd"); + builder.build(true); derive("structinheritance", builder, builder.getSchema("bad")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index 4994cffb92a..c0fa3e8311f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.searchdefinition.*; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -29,7 +30,7 @@ import static org.junit.Assert.assertTrue; public class SummaryMapTestCase extends AbstractSchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/simple.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); SummaryMap summaryMap=new SummaryMap(schema); Iterator transforms=summaryMap.resultTransformIterator(); @@ -73,7 +74,7 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { } @Test public void testPositionDeriving() { - Schema schema = new Schema("store"); + Schema schema = new Schema("store", MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType("store"); schema.addDocument(document); String fieldName = "location"; @@ -147,7 +148,7 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { @Test public void testFailOnSummaryFieldSourceCollision() { try { - SchemaBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); } catch (Exception e) { assertTrue(e.getMessage().matches(".*equally named field.*")); } @@ -189,13 +190,13 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { } private Schema buildSearch(String field) throws ParseException { - var builder = new SchemaBuilder(new RankProfileRegistry()); - builder.importString(joinLines("search test {", - " document test {", - field, - " }", - "}")); - builder.build(); + var builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines("search test {", + " document test {", + field, + " }", + "}")); + builder.build(true); return builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index c996fb0c1b9..9a36ef90cd7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.search.SummaryConfig; @@ -35,7 +35,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " }", " }", "}"); - Schema schema = SchemaBuilder.createFromString(sd).getSchema(); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); assertEquals(SummaryClassField.Type.RAW, summary.getField("raw_field").getType()); } @@ -50,14 +50,14 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " }", " }", "}"); - Schema schema = SchemaBuilder.createFromString(sd).getSchema(); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); assertEquals(SummaryClassField.Type.DATA, summary.getField("raw_field").getType()); } @Test public void testDeriving() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/simple.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); assertEquals("default", summary.getName()); @@ -134,22 +134,22 @@ public class SummaryTestCase extends AbstractSchemaTestCase { } private static Schema buildCampaignAdModel() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString("search campaign { document campaign {} }"); - builder.importString(joinLines("search ad {", - " document ad {", - " field campaign_ref type reference<campaign> {", - " indexing: summary | attribute", - " }", - " field other_campaign_ref type reference<campaign> {", - " indexing: summary | attribute", - " }", - " }", - " document-summary my_summary {", - " summary other_campaign_ref type reference<campaign> {}", - " }", - "}")); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema("search campaign { document campaign {} }"); + builder.addSchema(joinLines("search ad {", + " document ad {", + " field campaign_ref type reference<campaign> {", + " indexing: summary | attribute", + " }", + " field other_campaign_ref type reference<campaign> {", + " indexing: summary | attribute", + " }", + " }", + " document-summary my_summary {", + " summary other_campaign_ref type reference<campaign> {}", + " }", + "}")); + builder.build(true); return builder.getSchema("ad"); } @@ -168,7 +168,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " summary foo type string {}", " }", "}"); - var search = SchemaBuilder.createFromString(sd).getSchema(); + var search = ApplicationBuilder.createFromString(sd).getSchema(); assertOmitSummaryFeatures(true, search, "bar"); assertOmitSummaryFeatures(false, search, "baz"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java index c3a78bab441..4e629fcb4d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -13,20 +13,22 @@ import java.io.IOException; * @author arnej27959 */ public class TwoStreamingStructsTestCase extends AbstractExportingTestCase { + @Test public void testTwoStreamingStructsExporting() throws ParseException, IOException { String root = "src/test/derived/twostreamingstructs"; - SchemaBuilder builder = new SchemaBuilder(); - builder.importFile(root + "/streamingstruct.sd"); - builder.importFile(root + "/whatever.sd"); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(root + "/streamingstruct.sd"); + builder.addSchemaFile(root + "/whatever.sd"); + builder.build(true); assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); - builder = new SchemaBuilder(); - builder.importFile(root + "/streamingstruct.sd"); - builder.importFile(root + "/whatever.sd"); - builder.build(); + builder = new ApplicationBuilder(); + builder.addSchemaFile(root + "/streamingstruct.sd"); + builder.addSchemaFile(root + "/whatever.sd"); + builder.build(true); assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 13289d72884..dbb32e61144 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; @@ -25,7 +26,7 @@ public class TypeConversionTestCase extends AbstractSchemaTestCase { /** Tests that exact-string stuff is not spilled over to the default index */ @Test public void testExactStringToStringTypeConversion() { - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java index 138992477c0..5ab5a8057e8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java @@ -24,7 +24,7 @@ public class VsmFieldsTestCase { @SuppressWarnings("deprecation") @Test public void reference_type_field_is_unsearchable() { - Schema schema = new Schema("test", new Application(MockApplicationPackage.createEmpty()), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); schema.addDocument(new SDDocumentType("test")); SDField refField = new TemporarySDField("ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type"))); refField.parseIndexingScript("{ summary }"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java index 481a4db11ec..99692e70041 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.document; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -17,7 +17,7 @@ public class ComplexAttributeFieldUtilsTestCase { private final ImmutableSDField field; FixtureBase(String fieldName, String sdContent) throws ParseException { - Schema schema = SchemaBuilder.createFromString(sdContent).getSchema(); + Schema schema = ApplicationBuilder.createFromString(sdContent).getSchema(); field = schema.getConcreteField(fieldName); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index 715b7b803e2..9ca97f4dbc7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -48,7 +48,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { } private static Schema createSearch(String documentType) { - return new Schema(documentType, new Application(MockApplicationPackage.createEmpty()), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + return new Schema(documentType, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); } private static Schema createSearchWithDocument(String documentName) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java index 6a736a5331e..534c82157e4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import java.io.IOException; @@ -14,13 +14,13 @@ import static org.junit.Assert.*; public abstract class AssertSearchBuilder { public static void assertBuilds(String searchDefinitionFileName) throws IOException, ParseException { - assertNotNull(SchemaBuilder.buildFromFile(searchDefinitionFileName)); + assertNotNull(ApplicationBuilder.buildFromFile(searchDefinitionFileName)); } public static void assertBuildFails(String searchDefinitionFileName, String expectedException) throws IOException, ParseException { try { - SchemaBuilder.buildFromFile(searchDefinitionFileName); + ApplicationBuilder.buildFromFile(searchDefinitionFileName); fail(searchDefinitionFileName); } catch (IllegalArgumentException e) { assertEquals(expectedException, e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java index f1aa64c1a60..67b369fd951 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.parser.ParseException; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse; public class AttributesExactMatchTestCase extends AbstractSchemaTestCase { @Test public void testAttributesExactMatch() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); assertEquals(schema.getConcreteField("color").getMatching().getType(), Matching.Type.EXACT); assertEquals(schema.getConcreteField("artist").getMatching().getType(), Matching.Type.WORD); assertEquals(schema.getConcreteField("drummer").getMatching().getType(), Matching.Type.WORD); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index f74e46d92dc..efec9206ed9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -31,7 +31,7 @@ public class BoldingTestCase extends AbstractSchemaTestCase { @Test public void testBoldOnNonString() throws ParseException { try { - SchemaBuilder.createFromString(boldonnonstring); + ApplicationBuilder.createFromString(boldonnonstring); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("'bolding: on' for non-text field 'year4' (datatype int (code: 0)) is not allowed", @@ -52,7 +52,7 @@ public class BoldingTestCase extends AbstractSchemaTestCase { @Test public void testBoldOnArray() throws ParseException { try { - SchemaBuilder.createFromString(boldonarray); + ApplicationBuilder.createFromString(boldonarray); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals("'bolding: on' for non-text field 'myarray' (datatype Array<string> (code: -1486737430)) is not allowed", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java index 342ff4ba2d4..db27bbbb84d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; -import static com.yahoo.searchdefinition.SchemaBuilder.createFromString; +import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java index 6f8ab24de8e..a17992fad18 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Dictionary; @@ -30,7 +30,7 @@ public class DictionaryTestCase { return builder.build(); } private Schema createSearch(String def) throws ParseException { - SchemaBuilder sb = SchemaBuilder.createFromString(def); + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); return sb.getSchema(); } @Test @@ -196,7 +196,7 @@ public class DictionaryTestCase { " }", "}"); try { - SchemaBuilder sb = SchemaBuilder.createFromString(def); + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); fail("Controlling dictionary for non-numeric fields are not yet supported."); } catch (IllegalArgumentException e) { assertEquals("For schema 'test', field 'n1': You can only specify 'dictionary:' for numeric or string fields", e.getMessage()); @@ -214,7 +214,7 @@ public class DictionaryTestCase { " }", "}"); try { - SchemaBuilder sb = SchemaBuilder.createFromString(def); + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); fail("Controlling dictionary for non-fast-search fields are not allowed."); } catch (IllegalArgumentException e) { assertEquals("For schema 'test', field 'n1': You must specify 'attribute:fast-search' to allow dictionary control", e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java index 03125c48d1d..b03aff455c5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -43,15 +43,15 @@ public class DisallowComplexMapAndWsetKeyTypesTestCase { private void testFieldType(String fieldType) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " struct mystruct {}\n" + " field a type " + fieldType + " {}\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java index dd958fb34cd..0c25cef49a1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Rule; import org.junit.Test; @@ -20,15 +20,15 @@ public class FastAccessValidatorTest { @Test public void throws_exception_on_incompatible_use_of_fastaccess() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(new RankProfileRegistry()); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema( TestUtil.joinLines( "schema parent {", " document parent {", " field int_field type int { indexing: attribute }", " }", "}")); - builder.importString( + builder.addSchema( TestUtil.joinLines( "schema test {", " document test { ", @@ -55,7 +55,7 @@ public class FastAccessValidatorTest { "For schema 'test': The following attributes have a type that is incompatible " + "with fast-access: predicate_attribute, tensor_attribute, reference_attribute. " + "Predicate, tensor and reference attributes are incompatible with fast-access."); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java index 43077fadfcd..833a6effe4a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -18,7 +18,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/extrafield.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/extrafield.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -30,7 +30,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatSummaryFieldsAreIncluded() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/summaryfield.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/summaryfield.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -38,12 +38,14 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { assertNotNull(docType.getField("foo")); assertNotNull(docType.getField("bar")); assertNotNull(docType.getField("cox")); - assertEquals(3, docType.getFieldCount()); + assertNotNull(docType.getField("mytags")); + assertNotNull(docType.getField("alltags")); + assertEquals(5, docType.getFieldCount()); } @Test public void testRequireThatBoldedSummaryFieldsAreIncluded() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/boldedsummaryfields.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/boldedsummaryfields.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -57,7 +59,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatUntransformedSummaryFieldsAreIgnored() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/untransformedsummaryfields.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/untransformedsummaryfields.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -70,7 +72,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatDynamicSummaryFieldsAreIgnored() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/dynamicsummaryfields.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/dynamicsummaryfields.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -82,9 +84,9 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatDerivedConfigurationWorks() throws IOException, ParseException { - SchemaBuilder sb = new SchemaBuilder(); - sb.importFile("src/test/examples/nextgen/simple.sd"); - sb.build(); + ApplicationBuilder sb = new ApplicationBuilder(); + sb.addSchemaFile("src/test/examples/nextgen/simple.sd"); + sb.build(true); assertNotNull(sb.getSchema()); new DerivedConfiguration(sb.getSchema(), sb.getRankProfileRegistry()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java index bd645b68728..443eff07f41 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.document.*; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -16,7 +16,7 @@ import static org.junit.Assert.*; public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); @@ -25,7 +25,7 @@ public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { } @Test public void testRequireThatImplicitStructsAreUsed() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/nextgen/implicitstructtypes.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/implicitstructtypes.sd"); assertNotNull(schema); SDDocumentType docType = schema.getDocument(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java index 9c1e5d0d8d7..cc2bc7d7bf6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.Test; @@ -29,7 +29,7 @@ public class ImplicitSummariesTestCase { LogHandler log = new LogHandler(); Logger.getLogger("").addHandler(log); - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/implicitsummaries_attribute.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaries_attribute.sd"); assertNotNull(schema); assertTrue(log.records.isEmpty()); } @@ -60,19 +60,19 @@ public class ImplicitSummariesTestCase { @Test public void attribute_combiner_transform_is_set_on_array_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array").getTransform()); } @Test public void attribute_combiner_transform_is_set_on_map_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("str_elem_map").getTransform()); } @Test public void attribute_combiner_transform_is_not_set_when_map_of_struct_has_some_struct_field_attributes() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); assertEquals(SummaryTransform.NONE, schema.getSummaryField("int_elem_map").getTransform()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java index 44b1d9387f1..175b8d6fe1e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.DocumentSummary; @@ -17,7 +17,7 @@ public class ImplicitSummaryFieldsTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/implicitsummaryfields.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaryfields.sd"); assertNotNull(schema); DocumentSummary docsum = schema.getSummary("default"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java index 6eab1dddc79..b15b81b717d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.document.ImportedComplexField; import com.yahoo.searchdefinition.document.ImportedField; @@ -68,21 +68,21 @@ public class ImportedFieldsTestCase { } private static Schema buildAdSearch(String sdContent) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "schema campaign {", " document campaign {", " field budget type int { indexing: attribute }", " }", "}")); - builder.importString(joinLines( + builder.addSchema(joinLines( "schema person {", " document person {", " field name type string { indexing: attribute }", " }", "}")); - builder.importString(sdContent); - builder.build(); + builder.addSchema(sdContent); + builder.build(true); return builder.getSchema("ad"); } @@ -312,19 +312,19 @@ public class ImportedFieldsTestCase { } private static Schema buildChildSearch(String parentSdContent, String sdContent) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(parentSdContent); - builder.importString(sdContent); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(parentSdContent); + builder.addSchema(sdContent); + builder.build(true); return builder.getSchema("child"); } private static Schema buildChildSearch(String grandParentSdContent, String parentSdContent, String sdContent) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(grandParentSdContent); - builder.importString(parentSdContent); - builder.importString(sdContent); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(grandParentSdContent); + builder.addSchema(parentSdContent); + builder.addSchema(sdContent); + builder.build(true); return builder.getSchema("child"); } @@ -488,41 +488,41 @@ public class ImportedFieldsTestCase { assertTrue(attrs.containsAttribute("entries.value")); } - private SchemaBuilder buildParentsUsingInheritance() throws ParseException { - var builder = new SchemaBuilder(); - builder.importString(joinLines("schema parent_a {", - "document parent_a {", - " struct Entry {", - " field key type string {}", - " field value type string {}", - " }", - " field entries type array<Entry> {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - " }", - "}", - "}")); - - builder.importString(joinLines("schema parent_b {", - "document parent_b inherits parent_a {", - "}", - "}")); - - builder.importString(joinLines("schema child {", - "document child {", - " field ref_parent_a type reference<parent_a> {", - " indexing: attribute", - " }", - " field ref_parent_b type reference<parent_b> {", - " indexing: attribute", - " }", - "}", - "import field ref_parent_a.entries as entries_from_a {}", - "import field ref_parent_b.entries as entries_from_b {}", - "}")); - - builder.build(); + private ApplicationBuilder buildParentsUsingInheritance() throws ParseException { + var builder = new ApplicationBuilder(); + builder.addSchema(joinLines("schema parent_a {", + "document parent_a {", + " struct Entry {", + " field key type string {}", + " field value type string {}", + " }", + " field entries type array<Entry> {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + " }", + "}", + "}")); + + builder.addSchema(joinLines("schema parent_b {", + "document parent_b inherits parent_a {", + "}", + "}")); + + builder.addSchema(joinLines("schema child {", + "document child {", + " field ref_parent_a type reference<parent_a> {", + " indexing: attribute", + " }", + " field ref_parent_b type reference<parent_b> {", + " indexing: attribute", + " }", + "}", + "import field ref_parent_a.entries as entries_from_a {}", + "import field ref_parent_b.entries as entries_from_b {}", + "}")); + + builder.build(true); return builder; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index 3105e3c7efd..9e4ba1c6728 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -2,11 +2,12 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -120,7 +121,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { "clear_state | guard { input smallattribute | attribute smallattribute; }", "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", "clear_state | guard { input title . \" \" . input category | tokenize | summary exact | index exact; }"), - SchemaBuilder.buildFromFile("src/test/examples/simple.sd")); + ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); } @Test @@ -129,7 +130,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { Arrays.asList("clear_state | guard { input title_src | lowercase | normalize | " + " tokenize | index title; }", "clear_state | guard { input title_src | summary title_s; }"), - SchemaBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); + ApplicationBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); } @Test @@ -154,7 +155,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { private static ScriptExpression processField(SDField unprocessedField) { SDDocumentType sdoc = new SDDocumentType("test"); sdoc.addField(unprocessedField); - Schema schema = new Schema("test"); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); schema.addDocument(sdoc); new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false, Set.of()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java index 5b5c5cedc0d..74a8cdfdb6a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.derived.AbstractExportingTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -64,7 +64,7 @@ public class IndexingValidationTestCase extends AbstractExportingTestCase { assertIndexing( Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"BEST\" | index my_index | summary my_index }", "clear_state | guard { input my_input | tokenize normalize stem:\"BEST\" | index my_extra | summary my_extra }"), - SchemaBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); + ApplicationBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index 4b13590c777..2465839b72b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; @@ -24,7 +24,7 @@ public class IntegerIndex2AttributeTestCase extends AbstractSchemaTestCase { @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/integerindex2attribute.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/integerindex2attribute.sd"); new IntegerIndex2Attribute(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); SDField f; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java index 7eea2fca0e6..b48927d58a3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -175,18 +175,18 @@ public class MatchedElementsOnlyResolverTestCase { } private Schema buildSearch(String field, String summary) throws ParseException { - var builder = new SchemaBuilder(new RankProfileRegistry()); - builder.importString(joinLines("search test {", - " document test {", - " struct elem {", - " field name type string {}", - " field weight type int {}", - " }", - field, - " }", - summary, - "}")); - builder.build(); + var builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + field, + " }", + summary, + "}")); + builder.build(true); return builder.getSchema(); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java index 1adb909ff21..7169a97f3fc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDField; @@ -25,7 +25,7 @@ public class NGramTestCase extends AbstractSchemaTestCase { @Test public void testNGram() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/ngram.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/ngram.sd"); assertNotNull(schema); SDField gram1 = schema.getConcreteField("gram_1"); @@ -55,7 +55,7 @@ public class NGramTestCase extends AbstractSchemaTestCase { @Test public void testInvalidNGramSetting1() throws IOException, ParseException { try { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/invalidngram1.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram1.sd"); fail("Should cause an exception"); } catch (IllegalArgumentException e) { @@ -66,7 +66,7 @@ public class NGramTestCase extends AbstractSchemaTestCase { @Test public void testInvalidNGramSetting2() throws IOException, ParseException { try { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/invalidngram2.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram2.sd"); fail("Should cause an exception"); } catch (IllegalArgumentException e) { @@ -77,7 +77,7 @@ public class NGramTestCase extends AbstractSchemaTestCase { @Test public void testInvalidNGramSetting3() throws IOException, ParseException { try { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/invalidngram3.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram3.sd"); fail("Should cause an exception"); } catch (IllegalArgumentException e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java index 5fce16f381a..4eeab15fdd2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java @@ -1,19 +1,70 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; +import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; +import java.util.Optional; + import static com.yahoo.config.model.test.TestUtil.joinLines; -import static com.yahoo.searchdefinition.SchemaBuilder.createFromString; +import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; +import static com.yahoo.searchdefinition.ApplicationBuilder.createFromStrings; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class PagedAttributeValidatorTestCase { @Test - public void dense_tensor_attribute_does_support_paged_setting() throws ParseException { - createFromString(getSd("tensor(x[2],y[2])")); + public void dense_tensor_attribute_supports_paged_setting() throws ParseException { + assertPagedSupported("tensor(x[2],y[2])"); + } + + @Test + public void primitive_attribute_types_support_paged_setting() throws ParseException { + assertPagedSupported("int"); + assertPagedSupported("array<int>"); + assertPagedSupported("weightedset<int>"); + + assertPagedSupported("string"); + assertPagedSupported("array<string>"); + assertPagedSupported("weightedset<string>"); + } + + @Test + public void struct_field_attributes_support_paged_setting() throws ParseException { + var sd = joinLines("schema test {", + " document test {", + " struct elem {", + " field first type int {}", + " field second type string {}", + " }", + " field foo type array<elem> {", + " indexing: summary", + " struct-field first {", + " indexing: attribute", + " attribute: paged", + " }", + " struct-field second {", + " indexing: attribute", + " attribute: paged", + " }", + " }", + " }", + "}"); + + var appBuilder = createFromString(sd); + var field = appBuilder.getSchema().getField("foo"); + assertTrue(field.getStructField("first").getAttribute().isPaged()); + assertTrue(field.getStructField("second").getAttribute().isPaged()); + } + + private void assertPagedSupported(String fieldType) throws ParseException { + var appBuilder = createFromString(getSd(fieldType)); + var attribute = appBuilder.getSchema().getAttribute("foo"); + assertTrue(attribute.isPaged()); } @Test @@ -22,25 +73,42 @@ public class PagedAttributeValidatorTestCase { } @Test - public void non_tensor_attribute_does_not_support_paged_setting() throws ParseException { - assertPagedSettingNotSupported("string"); + public void predicate_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("predicate"); + } + + @Test + public void reference_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("reference<parent>", Optional.of(getSd("parent", "int"))); } private void assertPagedSettingNotSupported(String fieldType) throws ParseException { + assertPagedSettingNotSupported(fieldType, Optional.empty()); + } + + private void assertPagedSettingNotSupported(String fieldType, Optional<String> parentSd) throws ParseException { try { - createFromString(getSd(fieldType)); + if (parentSd.isPresent()) { + createFromStrings(new BaseDeployLogger(), parentSd.get(), getSd(fieldType)); + } else { + createFromString(getSd(fieldType)); + } fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is only supported for dense tensor types", + assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types", e.getMessage()); } } - private String getSd(String type) { + private String getSd(String fieldType) { + return getSd("test", fieldType); + } + + private String getSd(String docType, String fieldType) { return joinLines( - "schema test {", - " document test {", - " field foo type " + type + "{", + "schema " + docType + " {", + " document " + docType + " {", + " field foo type " + fieldType + "{", " indexing: attribute", " attribute: paged", " }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java index 74fa7c72554..b14c7287537 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java @@ -23,7 +23,6 @@ import com.yahoo.searchdefinition.document.TemporarySDField; */ public class ParentChildSearchModel { - private final Application application = new Application(MockApplicationPackage.createEmpty()); public Schema parentSchema; public Schema childSchema; @@ -33,7 +32,7 @@ public class ParentChildSearchModel { } protected Schema createSearch(String name) { - Schema result = new Schema(name, application, new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + Schema result = new Schema(name, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); result.addDocument(new SDDocumentType(name)); return result; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java index c3beeeaa17e..eaf38396efc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.vespa.documentmodel.SummaryField; @@ -27,7 +27,7 @@ public class PositionTestCase { @Test public void inherited_position_zcurve_field_is_not_added_to_document_fieldset() throws Exception { - SchemaBuilder sb = SchemaBuilder.createFromFiles(Arrays.asList( + ApplicationBuilder sb = ApplicationBuilder.createFromFiles(Arrays.asList( "src/test/examples/position_base.sd", "src/test/examples/position_inherited.sd")); @@ -38,7 +38,7 @@ public class PositionTestCase { @Test public void requireThatPositionCanBeAttribute() throws Exception { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/position_attribute.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_attribute.sd"); assertNull(schema.getAttribute("pos")); assertNull(schema.getAttribute("pos.x")); assertNull(schema.getAttribute("pos.y")); @@ -50,7 +50,7 @@ public class PositionTestCase { @Test public void requireThatPositionCanNotBeIndex() throws Exception { try { - SchemaBuilder.buildFromFile("src/test/examples/position_index.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/position_index.sd"); fail(); } catch (IllegalArgumentException e) { assertEquals("For schema 'position_index', field 'pos': Indexing of data type 'position' is not " + @@ -60,7 +60,7 @@ public class PositionTestCase { @Test public void requireThatSummaryAloneDoesNotCreateZCurve() throws Exception { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/position_summary.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_summary.sd"); assertNull(schema.getAttribute("pos")); assertNull(schema.getAttribute("pos.x")); assertNull(schema.getAttribute("pos.y")); @@ -79,7 +79,7 @@ public class PositionTestCase { @Test public void requireThatExtraFieldCanBePositionAttribute() throws Exception { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/position_extra.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_extra.sd"); assertNull(schema.getAttribute("pos_ext")); assertNull(schema.getAttribute("pos_ext.x")); assertNull(schema.getAttribute("pos_ext.y")); @@ -90,7 +90,7 @@ public class PositionTestCase { @Test public void requireThatPositionArrayIsSupported() throws Exception { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/position_array.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_array.sd"); assertNull(schema.getAttribute("pos")); assertNull(schema.getAttribute("pos.x")); assertNull(schema.getAttribute("pos.y")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java index 87dd92f41d9..3a0ceebcb0a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -17,6 +17,6 @@ import java.io.IOException; public class RankModifierTestCase extends AbstractSchemaTestCase { @Test public void testLiteral() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 2707f60f828..2d0bdb58122 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -13,7 +13,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; @@ -68,7 +68,7 @@ class RankProfileSearchFixture { String rankProfiles, String constant, String field) throws ParseException { this.queryProfileRegistry = queryProfileRegistry; - SchemaBuilder builder = new SchemaBuilder(applicationpackage, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(applicationpackage, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); String sdContent = "search test {\n" + " " + (constant != null ? constant : "") + "\n" + " document test {\n" + @@ -77,8 +77,8 @@ class RankProfileSearchFixture { rankProfiles + "\n" + "}"; - builder.importString(sdContent); - builder.build(); + builder.addSchema(sdContent); + builder.build(true); schema = builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java index b1624b7fd7e..0ab7406b5b1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile.RankProperty; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -21,10 +21,10 @@ public class RankPropertyVariablesTestCase extends AbstractSchemaTestCase { @Test public void testRankPropVariables() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/rankpropvars.sd", - new BaseDeployLogger(), - rankProfileRegistry, - new QueryProfileRegistry()); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankpropvars.sd", + new BaseDeployLogger(), + rankProfileRegistry, + new QueryProfileRegistry()); assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar1", "foo"); assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar_2", "bar"); assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvarOne23", "baz"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index 4f18c9b68fd..7f5e8f8753a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -7,7 +7,7 @@ import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.TensorFieldType; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.TensorType; import com.yahoo.yolean.Exceptions; @@ -33,8 +33,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void tensorFirstPhaseMustProduceDouble() throws Exception { try { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[3]) {", @@ -48,7 +48,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -61,8 +61,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void tensorFirstPhaseFromConstantMustProduceDouble() throws Exception { try { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(d0[3]) {", @@ -96,7 +96,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -110,8 +110,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void tensorSecondPhaseMustProduceDouble() throws Exception { try { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[3]) {", @@ -128,7 +128,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -140,8 +140,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void tensorConditionsMustHaveTypeCompatibleBranches() throws Exception { try { - SchemaBuilder schemaBuilder = new SchemaBuilder(); - schemaBuilder.importString(joinLines( + ApplicationBuilder schemaBuilder = new ApplicationBuilder(); + schemaBuilder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[5]) {", @@ -158,7 +158,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -172,8 +172,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void testFunctionInvocationTypes() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[3]) {", @@ -194,7 +194,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[3])"), @@ -205,8 +205,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void testTensorFunctionInvocationTypes_Nested() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[1]) {", @@ -236,7 +236,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), @@ -247,8 +247,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void testAttributeInvocationViaBoundIdentifier() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search newsarticle {", " document newsarticle {", " field title type string {", @@ -280,14 +280,14 @@ public class RankingExpressionTypeResolverTestCase { " first-phase { expression: commonfirstphase(eustaticrank) }", " }", "}")); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "eurank"); } @Test public void testTensorFunctionInvocationTypes_NestedSameName() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test { ", " field a type tensor(x[10],y[1]) {", @@ -320,7 +320,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), @@ -331,8 +331,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void testTensorFunctionInvocationTypes_viaFuncWithExpr() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search test {", " document test {", " field t1 type tensor<float>(y{}) { indexing: attribute | summary }", @@ -345,7 +345,7 @@ public class RankingExpressionTypeResolverTestCase { " summary-features { test_func_via_func_with_expr }", " }", "}")); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "test"); assertEquals(TensorType.fromSpec("tensor<float>(y{})"), summaryFeatures(profile).get("test_func_via_func_with_expr").type(profile.typeContext(builder.getQueryProfileRegistry()))); @@ -353,8 +353,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void importedFieldsAreAvailable() throws Exception { - SchemaBuilder builder = new SchemaBuilder(); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( "search parent {", " document parent {", " field a type tensor(x[5],y[1000]) {", @@ -363,7 +363,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.importString(joinLines( + builder.addSchema(joinLines( "search child {", " document child { ", " field ref type reference<parent> {", @@ -378,14 +378,14 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); } @Test public void undeclaredQueryFeaturesAreAccepted() throws Exception { InspectableDeployLogger logger = new InspectableDeployLogger(); - SchemaBuilder builder = new SchemaBuilder(logger); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( "search test {", " document test { ", " field anyfield type double {" + @@ -410,8 +410,8 @@ public class RankingExpressionTypeResolverTestCase { @Test public void undeclaredQueryFeaturesAreAcceptedWithWarningWhenUsingTensors() throws Exception { InspectableDeployLogger logger = new InspectableDeployLogger(); - SchemaBuilder builder = new SchemaBuilder(logger); - builder.importString(joinLines( + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( "search test {", " document test { ", " field anyfield type tensor(d[2]) {", @@ -439,7 +439,7 @@ public class RankingExpressionTypeResolverTestCase { @Test public void noWarningWhenUsingTensorsWhenQueryFeaturesAreDeclared() throws Exception { InspectableDeployLogger logger = new InspectableDeployLogger(); - SchemaBuilder builder = new SchemaBuilder(logger); + ApplicationBuilder builder = new ApplicationBuilder(logger); QueryProfileType myType = new QueryProfileType("mytype"); myType.addField(new FieldDescription("rank.feature.query(foo)", new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), @@ -451,7 +451,7 @@ public class RankingExpressionTypeResolverTestCase { new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), builder.getQueryProfileRegistry().getTypeRegistry()); builder.getQueryProfileRegistry().getTypeRegistry().register(myType); - builder.importString(joinLines( + builder.addSchema(joinLines( "search test {", " document test { ", " field anyfield type tensor(d[2]) {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index 09aa09a626b..d44c102a5c3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -259,7 +259,7 @@ public class RankingExpressionWithOnnxTestCase { public void testFunctionGeneration() { final String name = "small_constants_and_functions"; final String rankProfiles = - " rank-profile my_profile {\n" + + " rank-profile my_profile {\n" + " function input() {\n" + " expression: tensor<float>(d0[3])(0.0)\n" + " }\n" + @@ -399,8 +399,10 @@ public class RankingExpressionWithOnnxTestCase { @Override public List<NamedReader> getFiles(Path path, String suffix) { + File[] files = getFileReference(path).listFiles(); + if (files == null) return List.of(); List<NamedReader> readers = new ArrayList<>(); - for (File file : getFileReference(path).listFiles()) { + for (File file : files) { if ( ! file.getName().endsWith(suffix)) continue; try { readers.add(new NamedReader(file.getName(), new FileReader(file))); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java index 93de116883a..ffc073ed434 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java @@ -9,7 +9,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; import com.yahoo.searchdefinition.expressiontransforms.TokenTransformer; import com.yahoo.searchdefinition.parser.ParseException; @@ -87,9 +87,9 @@ public class RankingExpressionWithTransformerTokensTestCase { " document test {}\n" + " rank-profile my_profile inherits default {}\n" + "}"; - SchemaBuilder schemaBuilder = new SchemaBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); - schemaBuilder.importString(sdContent); - schemaBuilder.build(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); + schemaBuilder.addSchema(sdContent); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankProfile rp = rankProfileRegistry.get(schema, "my_profile"); return new RankProfileTransformContext(rp, queryProfileRegistry, Collections.emptyMap(), null, Collections.emptyMap(), Collections.emptyMap()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index f4742be6b30..b42f932ef98 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -11,7 +11,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; @@ -31,7 +31,7 @@ import static org.junit.Assert.assertTrue; public class RankingExpressionsTestCase extends AbstractSchemaTestCase { private static Schema createSearch(String dir, ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - return SchemaBuilder.createFromDirectory(dir, new MockFileRegistry(), new TestableDeployLogger(), deployProperties, rankProfileRegistry).getSchema(); + return ApplicationBuilder.createFromDirectory(dir, new MockFileRegistry(), new TestableDeployLogger(), deployProperties, rankProfileRegistry).getSchema(); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java index c9437761e0d..d60cb9040d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.document.Field; import com.yahoo.document.ReferenceDataType; import com.yahoo.searchdefinition.DocumentGraphValidator; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Rule; @@ -28,7 +28,7 @@ public class ReferenceFieldTestCase { @Test public void reference_fields_are_parsed_from_search_definition() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); + ApplicationBuilder builder = new ApplicationBuilder(); String campaignSdContent = "search campaign {\n" + " document campaign {\n" + @@ -46,10 +46,10 @@ public class ReferenceFieldTestCase { " field salesperson_ref type reference<salesperson> { indexing: attribute }\n" + " }\n" + "}"; - builder.importString(campaignSdContent); - builder.importString(salespersonSdContent); - builder.importString(adSdContent); - builder.build(); + builder.addSchema(campaignSdContent); + builder.addSchema(salespersonSdContent); + builder.addSchema(adSdContent); + builder.build(true); Schema schema = builder.getSchema("ad"); assertSearchContainsReferenceField("campaign_ref", "campaign", schema.getDocument()); assertSearchContainsReferenceField("salesperson_ref", "salesperson", schema.getDocument()); @@ -57,7 +57,7 @@ public class ReferenceFieldTestCase { @Test public void cyclic_document_dependencies_are_detected() throws ParseException { - SchemaBuilder builder = new SchemaBuilder(); + ApplicationBuilder builder = new ApplicationBuilder(); String campaignSdContent = "search campaign {\n" + " document campaign {\n" + @@ -70,11 +70,11 @@ public class ReferenceFieldTestCase { " field campaign_ref type reference<campaign> { indexing: attribute }\n" + " }\n" + "}"; - builder.importString(campaignSdContent); - builder.importString(adSdContent); + builder.addSchema(campaignSdContent); + builder.addSchema(adSdContent); exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); exceptionRule.expectMessage("Document dependency cycle detected: campaign->ad->campaign."); - builder.build(); + builder.build(true); } private static void assertSearchContainsReferenceField(String expectedFieldname, @@ -87,4 +87,5 @@ public class ReferenceFieldTestCase { ReferenceDataType refField = (ReferenceDataType) dataType; assertEquals(referencedDocType, refField.getTargetType().getName()); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java index 922680e8f1a..d3d79fd7798 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -21,8 +21,8 @@ public class ReservedRankingExpressionFunctionNamesTestCase { public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { TestDeployLogger deployLogger = new TestDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SchemaBuilder builder = new SchemaBuilder(deployLogger, rankProfileRegistry); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(deployLogger, rankProfileRegistry); + builder.addSchema( "search test {\n" + " document test { \n" + " field a type string { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java index 24dc98e4d93..4cfe4c09052 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -17,7 +17,7 @@ public class SchemaMustHaveDocumentTest { @Test public void requireErrorWhenMissingDocument() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalid_sd_missing_document.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_missing_document.sd"); fail("SD without document"); } catch (IllegalArgumentException e) { if (!e.getMessage() diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java index 067a132cc4a..f224dc0688e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.Test; @@ -39,7 +39,7 @@ public class SummaryConsistencyTestCase { "", "}" ); - Schema schema = SchemaBuilder.createFromString(sd).getSchema(); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index eb3e61b9f7a..1eb5e66df93 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -21,7 +21,7 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTest @Test public void requireThatInvalidSourceIsCaught() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalidsummarysource.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalidsummarysource.sd"); fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); @@ -31,7 +31,7 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTest @Test public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalidimplicitsummarysource.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalidimplicitsummarysource.sd"); fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); @@ -41,7 +41,7 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTest @Test public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { try { - SchemaBuilder.buildFromFile("src/test/examples/invalidselfreferringsummary.sd"); + ApplicationBuilder.buildFromFile("src/test/examples/invalidselfreferringsummary.sd"); fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For schema 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); @@ -50,7 +50,7 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTest @Test public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/documentidinsummary.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/documentidinsummary.sd"); BaseDeployLogger deployLogger = new BaseDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); new SummaryFieldsMustHaveValidSource(schema, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index 82b288dc66d..9e53bd57d77 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; -import static com.yahoo.searchdefinition.SchemaBuilder.createFromString; +import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index eec2ef4c3a0..c71fd3dd489 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -14,7 +14,7 @@ import com.yahoo.searchdefinition.LargeRankExpressions; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; @@ -140,8 +140,8 @@ public class TensorTransformTestCase extends AbstractSchemaTestCase { private List<Pair<String, String>> buildSearch(String expression) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfileRegistry queryProfiles = setupQueryProfileTypes(); - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry, queryProfiles); - builder.importString( + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); + builder.addSchema( "search test {\n" + " document test { \n" + " field double_field type double { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index fe9d19310a9..22fd4e45c4a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -51,7 +51,7 @@ public class ValidateFieldTypesTest { private static Schema createSearch(String documentType) { return new Schema(documentType, - new Application(MockApplicationPackage.createEmpty()), + MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java index 2a737f4eb69..e56d6ccf343 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -16,7 +16,7 @@ public class WeightedSetSummaryToTestCase extends AbstractSchemaTestCase { @Test public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { - Schema schema = SchemaBuilder.buildFromFile("src/test/examples/weightedset-summaryto.sd"); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/weightedset-summaryto.sd"); assertNotNull(schema); } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java index 754979fb391..33d84cf4313 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -27,27 +27,27 @@ public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenc } private static class TestDocumentModelBuilder { - private final SchemaBuilder builder = new SchemaBuilder(); + private final ApplicationBuilder builder = new ApplicationBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { - builder.importString(joinLines("search campaign {", - " document campaign {", - " field cool_field type string { indexing: attribute }", - " field swag_field type long { indexing: attribute }", - " }", - "}")); + builder.addSchema(joinLines("search campaign {", + " document campaign {", + " field cool_field type string { indexing: attribute }", + " field swag_field type long { indexing: attribute }", + " }", + "}")); return this; } public TestDocumentModelBuilder addPerson() throws ParseException { - builder.importString(joinLines("search person {", - " document person {", - " field name type string { indexing: attribute }", - " }", - "}")); + builder.addSchema(joinLines("search person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); return this; } public DocumentModel build(String adSdContent) throws ParseException { - builder.importString(adSdContent); - builder.build(); + builder.addSchema(adSdContent); + builder.build(true); return builder.getModel(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index f9b9bf2610e..00f39a23a8f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.ReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -56,22 +56,22 @@ public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReference } private static class TestDocumentModelBuilder { - private final SchemaBuilder builder = new SchemaBuilder(); + private final ApplicationBuilder builder = new ApplicationBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { - builder.importString(joinLines("search campaign {", - " document campaign {}", - "}")); + builder.addSchema(joinLines("search campaign {", + " document campaign {}", + "}")); return this; } public TestDocumentModelBuilder addPerson() throws ParseException { - builder.importString(joinLines("search person {", - " document person {}", - "}")); + builder.addSchema(joinLines("search person {", + " document person {}", + "}")); return this; } public DocumentModel build(String adSdContent) throws ParseException { - builder.importString(adSdContent); - builder.build(); + builder.addSchema(adSdContent); + builder.build(true); return builder.getModel(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index 9dfd05b6fdc..dc88590a198 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; @@ -35,10 +35,10 @@ public class DocumentModelBuilderTestCase extends AbstractSchemaTestCase { @Test public void testDocumentTypesWithDocumentField() throws IOException, ParseException { - SchemaBuilder search = new SchemaBuilder(); - search.importFile("src/test/configmodel/types/other_doc.sd"); - search.importFile("src/test/configmodel/types/type_with_doc_field.sd"); - search.build(); + ApplicationBuilder search = new ApplicationBuilder(); + search.addSchemaFile("src/test/configmodel/types/other_doc.sd"); + search.addSchemaFile("src/test/configmodel/types/type_with_doc_field.sd"); + search.build(true); DocumentModel model = search.getModel(); DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); @@ -48,15 +48,15 @@ public class DocumentModelBuilderTestCase extends AbstractSchemaTestCase { @Test public void testMultipleInheritanceArray() throws IOException, ParseException { - SchemaBuilder search = new SchemaBuilder(); - search.importFile("src/test/cfg/search/data/travel/schemas/TTData.sd"); - search.importFile("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); - search.importFile("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); - search.build(); + ApplicationBuilder search = new ApplicationBuilder(); + search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTData.sd"); + search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); + search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); + search.build(true); } private DocumentModel createAndTestModel(String sd) throws IOException, ParseException { - SchemaBuilder search = SchemaBuilder.createFromFile(sd); + ApplicationBuilder search = ApplicationBuilder.createFromFile(sd); DocumentModel model = search.getModel(); assertEquals(2, model.getDocumentManager().getTypes().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 80a388a822e..4d8dd3dacfd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -26,7 +26,7 @@ public class HostResourceTest { TestService service = new TestService(root, 1); try { - service.initService(root.deployLogger()); + service.initService(root.getDeployState()); } catch (RuntimeException e) { assertTrue(e.getMessage().endsWith("No host found for service 'hostresourcetest$testservice0'. " + "The hostalias is probably missing from hosts.xml.")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 474013c17fc..8012a00076b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -291,7 +291,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { Service cc = model.getService("admin/cluster-controllers/0").get(); assertTrue(cc instanceof ClusterControllerContainer); - assertEquals("-Dio.netty.allocator.pageSize=4096 -Dio.netty.allocator.maxOrder=8", cc.getJvmOptions()); + assertEquals("-Dio.netty.allocator.pageSize=4096 -Dio.netty.allocator.maxOrder=5 -Dio.netty.allocator.numHeapArenas=1 -Dio.netty.allocator.numDirectArenas=1", cc.getJvmOptions()); } private boolean existsHostsWithClusterControllerConfigId(VespaModel model) { @@ -397,7 +397,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); - assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize()); + assertEquals(16, qrStartConfig.jvm().baseMaxDirectMemorySize()); assertReindexingConfigPresent(model); assertReindexingConfiguredOnAdminCluster(model); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java deleted file mode 100644 index e2386e145ca..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.provision.Cloud; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import org.junit.Before; - -/** - * @author gjoranv - */ -public class AwsAccessControlValidatorTest extends AccessControlValidatorTestBase { - - @Before - public void setup() { - validator = new AwsAccessControlValidator(); - zone = new Zone(Cloud.builder().requireAccessControl(true).build(), - SystemName.main, Environment.prod, RegionName.from("foo")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java index e2eae30d78d..b5b93be6cd7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -2,39 +2,44 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; -import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.jar.Manifest; +import java.util.jar.JarOutputStream; +import static com.yahoo.yolean.Exceptions.uncheck; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class BundleValidatorTest { - private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/"; + @Rule + public TemporaryFolder tempDir = new TemporaryFolder(); @Test public void basicBundleValidation() throws Exception { // Valid jar file - JarFile ok = new JarFile(new File(JARS_DIR + "ok.jar")); + JarFile ok = createTemporaryJarFile("ok"); BundleValidator bundleValidator = new BundleValidator(); - bundleValidator.validateJarFile(new BaseDeployLogger(), ok); + bundleValidator.validateJarFile(DeployState.createTestState(), ok); // No manifest - validateWithException("nomanifest.jar", "Non-existing or invalid manifest in " + JARS_DIR + "nomanifest.jar"); + validateWithException("nomanifest", "Non-existing or invalid manifest in nomanifest.jar"); } private void validateWithException(String jarName, String exceptionMessage) throws IOException { try { - JarFile jarFile = new JarFile(JARS_DIR + jarName); + JarFile jarFile = createTemporaryJarFile(jarName); BundleValidator bundleValidator = new BundleValidator(); - bundleValidator.validateJarFile(new BaseDeployLogger(), jarFile); + bundleValidator.validateJarFile(DeployState.createTestState(), jarFile); assert (false); } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), exceptionMessage); @@ -45,26 +50,48 @@ public class BundleValidatorTest { public void require_that_deploying_snapshot_bundle_gives_warning() throws IOException { final StringBuffer buffer = new StringBuffer(); - DeployLogger logger = createDeployLogger(buffer); - - new BundleValidator().validateJarFile(logger, new JarFile(JARS_DIR + "snapshot_bundle.jar")); + DeployState state = createDeployState(buffer); + JarFile jarFile = createTemporaryJarFile("snapshot_bundle"); + new BundleValidator().validateJarFile(state, jarFile); assertTrue(buffer.toString().contains("Deploying snapshot bundle")); } @Test public void outputs_deploy_warning_on_import_of_packages_from_deprecated_artifact() throws IOException { final StringBuffer buffer = new StringBuffer(); - DeployLogger logger = createDeployLogger(buffer); + DeployState state = createDeployState(buffer); BundleValidator validator = new BundleValidator(); - Manifest manifest = new Manifest(Files.newInputStream(Paths.get(JARS_DIR + "/manifest-producing-import-warnings.MF"))); - validator.validateManifest(logger, "my-app-bundle.jar", manifest); + JarFile jarFile = createTemporaryJarFile("import-warnings"); + validator.validateJarFile(state, jarFile); assertThat(buffer.toString()) - .contains("For JAR file 'my-app-bundle.jar': \n" + - "Manifest imports the following Java packages from 'org.json:json': [org.json]. \n" + - "The org.json library will no longer provided by jdisc runtime on Vespa 8. See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); + .contains("JAR file 'import-warnings.jar' imports the packages [org.json] from 'org.json:json'. \n" + + "This bundle is no longer provided on Vespa 8 - see https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); } - private DeployLogger createDeployLogger(StringBuffer buffer) { - return (__, message) -> buffer.append(message).append('\n'); + private DeployState createDeployState(StringBuffer buffer) { + DeployLogger logger = (__, message) -> buffer.append(message).append('\n'); + return DeployState.createTestState(logger); } + + private JarFile createTemporaryJarFile(String testArtifact) throws IOException { + Path jarFile = tempDir.newFile(testArtifact + ".jar").toPath(); + Path artifactDirectory = Paths.get("src/test/cfg/application/validation/testjars/" + testArtifact); + try (JarOutputStream out = new JarOutputStream(Files.newOutputStream(jarFile))) { + Files.walk(artifactDirectory).forEach(path -> { + Path relativePath = artifactDirectory.relativize(path); + String zipName = relativePath.toString(); + uncheck(() -> { + if (Files.isDirectory(path)) { + out.putNextEntry(new JarEntry(zipName + "/")); + } else { + out.putNextEntry(new JarEntry(zipName)); + out.write(Files.readAllBytes(path)); + } + out.closeEntry(); + }); + }); + } + return new JarFile(jarFile.toFile()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java index 689f1c70b6e..85050aa0cf9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java @@ -105,7 +105,7 @@ public class ComplexAttributeFieldsValidatorTestCase { DeployState deployState = createDeployState(servicesXml(), schema); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationParameters validationParameters = new ValidationParameters(CheckRouting.FALSE); - Validation.validate(model, validationParameters, deployState); + new Validation().validate(model, validationParameters, deployState); } private static DeployState createDeployState(String servicesXml, String schema) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java index 90d5c0b27a4..3b5c621a581 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java @@ -10,7 +10,7 @@ import java.io.Reader; import java.io.StringReader; import static com.yahoo.test.json.JsonTestHelper.inputJson; -import static com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; +import static com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; public class ConstantTensorJsonValidatorTest { @@ -88,7 +88,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_bound_tensor_outside_limits_is_disallowed() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Index 5 not within limits of bound dimension 'x'"); validateTensorJson( @@ -121,7 +121,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_non_integer_strings_in_address_points_are_disallowed_unbound() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Index 'a' for dimension 'x' is not an integer"); validateTensorJson( @@ -139,7 +139,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_tensor_coordinates_are_strings() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Tensor label is not a string (VALUE_NUMBER_INT)"); validateTensorJson( @@ -157,7 +157,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_non_integer_strings_in_address_points_are_disallowed_bounded() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Index 'a' for dimension 'x' is not an integer"); validateTensorJson( @@ -175,7 +175,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_missing_coordinates_fail() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Tensor address missing dimension(s) y, z"); validateTensorJson( @@ -193,7 +193,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_non_number_values_are_disallowed() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Tensor value is not a number (VALUE_STRING)"); validateTensorJson( @@ -211,7 +211,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_extra_dimensions_are_disallowed() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Tensor dimension 'z' does not exist"); validateTensorJson( @@ -229,7 +229,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_duplicate_dimensions_are_disallowed() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Duplicate tensor dimension 'y'"); validateTensorJson( @@ -247,7 +247,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_invalid_json_fails() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Failed to parse JSON stream"); validateTensorJson( @@ -265,7 +265,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_invalid_json_not_in_tensor_format_fails() { - expectedException.expect(InvalidConstantTensor.class); + expectedException.expect(InvalidConstantTensorException.class); expectedException.expectMessage("Expected field name 'cells', got 'stats'"); validateTensorJson(TensorType.fromSpec("tensor(x[], y[])"), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java index b1847e5d134..f6fa73dce74 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Test; -import static com.yahoo.vespa.model.application.validation.RankingConstantsValidator.TensorValidationFailed; +import static com.yahoo.vespa.model.application.validation.RankingConstantsValidator.TensorValidationException; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -20,7 +20,7 @@ public class RankingConstantsValidatorTest { try { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_fail/").create(); fail(); - } catch (TensorValidationFailed e) { + } catch (TensorValidationException e) { assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_2' (tensors/constant_tensor_2.json): Tensor label is not a string (VALUE_NUMBER_INT)")); assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_3' (tensors/constant_tensor_3.json): Tensor dimension 'cd' does not exist")); assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_4' (tensors/constant_tensor_4.json): Tensor dimension 'z' does not exist")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java index 16bff341c0b..93d42e07dd8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java @@ -37,7 +37,7 @@ public class ValidationOverridesValidatorTest { var deployState = createDeployState(validationOverridesXml); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, new ValidationParameters(), deployState); + new Validation().validate(model, new ValidationParameters(), deployState); } @Test @@ -58,7 +58,7 @@ public class ValidationOverridesValidatorTest { try { var deployState = createDeployState(validationOverridesXml); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, new ValidationParameters(), deployState); + new Validation().validate(model, new ValidationParameters(), deployState); fail("Did not get expected exception"); } catch (IllegalArgumentException e) { assertEquals(message, e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java deleted file mode 100644 index b9e92e1b866..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation.first; - -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.application.validation.AccessControlValidatorTestBase; -import org.junit.Before; - -/** - * @author gjoranv - */ -public class AccessControlOnFirstDeploymentValidatorTest extends AccessControlValidatorTestBase { - - @Before - public void setup() { - validator = new AccessControlOnFirstDeploymentValidator(); - zone = new Zone(Environment.prod, RegionName.from("foo")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index c746db168ee..77fa0b685fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.config.GenericConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; @@ -35,7 +36,6 @@ import java.util.List; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.vespa.config.search.core.ProtonConfig.Feeding.Shared_field_writer_executor; -import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -328,10 +328,64 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(200000, b.getRootGroup().getMmapNoCoreLimit().get().longValue()); assertEquals(2, s.getSearchNodes().size()); - assertEquals(200000, s.getSearchNodes().get(0).getMMapNoCoreLimit()); - assertEquals(200000, s.getSearchNodes().get(1).getMMapNoCoreLimit()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(0).getMMapNoCoreEnvVariable()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(1).getMMapNoCoreEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + } + + @Test + public void canAddEnvironmentVariable() { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\" />" + + " </group>" + + "</content>"); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(1, b.getStorageCluster().getChildren().size()); + + assertEquals(1, s.getSearchNodes().size()); + AbstractService node = s.getSearchNodes().get(0); + node.addEnvironmentVariable("MY_ENV_1", 7); + node.addEnvironmentVariable("MY_ENV_2", "7 8"); + assertTrue(node.getEnv().contains("MY_ENV_1=7")); + assertTrue(node.getEnv().contains("MY_ENV_2=\"7 8\"")); + node.addEnvironmentVariable("MY_PARSED_ENV_1=7"); + node.addEnvironmentVariable("MY_PARSED_ENV_2=7 8"); + assertTrue(node.getEnv().contains("MY_PARSED_ENV_1=\"7\"")); + assertTrue(node.getEnv().contains("MY_PARSED_ENV_2=\"7 8\"")); + } + + @Test + public void addsEnvironmentVariablesfromFeatureFlag() { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\" />" + + " </group>" + + "</content>", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2"))); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(1, b.getStorageCluster().getChildren().size()); + + assertEquals(1, s.getSearchNodes().size()); + AbstractService node = s.getSearchNodes().get(0); + assertTrue(node.getEnv().contains("MY_1_ENV=\"xyz abc\"")); + assertTrue(node.getEnv().contains("MY_2_ENV=\"2\"")); } @Test @@ -357,10 +411,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertTrue(b.getRootGroup().getCoreOnOOM().get()); assertEquals(2, s.getSearchNodes().size()); - assertTrue(s.getSearchNodes().get(0).getCoreOnOOM()); - assertTrue(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertFalse(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertFalse(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -383,10 +435,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getCoreOnOOM().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertFalse(s.getSearchNodes().get(0).getCoreOnOOM()); - assertFalse(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -409,10 +459,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getMmapNoCoreLimit().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertEquals(200000, s.getSearchNodes().get(0).getMMapNoCoreLimit()); - assertEquals(-1, s.getSearchNodes().get(1).getMMapNoCoreLimit()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(0).getMMapNoCoreEnvVariable()); - assertEquals("", s.getSearchNodes().get(1).getMMapNoCoreEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + assertFalse(s.getSearchNodes().get(1).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=")); } @Test @@ -435,10 +483,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getCoreOnOOM().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertTrue(s.getSearchNodes().get(0).getCoreOnOOM()); - assertFalse(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertFalse(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -471,15 +517,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(4, s.getSearchNodes().size()); for (SearchNode n : s.getSearchNodes()) { - assertEquals("proton", n.getNoVespaMalloc()); - assertEquals("VESPA_USE_NO_VESPAMALLOC=\"proton\" ", n.getNoVespaMallocEnvVariable()); - assertEquals("distributord", n.getVespaMallocDebug()); - assertEquals("VESPA_USE_VESPAMALLOC=\"storaged\" ", n.getVespaMallocEnvVariable()); - assertEquals("all", n.getVespaMallocDebugStackTrace()); - assertEquals("VESPA_USE_VESPAMALLOC_D=\"distributord\" ", n.getVespaMallocDebugEnvVariable()); - assertEquals("storaged", n.getVespaMalloc()); - assertEquals("VESPA_USE_VESPAMALLOC_DST=\"all\" ", n.getVespaMallocDebugStackTraceEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_NO_VESPAMALLOC=\"proton\" VESPA_USE_VESPAMALLOC=\"storaged\" VESPA_USE_VESPAMALLOC_D=\"distributord\" VESPA_USE_VESPAMALLOC_DST=\"all\" ", n.getEnvVariables()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_NO_VESPAMALLOC=\"proton\" VESPA_USE_VESPAMALLOC=\"storaged\" VESPA_USE_VESPAMALLOC_D=\"distributord\" VESPA_USE_VESPAMALLOC_DST=\"all\"", n.getEnv()); } } @@ -508,10 +546,10 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getVespaMallocDebugStackTrace().isPresent()); assertEquals(4, s.getSearchNodes().size()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_NO_VESPAMALLOC=\"proton\" ", s.getSearchNodes().get(0).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC_D=\"distributord\" ", s.getSearchNodes().get(1).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC_DST=\"all\" ", s.getSearchNodes().get(2).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC=\"storaged\" ", s.getSearchNodes().get(3).getEnvVariables()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_NO_VESPAMALLOC=\"proton\"", s.getSearchNodes().get(0).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC_D=\"distributord\"", s.getSearchNodes().get(1).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC_DST=\"all\"", s.getSearchNodes().get(2).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC=\"storaged\"", s.getSearchNodes().get(3).getEnv()); } @Test @@ -908,6 +946,9 @@ public class ContentBuilderTest extends DomBuilderTest { } private ContentCluster createContent(String xml) { + return createContent(xml, new TestProperties()); + } + private ContentCluster createContent(String xml, TestProperties props) { String combined = "" + "<services>"+ " <admin version='2.0'>" + @@ -917,12 +958,13 @@ public class ContentBuilderTest extends DomBuilderTest { "</services>"; + var deployStateBuilder = new DeployState.Builder().properties(props); VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withHosts(getHosts()) - .withServices(combined) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(); + .withHosts(getHosts()) + .withServices(combined) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); return m.getContentClusters().isEmpty() ? null diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index eb1cf668cc9..198b5713876 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -6,7 +6,6 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentId; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; @@ -178,7 +177,7 @@ public class ContainerClusterTest { public void testClusterControllerResourceUsage() { MockRoot root = createRoot(false); ClusterControllerContainerCluster cluster = createClusterControllerCluster(root); - addClusterController(root.deployLogger(), cluster, "host-c1", root.getDeployState()); + addClusterController(cluster, "host-c1", root.getDeployState()); assertEquals(1, cluster.getContainers().size()); QrStartConfig.Builder qrBuilder = new QrStartConfig.Builder(); cluster.getConfig(qrBuilder); @@ -190,7 +189,7 @@ public class ContainerClusterTest { root.freezeModelTopology(); ThreadpoolConfig threadpoolConfig = root.getConfig(ThreadpoolConfig.class, "container0/component/default-threadpool"); - assertEquals(10, threadpoolConfig.maxthreads()); + assertEquals(2, threadpoolConfig.maxthreads()); assertEquals(50, threadpoolConfig.queueSize()); } @@ -198,7 +197,7 @@ public class ContainerClusterTest { public void testThatLinguisticsIsExcludedForClusterControllerCluster() { MockRoot root = createRoot(false); ClusterControllerContainerCluster cluster = createClusterControllerCluster(root); - addClusterController(root.deployLogger(), cluster, "host-c1", root.getDeployState()); + addClusterController(cluster, "host-c1", root.getDeployState()); assertFalse(contains("com.yahoo.language.provider.DefaultLinguisticsProvider", cluster.getAllComponents())); } @@ -365,71 +364,61 @@ public class ContainerClusterTest { assertNames(main, ApplicationId.from("t1", "a1", "i1"), Set.of(), - List.of("search-cluster.i1.a1.t1.endpoint.suffix"), - List.of("search-cluster--i1--a1--t1.endpoint.suffix")); + List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "default", "default"), Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix"), - List.of("search-cluster--default--t1.endpoint.suffix")); + List.of("search-cluster.default.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.a1.t1.endpoint.suffix"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), - List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); // cd system: assertNames(cd, ApplicationId.from("t1", "a1", "i1"), Set.of(), - List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--i1--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(), - List.of("search-cluster.cd.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "default", "default"), Set.of(), - List.of("search-cluster.cd.default.t1.endpoint.suffix"), - List.of("search-cluster--cd--default--t1.endpoint.suffix")); + List.of("search-cluster.cd.default.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.cd.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", global, List.of("a--b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), + new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), - List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix", "a--b.x.y.z", "rotation-2.x.y.z")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); } - private void assertNames(SystemName systemName, ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedSharedL4Names, List<String> expectedSharedNames) { + private void assertNames(SystemName systemName, ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedSharedL4Names) { Zone zone = new Zone(systemName, Environment.defaultEnvironment(), RegionName.defaultName()); DeployState state = new DeployState.Builder() .zone(zone) @@ -445,7 +434,7 @@ public class ContainerClusterTest { cluster.doPrepare(state); List<ApplicationClusterEndpoint> endpoints = cluster.endpoints(); - assertNames(expectedSharedNames, endpoints.stream().filter(e -> e.routingMethod() == shared).collect(Collectors.toList())); + assertNames(List.of(), endpoints.stream().filter(e -> e.routingMethod() == shared).collect(Collectors.toList())); assertNames(expectedSharedL4Names, endpoints.stream().filter(e -> e.routingMethod() == sharedLayer4).collect(Collectors.toList())); List<ContainerEndpoint> endpointsWithWeight = @@ -493,17 +482,16 @@ public class ContainerClusterTest { HostResource hostResource) { ApplicationContainer container = new ApplicationContainer(cluster, name, 0, root.getDeployState()); container.setHostResource(hostResource); - container.initService(root.deployLogger()); + container.initService(root.getDeployState()); cluster.addContainer(container); } - private static void addClusterController(DeployLogger deployLogger, - ClusterControllerContainerCluster cluster, + private static void addClusterController(ClusterControllerContainerCluster cluster, String hostName, DeployState deployState) { ClusterControllerContainer container = new ClusterControllerContainer(cluster, 1, false, deployState, false); container.setHostResource(new HostResource(new Host(null, hostName))); - container.initService(deployLogger); + container.initService(deployState); cluster.addContainer(container); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 4ca85a19c35..29cff23d3bd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -73,7 +73,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); assertEquals(2, config.myid()); - assertEquals("", config.snapshotMethod()); + assertEquals("gz", config.snapshotMethod()); } @Test(expected = IllegalArgumentException.class) @@ -152,8 +152,7 @@ public class ConfigserverClusterTest { .useVespaVersionInRequest(true) .hostedVespa(hostedVespa) .environment("test") - .region("bar") - .zooKeeperSnapshotMethod(hostedVespa ? "gz" : ""); + .region("bar"); Optional.of(configServerHostnames) .filter(hostnames -> !hostnames.isEmpty()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index fc7f8674149..95c3262f2f5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; -import java.util.Objects; import java.util.Optional; /** @@ -18,7 +17,6 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); - private String zooKeeperSnapshotMethod = ""; @Override public Optional<Integer> rpcPort() { @@ -108,9 +106,6 @@ public class TestOptions implements CloudConfigOptions { return Optional.empty(); } - @Override - public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } - public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -136,10 +131,4 @@ public class TestOptions implements CloudConfigOptions { return this; } - public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { - Objects.requireNonNull(snapshotMethod); - this.zooKeeperSnapshotMethod = snapshotMethod; - return this; - } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java index 5630d3cc186..f246b87d9bf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.ml; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.tensor.Tensor; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; /** * Tests the ModelsEvaluatorTester. @@ -17,6 +19,7 @@ public class ModelsEvaluatorTest { @Test public void testModelsEvaluatorTester() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); ModelsEvaluator modelsEvaluator = ModelsEvaluatorTester.create("src/test/cfg/application/stateless_eval"); assertEquals(3, modelsEvaluator.models().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java index a9bab88a065..2cb0e8f0ab1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.util.List; import java.util.Set; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -80,8 +81,8 @@ public class SourceGroupTest { searchChains.validate(); fail("Expected exception"); } catch (Exception e) { - assertTrue(e.getMessage().contains("Same id used for a source")); - assertTrue(e.getMessage().contains("'sameId'")); + assertEquals("Id 'sameId' is used both for a source and another search chain/provider", + e.getMessage()); } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SchemaBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java index b50261b7568..3490568770c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SchemaBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java @@ -31,7 +31,7 @@ import static org.junit.Assert.fail; /** * @author gjoranv */ -public class SchemaBuilderTest extends ContainerModelBuilderTestBase { +public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { private ChainsConfig chainsConfig() { return root.getConfig(ChainsConfig.class, "default/component/com.yahoo.search.handler.SearchHandler"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index f7dbd65ab7d..16b4a8bed9b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -778,7 +778,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .build(); createModel(root, state, null, clusterElem); } catch (RuntimeException e) { - assertEquals("cloud secret store is not supported in non-public system, please see documentation", + assertEquals("Cloud secret store is not supported in non-public system, see the documentation", e.getMessage()); return; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index a17459ea800..814b2f94e44 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -181,7 +181,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { "-XX:+ParallelGCThreads=8 foo bar"); fail(); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Invalid JVM GC options in services.xml: bar,foo")); + assertTrue(e.getMessage().startsWith("Invalid or misplaced JVM GC options in services.xml: bar,foo")); } } @@ -205,8 +205,10 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { assertEquals(Level.WARNING, firstOption.getFirst()); Collections.sort(strings); - assertEquals("Invalid JVM " + (optionName.equals("gc-options") ? "GC " : "") + - "options in services.xml: " + String.join(",", strings), firstOption.getSecond()); + assertEquals("Invalid or misplaced JVM" + (optionName.equals("gc-options") ? " GC" : "") + + " options in services.xml: " + String.join(",", strings) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm" + , firstOption.getSecond()); } private void buildModelWithJvmOptions(boolean isHosted, TestLogger logger, String optionName, String override) throws IOException, SAXException { @@ -267,7 +269,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { "-Xms2G foo bar"); fail(); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Invalid JVM options in services.xml: bar,foo")); + assertTrue(e.getMessage().contains("Invalid or misplaced JVM options in services.xml: bar,foo")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java index 8cc7805fe3e..c9863cf2144 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -21,7 +21,6 @@ import static org.junit.Assert.fail; public class ClusterResourceLimitsTest { private static class Fixture { - private final boolean enableFeedBlockInDistributor; private final boolean hostedVespa; private final ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private final ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); @@ -30,12 +29,7 @@ public class ClusterResourceLimitsTest { this(false); } - public Fixture(boolean enableFeedBlockInDistributor) { - this(enableFeedBlockInDistributor, false); - } - - public Fixture(boolean enableFeedBlockInDistributor, boolean hostedVespa) { - this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; + public Fixture(boolean hostedVespa) { this.hostedVespa = hostedVespa; } @@ -57,8 +51,7 @@ public class ClusterResourceLimitsTest { } public ClusterResourceLimits build() { ModelContext.FeatureFlags featureFlags = new TestProperties(); - var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - hostedVespa, + var builder = new ClusterResourceLimits.Builder(hostedVespa, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()); builder.setClusterControllerBuilder(ctrlBuilder); @@ -71,50 +64,32 @@ public class ClusterResourceLimitsTest { public void content_node_limits_are_derived_from_cluster_controller_limits_if_not_set() { assertLimits(0.4, 0.7, 0.7, 0.85, new Fixture().ctrlDisk(0.4).ctrlMemory(0.7)); - assertLimits(0.4, null, 0.7, null, + assertLimits(0.4, 0.8, 0.7, 0.9, new Fixture().ctrlDisk(0.4)); - assertLimits(null, 0.7, null, 0.85, + assertLimits(0.75, 0.7, 0.875, 0.85, new Fixture().ctrlMemory(0.7)); - - - assertLimits(0.4, 0.7, 0.7, 0.85, - new Fixture(true).ctrlDisk(0.4).ctrlMemory(0.7)); - assertLimits(0.4, 0.8, 0.7, 0.9, - new Fixture(true).ctrlDisk(0.4)); - assertLimits(0.8, 0.7, 0.9, 0.85, - new Fixture(true).ctrlMemory(0.7)); } @Test public void content_node_limits_can_be_set_explicit() { assertLimits(0.4, 0.7, 0.9, 0.95, new Fixture().ctrlDisk(0.4).ctrlMemory(0.7).nodeDisk(0.9).nodeMemory(0.95)); - assertLimits(0.4, null, 0.95, null, + assertLimits(0.4, 0.8, 0.95, 0.9, new Fixture().ctrlDisk(0.4).nodeDisk(0.95)); - assertLimits(null, 0.7, null, 0.95, + assertLimits(0.75, 0.7, 0.875, 0.95, new Fixture().ctrlMemory(0.7).nodeMemory(0.95)); - - assertLimits(0.4, 0.7, 0.9, 0.95, - new Fixture(true).ctrlDisk(0.4).ctrlMemory(0.7).nodeDisk(0.9).nodeMemory(0.95)); - assertLimits(0.4, 0.8, 0.95, 0.9, - new Fixture(true).ctrlDisk(0.4).nodeDisk(0.95)); - assertLimits(0.8, 0.7, 0.9, 0.95, - new Fixture(true).ctrlMemory(0.7).nodeMemory(0.95)); } @Test public void cluster_controller_limits_are_equal_to_content_node_limits_minus_one_percent_if_not_set() { assertLimits(0.89, 0.94, 0.9, 0.95, new Fixture().nodeDisk(0.9).nodeMemory(0.95)); - assertLimits(0.89, null, 0.9, null, + assertLimits(0.89, 0.8, 0.9, 0.9, new Fixture().nodeDisk(0.9)); - assertLimits(null, 0.94, null, 0.95, + assertLimits(0.75, 0.94, 0.875, 0.95, new Fixture().nodeMemory(0.95)); - assertLimits(null, 0.0, null, 0.005, + assertLimits(0.75, 0.0, 0.875, 0.005, new Fixture().nodeMemory(0.005)); - - assertLimits(0.89, 0.94, 0.9, 0.95, - new Fixture(true).nodeDisk(0.9).nodeMemory(0.95)); } @Test @@ -127,7 +102,7 @@ public class ClusterResourceLimitsTest { @Test public void default_resource_limits_when_feed_block_is_enabled_in_distributor() { - assertLimits(0.8, 0.8, 0.9, 0.9, + assertLimits(0.75, 0.8, 0.875, 0.9, new Fixture(true)); } @@ -193,7 +168,6 @@ public class ClusterResourceLimitsTest { "</cluster>"); ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, - true, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()); return builder.build(new ModelElement((limitsInXml ? clusterXml : noLimitsXml).getDocumentElement())); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 87a962339e9..9f571167d8c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1045,6 +1045,28 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); } + private ProtonConfig.Replay_throttling_policy.Type.Enum resolveReplayThrottlePolicyType(Optional<String> throttlerType) { + TestProperties testProperties = new TestProperties(); + if (throttlerType.isPresent()) { + testProperties.setPersistenceAsyncThrottling(throttlerType.get()); + } + VespaModel model = createEnd2EndOneNode(testProperties); + ContentCluster cc = model.getContentClusters().get("storage"); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + cc.getSearch().getConfig(protonBuilder); + ProtonConfig protonConfig = new ProtonConfig(protonBuilder); + assertEquals(1, protonConfig.documentdb().size()); + return protonConfig.replay_throttling_policy().type(); + } + + @Test + public void replay_throttling_policy_type_controlled_by_properties() { + assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.empty())); + assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.of("UNLIMITED"))); + assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.UNLIMITED, resolveReplayThrottlePolicyType(Optional.of("INVALID"))); + assertEquals(ProtonConfig.Replay_throttling_policy.Type.Enum.DYNAMIC, resolveReplayThrottlePolicyType(Optional.of("DYNAMIC"))); + } + private long resolveMaxTLSSize(Optional<Flavor> flavor) throws Exception { TestProperties testProperties = new TestProperties(); @@ -1120,54 +1142,38 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - public void distributor_merge_busy_wait_controlled_by_properties() throws Exception { - assertEquals(1, resolveDistributorMergeBusyWaitConfig(Optional.empty())); - assertEquals(5, resolveDistributorMergeBusyWaitConfig(Optional.of(5))); - } - - private int resolveDistributorMergeBusyWaitConfig(Optional<Integer> mergeBusyWait) throws Exception { - var props = new TestProperties(); - if (mergeBusyWait.isPresent()) { - props.setDistributorMergeBusyWait(mergeBusyWait.get()); - } - var cluster = createOneNodeCluster(props); - var builder = new StorDistributormanagerConfig.Builder(); - cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).inhibit_merge_sending_on_busy_node_duration_sec(); - } - - @Test - public void distributor_enhanced_maintenance_scheduling_controlled_by_properties() throws Exception { - assertFalse(resolveDistributorEnhancedSchedulingConfig(Optional.of(false))); - assertTrue(resolveDistributorEnhancedSchedulingConfig(Optional.empty())); + public void unordered_merge_chaining_config_controlled_by_properties() throws Exception { + assertFalse(resolveUnorderedMergeChainingConfig(Optional.of(false))); + assertTrue(resolveUnorderedMergeChainingConfig(Optional.empty())); } - private boolean resolveDistributorEnhancedSchedulingConfig(Optional<Boolean> enhancedScheduling) throws Exception { + private boolean resolveUnorderedMergeChainingConfig(Optional<Boolean> unorderedMergeChaining) throws Exception { var props = new TestProperties(); - if (enhancedScheduling.isPresent()) { - props.distributorEnhancedMaintenanceScheduling(enhancedScheduling.get()); + if (unorderedMergeChaining.isPresent()) { + props.setUnorderedMergeChaining(unorderedMergeChaining.get()); } var cluster = createOneNodeCluster(props); var builder = new StorDistributormanagerConfig.Builder(); cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).implicitly_clear_bucket_priority_on_schedule(); + return (new StorDistributormanagerConfig(builder)).use_unordered_merge_chaining(); } @Test - public void unordered_merge_chaining_config_controlled_by_properties() throws Exception { - assertFalse(resolveUnorderedMergeChainingConfig(Optional.of(false))); - assertTrue(resolveUnorderedMergeChainingConfig(Optional.empty())); + public void inhibit_default_merges_when_global_merges_pending_controlled_by_properties() throws Exception { + assertFalse(resolveInhibitDefaultMergesConfig(Optional.empty())); + assertFalse(resolveInhibitDefaultMergesConfig(Optional.of(false))); + assertTrue(resolveInhibitDefaultMergesConfig(Optional.of(true))); } - private boolean resolveUnorderedMergeChainingConfig(Optional<Boolean> unorderedMergeChaining) throws Exception { + private boolean resolveInhibitDefaultMergesConfig(Optional<Boolean> inhibitDefaultMerges) throws Exception { var props = new TestProperties(); - if (unorderedMergeChaining.isPresent()) { - props.setUnorderedMergeChaining(unorderedMergeChaining.get()); + if (inhibitDefaultMerges.isPresent()) { + props.inhibitDefaultMergesWhenGlobalMergesPending(inhibitDefaultMerges.get()); } var cluster = createOneNodeCluster(props); var builder = new StorDistributormanagerConfig.Builder(); cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).use_unordered_merge_chaining(); + return (new StorDistributormanagerConfig(builder)).inhibit_default_merges_when_global_merges_pending(); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index 68e722f45d3..ff399fd2294 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -77,12 +77,6 @@ public class ContentSchemaClusterTest { return new ProtonConfig(builder); } - private static ContentCluster createClusterWithFeatureFlag(String clusterXml, boolean enableFeedBlockInDistributor) throws Exception { - var deployStateBuilder = new DeployState.Builder().properties( - new TestProperties().enableFeedBlockInDistributor(enableFeedBlockInDistributor)); - return createCluster(clusterXml, deployStateBuilder); - } - private static void assertProtonResourceLimits(double expDiskLimit, double expMemoryLimit, String clusterXml) throws Exception { assertProtonResourceLimits(expDiskLimit, expMemoryLimit, createCluster(clusterXml)); } @@ -124,7 +118,7 @@ public class ContentSchemaClusterTest { @Test public void requireThatOnlyMemoryLimitCanBeSet() throws Exception { - assertProtonResourceLimits(0.9, 0.77, + assertProtonResourceLimits(0.875, 0.77, new ContentClusterBuilder().protonMemoryLimit(0.77).getXml()); } @@ -142,17 +136,10 @@ public class ContentSchemaClusterTest { } @Test - public void default_resource_limits_when_feed_block_is_disabled_in_distributor() throws Exception { - var cluster = createClusterWithFeatureFlag(new ContentClusterBuilder().getXml(), false); - assertProtonResourceLimits(0.8, 0.8, cluster); - assertClusterControllerResourceLimits(0.8, 0.8, cluster); - } - - @Test - public void default_resource_limits_when_feed_block_is_enabled_in_distributor() throws Exception { - var cluster = createClusterWithFeatureFlag(new ContentClusterBuilder().getXml(), true); - assertProtonResourceLimits(0.9, 0.9, cluster); - assertClusterControllerResourceLimits(0.8, 0.8, cluster); + public void default_resource_limits_with_feed_block_in_distributor() throws Exception { + var cluster = createCluster(new ContentClusterBuilder().getXml()); + assertProtonResourceLimits(0.875, 0.9, cluster); + assertClusterControllerResourceLimits(0.75, 0.8, cluster); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index 90e3aff0c45..d23dca65dd1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -107,7 +107,7 @@ public class DistributorTest { assertEquals(1024, conf.splitcount()); assertEquals(512, conf.joincount()); - assertEquals(33544432, conf.splitsize()); + assertEquals(16772216, conf.splitsize()); assertEquals(16000000, conf.joinsize()); assertEquals(8, conf.minsplitcount()); assertTrue(conf.inlinebucketsplitting()); @@ -138,7 +138,7 @@ public class DistributorTest { assertEquals(1024, conf.splitcount()); assertEquals(512, conf.joincount()); - assertEquals(33544432, conf.splitsize()); + assertEquals(16772216, conf.splitsize()); assertEquals(16000000, conf.joinsize()); assertEquals(14, conf.minsplitcount()); assertTrue(conf.inlinebucketsplitting()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index b27a8a192bc..f9fcb777408 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -19,14 +19,12 @@ public class FleetControllerClusterTest { private ClusterControllerConfig parse(String xml, TestProperties props) { Document doc = XML.getDocument(xml); var deployState = new DeployState.Builder().properties(props).build(); - boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); MockRoot root = new MockRoot("", deployState); var clusterElement = new ModelElement(doc.getDocumentElement()); ModelContext.FeatureFlags featureFlags = new TestProperties(); return new ClusterControllerConfig.Builder("storage", clusterElement, - new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - false, + new ClusterResourceLimits.Builder(false, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()) .build(clusterElement).getClusterControllerLimits()) @@ -34,7 +32,7 @@ public class FleetControllerClusterTest { } private ClusterControllerConfig parse(String xml) { - return parse(xml, new TestProperties().enableFeedBlockInDistributor(true)); + return parse(xml, new TestProperties()); } @Test @@ -104,17 +102,16 @@ public class FleetControllerClusterTest { assertEquals(0.0, config.min_node_ratio_per_group(), 0.01); } - @Test public void default_cluster_feed_block_limits_are_set() { - assertLimits(0.8, 0.8, getConfigForBasicCluster()); + assertLimits(0.75, 0.8, getConfigForBasicCluster()); } @Test public void resource_limits_can_be_set_in_tuning() { assertLimits(0.6, 0.7, getConfigForResourceLimitsTuning(0.6, 0.7)); assertLimits(0.6, 0.8, getConfigForResourceLimitsTuning(0.6, null)); - assertLimits(0.8, 0.7, getConfigForResourceLimitsTuning(null, 0.7)); + assertLimits(0.75, 0.7, getConfigForResourceLimitsTuning(null, 0.7)); } private static final double DELTA = 0.00001; @@ -124,7 +121,7 @@ public class FleetControllerClusterTest { assertEquals(3, limits.size()); assertEquals(expDisk, limits.get("disk"), DELTA); assertEquals(expMemory, limits.get("memory"), DELTA); - assertEquals(0.89, limits.get("attribute-address-space"), DELTA); + assertEquals(0.9, limits.get("attribute-address-space"), DELTA); } private FleetcontrollerConfig getConfigForResourceLimitsTuning(Double diskLimit, Double memoryLimit) { @@ -143,17 +140,6 @@ public class FleetControllerClusterTest { } @Test - public void feature_flag_controls_enable_cluster_feed_block() { - verifyThatFeatureFlagControlsEnableClusterFeedBlock(true); - verifyThatFeatureFlagControlsEnableClusterFeedBlock(false); - } - - private void verifyThatFeatureFlagControlsEnableClusterFeedBlock(boolean flag) { - var config = getConfigForBasicCluster(new TestProperties().enableFeedBlockInDistributor(flag)); - assertEquals(flag, config.enable_cluster_feed_block()); - } - - @Test public void feature_flag_controls_min_node_ratio_per_group() { verifyFeatureFlagControlsMinNodeRatioPerGroup(0.0, new TestProperties()); verifyFeatureFlagControlsMinNodeRatioPerGroup(0.3, @@ -175,6 +161,6 @@ public class FleetControllerClusterTest { } private FleetcontrollerConfig getConfigForBasicCluster() { - return getConfigForBasicCluster(new TestProperties().enableFeedBlockInDistributor(true)); + return getConfigForBasicCluster(new TestProperties()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index cea8d724c30..bcb197ed540 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -444,7 +444,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { for (SearchClusterSpec cluster : searchClusterSpecs) { for (SearchDefSpec def : cluster.searchDefs) { - sds.add(ApplicationPackageUtils.generateSearchDefinition(def.typeName, def.field1Name, def.field2Name)); + sds.add(ApplicationPackageUtils.generateSchema(def.typeName, def.field1Name, def.field2Name)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index cf877d3bf88..509055df7f4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -160,21 +160,22 @@ public class StorageClusterTest { } @Test - public void ignore_merge_queue_limit_can_be_controlled_by_feature_flag() { - var config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(true)); - assertTrue(config.disable_queue_limits_for_chained_merges()); - - config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(false)); - assertFalse(config.disable_queue_limits_for_chained_merges()); + public void merge_throttling_policy_config_defaults_to_static() { + var config = configFromProperties(new TestProperties()); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); } @Test - public void async_apply_bucket_diff_can_be_controlled_by_feature_flag() { - var config = filestorConfigFromProperties(new TestProperties()); - assertTrue(config.async_apply_bucket_diff()); + public void merge_throttling_policy_config_is_derived_from_flag() { + var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type()); - config = filestorConfigFromProperties(new TestProperties().setAsyncApplyBucketDiff(false)); - assertFalse(config.async_apply_bucket_diff()); + // Invalid enum values fall back to the default + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); } @Test @@ -308,20 +309,40 @@ public class StorageClusterTest { @Test public void persistence_async_throttle_config_defaults_to_unlimited() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); } @Test public void persistence_async_throttle_config_is_derived_from_flag() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC, config.async_operation_throttler().type()); // Invalid enum values fall back to the default config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); + } + + @Test + public void persistence_dynamic_throttling_parameters_have_sane_defaults() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); + assertEquals(1.2, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); + assertEquals(0.95, config.async_operation_throttler().window_size_backoff(), 0.0001); + } + + @Test + public void persistence_dynamic_throttling_parameters_can_be_set_through_feature_flags() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties() + .setPersistenceThrottlingWsDecrementFactor(1.5) + .setPersistenceThrottlingWsBackoff(0.8))); + assertEquals(1.5, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); + assertEquals(0.8, config.async_operation_throttler().window_size_backoff(), 0.0001); } @Test @@ -432,8 +453,8 @@ public class StorageClusterTest { ContentClusterUtils.createCluster(xml, root); fail("Did not fail when having both group and nodes"); } catch (RuntimeException e) { - e.printStackTrace(); - assertEquals("Both group and nodes exists, only one of these tags is legal", e.getMessage()); + assertEquals("Both <group> and <nodes> is specified: Only one of these tags can be used in the same configuration", + e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSchemaBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentApplicationBuilderTest.java index b8788ece7b6..11aeea5f90d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSchemaBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentApplicationBuilderTest.java @@ -15,7 +15,7 @@ import static org.junit.Assert.assertNull; /** * @author Simon Thoresen Hult */ -public class DomContentSchemaBuilderTest { +public class DomContentApplicationBuilderTest { @Test public void requireThatDefaultsAreNull() throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index 7627ba6319b..c60817704cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.handler.ModelsEvaluationHandler; @@ -30,6 +31,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; /** * Tests stateless model evaluation (turned on by the "model-evaluation" tag in "container") @@ -61,6 +63,7 @@ public class ModelEvaluationTest { @Test public void testMl_serving() throws IOException { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/ml_serving"); Path storedAppDir = appDir.append("copy"); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index 6e096dd68e4..bad6fcc68c2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; @@ -29,6 +30,7 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; /** * Tests stateless model evaluation (turned on by the "model-evaluation" tag in "container") @@ -39,7 +41,8 @@ import static org.junit.Assert.assertTrue; public class StatelessOnnxEvaluationTest { @Test - public void testStatelessOnnxModelNameCollision() throws IOException { + public void testStatelessOnnxModelNameCollision() { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/onnx_name_collision"); try { ImportedModelTester tester = new ImportedModelTester("onnx", appDir); @@ -60,6 +63,7 @@ public class StatelessOnnxEvaluationTest { @Test public void testStatelessOnnxModelEvaluation() throws IOException { + assumeTrue(OnnxEvaluator.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/onnx"); Path storedAppDir = appDir.append("copy"); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java index 3d4991eebed..30678205e3a 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java @@ -8,7 +8,6 @@ import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; -import org.junit.Ignore; import org.junit.Test; import java.io.BufferedReader; @@ -36,7 +35,6 @@ public class RoutingTestCase { private static final boolean WRITE_FILES = false; @Test - @Ignore // TODO: Why? public void testRoutingContent() throws IOException { assertApplication(new File("src/test/cfg/routing/contentsimpleconfig")); assertApplication(new File("src/test/cfg/routing/content_two_clusters")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index 6830ce79f9f..e58c29cc4fd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -2,11 +2,12 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.component.ComponentId; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.container.QrSearchersConfig; import com.yahoo.document.DataType; import com.yahoo.search.config.ClusterConfig; import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -40,7 +41,7 @@ public class SchemaClusterTest { public void testSdConfigLogical() { // sd1 SDDocumentType sdt1 = new SDDocumentType("s1"); - Schema schema1 = new Schema("s1"); + Schema schema1 = new Schema("s1", MockApplicationPackage.createEmpty()); SDField f1 = new SDField("f1", DataType.STRING); f1.addAttribute(new Attribute("f1", DataType.STRING)); f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); @@ -49,17 +50,17 @@ public class SchemaClusterTest { // sd2 SDDocumentType sdt2 = new SDDocumentType("s2"); - Schema schema2 = new Schema("s2"); + Schema schema2 = new Schema("s2", MockApplicationPackage.createEmpty()); SDField f2=new SDField("f2", DataType.STRING); f2.addAttribute(new Attribute("f2", DataType.STRING)); f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); sdt2.addField(f2); schema2.addDocument(sdt2); - SchemaBuilder builder = new SchemaBuilder(); - builder.importRawSchema(schema1); - builder.importRawSchema(schema2); - builder.build(); + ApplicationBuilder builder = new ApplicationBuilder(); + builder.add(schema1); + builder.add(schema2); + builder.build(true); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java index 226045f4d8a..a347c0a9c43 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java @@ -40,11 +40,11 @@ public class SchemaNodeTest { TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); tls.setHostResource(new HostResource(host)); tls.setBasePort(100); - tls.initService(root.deployLogger()); + tls.initService(root.getDeployState()); node.setTls(tls); node.setHostResource(new HostResource(host)); node.setBasePort(200); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); root.freezeModelTopology(); } @@ -78,7 +78,7 @@ public class SchemaNodeTest { MockRoot root = new MockRoot(""); SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode"))); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); } @@ -87,7 +87,7 @@ public class SchemaNodeTest { MockRoot root = new MockRoot(""); SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); assertTrue(node.getPreShutdownCommand().get().contains("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java index 3388f9d2670..3de8cfe540f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.test; import com.yahoo.component.ComponentId; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModel; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelRegistry; @@ -10,6 +9,7 @@ import com.yahoo.config.model.MapConfigModelRegistry; import com.yahoo.config.model.admin.AdminModel; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -152,13 +152,13 @@ public class ModelAmendingTestCase { @Override public void doBuild(AdminModelAmender model, Element spec, ConfigModelContext modelContext) { for (AdminModel adminModel : model.adminModels) - amend(modelContext.getDeployLogger(), adminModel); + amend(modelContext.getDeployState(), adminModel); } - private void amend(DeployLogger deployLogger, AdminModel adminModel) { + private void amend(DeployState deployState, AdminModel adminModel) { for (HostResource host : adminModel.getAdmin().hostSystem().getHosts()) { if ( ! host.getHost().getChildrenByTypeRecursive(AmendedService.class).isEmpty()) continue; // already amended - adminModel.getAdmin().addAndInitializeService(deployLogger, host, new AmendedService(host.getHost())); + adminModel.getAdmin().addAndInitializeService(deployState, host, new AmendedService(host.getHost())); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index bf4d066e454..7f8bca825d2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -242,7 +242,7 @@ public class VespaModelTestCase { .build(); DeployState deployState = builder.deployLogger(logger).applicationPackage(app).build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, new ValidationParameters(ValidationParameters.IgnoreValidationErrors.TRUE), deployState); + new Validation().validate(model, new ValidationParameters(ValidationParameters.IgnoreValidationErrors.TRUE), deployState); assertFalse(logger.msgs.isEmpty()); } @@ -312,7 +312,7 @@ public class VespaModelTestCase { .deployLogger(logger) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, new ValidationParameters(), deployState); + new Validation().validate(model, new ValidationParameters(), deployState); assertContainsWarning(logger.msgs, "Directory searchdefinitions/ should not be used for schemas, use schemas/ instead"); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index 1992b01788f..3a5bb3dc1b3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -11,10 +11,9 @@ import java.util.List; */ public class ApplicationPackageUtils { - public static String generateSearchDefinition(String name, String field1, String field2) { - return "" + - "search " + name + "{" + - " document " + name + "{" + + public static String generateSchema(String name, String field1, String field2) { + return "schema " + name + " {" + + " document " + name + " {" + " field " + field1 + " type string {\n" + " indexing: index | summary\n" + " summary: dynamic\n" + @@ -51,7 +50,7 @@ public class ApplicationPackageUtils { List<String> sds = new ArrayList<>(); int i = 0; for (String sdName : sdNames) { - sds.add(generateSearchDefinition(sdName, "f" + (i + 1), "f" + (i + 2))); + sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2))); i = i + 2; } return sds; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java index 351b080eb5b..565e4c7c076 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -66,7 +66,7 @@ public class VespaModelCreatorWithFilePkg { // is constructed in a special way and cannot always be validated in // this step for unit tests) ValidationParameters validationParameters = new ValidationParameters(IgnoreValidationErrors.TRUE, FailOnIncompatibleChange.TRUE, CheckRouting.FALSE); - Validation.validate(model, validationParameters, deployState); + new Validation().validate(model, validationParameters, deployState); return model; } catch (Exception e) { throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index 66dc63bbb02..d777fefbbe5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -75,7 +75,7 @@ public class VespaModelCreatorWithMockPkg { // is constructed in a special way and cannot always be validated in // this step for unit tests) ValidationParameters validationParameters = new ValidationParameters(CheckRouting.FALSE); - configChangeActions = Validation.validate(model, validationParameters, deployState); + configChangeActions = new Validation().validate(model, validationParameters, deployState); } return model; } catch (Exception e) { |