diff options
Diffstat (limited to 'config-model')
94 files changed, 664 insertions, 545 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 061ad42e028..364dd1742ae 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -65,8 +65,8 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra * Creates and initializes a new Vespa from the service config file * in the given application directory. * - * @param parent The parent, usually VespaModel - * @param name The name, used as configId + * @param parent the parent, usually VespaModel + * @param name the name, used as configId * @param documentModel DocumentModel to serve global document config from. */ public ApplicationConfigProducerRoot(AbstractConfigProducer parent, String name, DocumentModel documentModel, Version vespaVersion, ApplicationId applicationId) { diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 557df4bd106..100b5ec5a24 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -11,6 +11,16 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.model.graph.ModelNode; import com.yahoo.config.model.provision.HostsXmlProvisioner; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; @@ -29,7 +39,6 @@ import java.io.IOException; import java.io.Reader; import java.io.Serializable; import java.io.StringReader; -import java.util.*; import java.util.logging.Logger; /** diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 1d0541b67d1..68924dde3e1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -22,7 +22,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; -import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.model.provision .SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; @@ -460,7 +460,7 @@ public class DeployState implements ConfigDefinitionStore { private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles, ValidationParameters validationParameters) { - Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); + Collection<NamedReader> readers = applicationPackage.getSchemas(); Map<String, String> names = new LinkedHashMap<>(); SearchBuilder builder = new SearchBuilder(applicationPackage, logger, properties, rankProfileRegistry, queryProfiles.getRegistry()); for (NamedReader reader : readers) { @@ -470,14 +470,14 @@ public class DeployState implements ConfigDefinitionStore { String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); names.put(topLevelName, sdName); if ( ! sdName.equals(topLevelName)) { - throw new IllegalArgumentException("Schema definition file name ('" + sdName + "') and name of " + + throw new IllegalArgumentException("Schema file name ('" + sdName + "') and name of " + "top level element ('" + topLevelName + "') are not equal for file '" + readerName + "'"); } } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse sd file '" + reader.getName() + "'", e); + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); } catch (IOException e) { - throw new IllegalArgumentException("Could not read sd file '" + reader.getName() + "'", e); + throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); } finally { closeIgnoreException(reader.getReader()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2a530b78b86..2927df57bc1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import java.net.URI; +import java.security.cert.X509Certificate; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -50,7 +51,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useAccessControlTlsHandshakeClientAuth; private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean useBucketExecutorForPruneRemoved; private boolean enableFeedBlockInDistributor = true; private boolean useExternalRankExpression = false; private int clusterControllerMaxHeapSizeInMb = 128; @@ -59,7 +59,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int numDistributorStripes = 0; + private int maxConcurrentMergesPerNode = 16; + private int maxMergeQueueSize = 1024; private boolean allowDisableMtls = true; + private List<X509Certificate> operatorCertificates = Collections.emptyList(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -90,7 +93,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; } @@ -99,8 +101,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public int numDistributorStripes() { return numDistributorStripes; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } + @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; } @Override public boolean distributeExternalRankExpressions() { return useExternalRankExpression; } + @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } + @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } public TestProperties useExternalRankExpression(boolean value) { useExternalRankExpression = value; @@ -133,6 +138,15 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMaxConcurrentMergesPerNode(int maxConcurrentMergesPerNode) { + this.maxConcurrentMergesPerNode = maxConcurrentMergesPerNode; + return this; + } + public TestProperties setMaxMergeQueueSize(int maxMergeQueueSize) { + this.maxMergeQueueSize = maxMergeQueueSize; + return this; + } + public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -198,11 +212,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties useBucketExecutorForPruneRemoved(boolean enabled) { - useBucketExecutorForPruneRemoved = enabled; - return this; - } - public TestProperties enableFeedBlockInDistributor(boolean enabled) { enableFeedBlockInDistributor = enabled; return this; @@ -243,6 +252,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setOperatorCertificates(List<X509Certificate> operatorCertificates) { + this.operatorCertificates = List.copyOf(operatorCertificates); + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index e9fe0824f30..411a37bb70a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -115,7 +115,7 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> getSearchDefinitions() { + public List<NamedReader> getSchemas() { ArrayList<NamedReader> readers = new ArrayList<>(); SearchBuilder searchBuilder = new SearchBuilder(this, new BaseDeployLogger(), @@ -134,11 +134,6 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> searchDefinitionContents() { - return new ArrayList<>(); - } - - @Override public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { return Collections.emptyMap(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 39ab443554c..68e03ecb188 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -73,12 +73,10 @@ public class RankProfile implements Cloneable { protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); /** The ranking expression to be used for first phase */ - private RankingExpression firstPhaseRanking = null; + private RankingExpressionFunction firstPhaseRanking = null; /** The ranking expression to be used for second phase */ - private RankingExpression secondPhaseRanking = null; - - private Set<String> externalFileExpressions = new HashSet<>(); + private RankingExpressionFunction secondPhaseRanking = null; /** Number of hits to be reranked in second phase, -1 means use default */ private int rerankCount = -1; @@ -128,8 +126,8 @@ public class RankProfile implements Cloneable { /** * Creates a new rank profile for a particular search definition * - * @param name the name of the new profile - * @param search the search definition owning this profile + * @param name the name of the new profile + * @param search the search definition owning this profile * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing * and looking up rank profiles. */ @@ -169,10 +167,6 @@ public class RankProfile implements Cloneable { return search != null ? search.rankingConstants() : model.rankingConstants(); } - public RankExpressionFiles rankExpressionFiles() { - return search != null ? search.rankExpressionFiles() : model.rankExpressionFiles(); - } - public Map<String, OnnxModel> onnxModels() { return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); } @@ -198,9 +192,7 @@ public class RankProfile implements Cloneable { } /** Returns the name of the profile this one inherits, or null if none is inherited */ - public String getInheritedName() { - return inheritedName; - } + public String getInheritedName() { return inheritedName; } /** Returns the inherited rank profile, or null if there is none */ public RankProfile getInherited() { @@ -243,7 +235,7 @@ public class RankProfile implements Cloneable { public MatchPhaseSettings getMatchPhaseSettings() { MatchPhaseSettings settings = this.matchPhaseSettings; - if (settings != null ) return settings; + if (settings != null) return settings; if (getInherited() != null) return getInherited().getMatchPhaseSettings(); return null; } @@ -264,7 +256,7 @@ public class RankProfile implements Cloneable { * @return the rank setting found, or null. */ RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { - for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext();) { + for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext(); ) { RankSetting setting = i.next(); if (setting.getFieldName().equals(field) && setting.getType().equals(type)) { @@ -369,55 +361,26 @@ public class RankProfile implements Cloneable { * Returns null if no expression is set. */ public RankingExpression getFirstPhaseRanking() { - if (firstPhaseRanking != null) return firstPhaseRanking; - if (getInherited() != null) return getInherited().getFirstPhaseRanking(); - return null; - } - - void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking = rankingExpression; - } - - public String getUniqueExpressionName(String name) { - return getName() + "_" + name; - } - public String getFirstPhaseFile() { - String name = FIRST_PHASE; - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if ((firstPhaseRanking == null) && (getInherited() != null)) { - return getInherited().getFirstPhaseFile(); - } - return null; + RankingExpressionFunction function = getFirstPhase(); + if (function == null) return null; + return function.function.getBody(); } - public String getSecondPhaseFile() { - String name = SECOND_PHASE; - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if ((secondPhaseRanking == null) && (getInherited() != null)) { - return getInherited().getSecondPhaseFile(); - } + public RankingExpressionFunction getFirstPhase() { + if (firstPhaseRanking != null) return firstPhaseRanking; + RankProfile inherited = getInherited(); + if (inherited != null) return inherited.getFirstPhase(); return null; } - public String getExpressionFile(String name) { - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if (getInherited() != null) { - return getInherited().getExpressionFile(name); - } - return null; + void setFirstPhaseRanking(RankingExpression rankingExpression) { + this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, Collections.emptyList(), rankingExpression), false); } public void setFirstPhaseRanking(String expression) { try { - this.firstPhaseRanking = parseRankingExpression(FIRST_PHASE, expression); - } - catch (ParseException e) { + firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, Collections.emptyList(), expression), false); + } catch (ParseException e) { throw new IllegalArgumentException("Illegal first phase ranking function", e); } } @@ -427,14 +390,21 @@ public class RankProfile implements Cloneable { * Returns null if no expression is set. */ public RankingExpression getSecondPhaseRanking() { + RankingExpressionFunction function = getSecondPhase(); + if (function == null) return null; + return function.function().getBody(); + } + + public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - if (getInherited() != null) return getInherited().getSecondPhaseRanking(); + RankProfile inherited = getInherited(); + if (inherited != null) return inherited.getSecondPhase(); return null; } public void setSecondPhaseRanking(String expression) { try { - this.secondPhaseRanking = parseRankingExpression(SECOND_PHASE, expression); + secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, Collections.emptyList(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal second phase ranking function", e); @@ -603,7 +573,7 @@ public class RankProfile implements Cloneable { /** Adds a function */ public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { - addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); + addFunction(parseRankingExpression(name, arguments, expression), inline); } catch (ParseException e) { throw new IllegalArgumentException("Could not parse function '" + name + "'", e); @@ -691,20 +661,20 @@ public class RankProfile implements Cloneable { return retval; } - private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException { + private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { if (expression.trim().length() == 0) - throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + "."); + throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + name + "."); - try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) { - return new RankingExpression(expressionName, rankingExpressionReader); + try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { + return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + - "' in " + getName()+ ", " + expressionName + "."); + "' in " + getName()+ ", " + name + "."); throw (ParseException)exception.initCause(e); } catch (IOException e) { - throw new RuntimeException("IOException parsing ranking expression '" + expressionName + "'"); + throw new RuntimeException("IOException parsing ranking expression '" + name + "'"); } } @@ -725,10 +695,6 @@ public class RankProfile implements Cloneable { throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file + "' in subdirectory, which is not supported."); - if (search.getDeployProperties().featureFlags().distributeExternalRankExpressions()) { - rankExpressionFiles().add(new RankExpressionFile(getUniqueExpressionName(expName), fileName), search.getDeployLogger()); - externalFileExpressions.add(expName); - } return search.getRankingExpression(fileName); } @@ -777,17 +743,17 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> inlineFunctions = compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); // Function compiling second pass: compile all functions and insert previously compiled inline functions + // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) { - if (constants.get(functionEntry.getKey()) != null) + if (constants.containsKey(functionEntry.getKey())) throw new IllegalArgumentException("Cannot have both a constant and function named '" + functionEntry.getKey() + "'"); } @@ -811,15 +777,15 @@ public class RankProfile implements Cloneable { // A straightforward iteration will either miss those functions, or may cause a ConcurrentModificationException while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) { RankingExpressionFunction rankingExpressionFunction = entry.getValue(); - RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, featureTypes, + RankingExpressionFunction compiled = compile(rankingExpressionFunction, queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withExpression(compiled)); + compiledFunctions.put(entry.getKey(), compiled); } return compiledFunctions; } - private Map.Entry<String, RankingExpressionFunction> findUncompiledFunction(Map<String, RankingExpressionFunction> functions, - Set<String> compiledFunctionNames) { + private static Map.Entry<String, RankingExpressionFunction> findUncompiledFunction(Map<String, RankingExpressionFunction> functions, + Set<String> compiledFunctionNames) { for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) { if ( ! compiledFunctionNames.contains(entry.getKey())) return entry; @@ -827,25 +793,25 @@ public class RankProfile implements Cloneable { return null; } - private RankingExpression compile(RankingExpression expression, + private RankingExpressionFunction compile(RankingExpressionFunction function, QueryProfileRegistry queryProfiles, Map<Reference, TensorType> featureTypes, ImportedMlModels importedModels, Map<String, Value> constants, Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { - if (expression == null) return null; + if (function == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, featureTypes, importedModels, constants, inlineFunctions); - expression = expressionTransforms.transform(expression, context); + RankingExpression expression = expressionTransforms.transform(function.function().getBody(), context); for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); } - return expression; + return function.withExpression(expression); } /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 41ac1e17d93..6b589a22de5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -55,15 +55,18 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, + AttributeFields attributeFields, ModelContext.Properties deployProperties) { this.name = rankProfile.getName(); - compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields, deployProperties).derive()); + compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), + attributeFields, deployProperties).derive()); } /** * Only for testing */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, + ImportedMlModels importedModels, AttributeFields attributeFields) { this(rankProfile, queryProfiles, importedModels, attributeFields, new TestProperties()); } @@ -120,61 +123,74 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private static class Deriver { - /** - * The field rank settings of this profile - */ - private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); - - private final RankProfile rankProfile; - private RankingExpression firstPhaseRanking = null; - private RankingExpression secondPhaseRanking = null; - - private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>(); - - private Set<ReferenceNode> rankFeatures = new LinkedHashSet<>(); - - private List<RankProfile.RankProperty> rankProperties = new ArrayList<>(); + private final Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); + private final Set<ReferenceNode> summaryFeatures; + private final Set<ReferenceNode> rankFeatures; + private final List<RankProfile.RankProperty> rankProperties; /** * Rank properties for weight settings to make these available to feature executors */ - private List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); - - private boolean ignoreDefaultRankFeatures = false; - - private RankProfile.MatchPhaseSettings matchPhaseSettings = null; - - private int rerankCount = -1; - private int keepRankCount = -1; - private int numThreadsPerSearch = -1; - private int minHitsPerThread = -1; - private int numSearchPartitions = -1; - private double termwiseLimit = 1.0; - private double rankScoreDropLimit = -Double.MAX_VALUE; + private final List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); + + private final boolean ignoreDefaultRankFeatures; + private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final int rerankCount; + private final int keepRankCount; + private final int numThreadsPerSearch; + private final int minHitsPerThread; + private final int numSearchPartitions; + private final double termwiseLimit; + private final double rankScoreDropLimit; /** * The rank type definitions used to derive settings for the native rank features */ private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); - private final Map<String, String> attributeTypes; private final Map<String, String> queryFeatureTypes; - private final boolean useExternalExpressionFiles; + private final Set<String> filterFields = new java.util.LinkedHashSet<>(); - private Set<String> filterFields = new java.util.LinkedHashSet<>(); + private RankingExpression firstPhaseRanking; + private RankingExpression secondPhaseRanking; /** * Creates a raw rank profile from the given rank profile */ - Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - AttributeFields attributeFields, ModelContext.Properties deployProperties) + Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.rankProfile = rankProfile; - RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); - useExternalExpressionFiles = deployProperties.featureFlags().useExternalRankExpressions(); - deriveRankingFeatures(compiled, deployProperties); + firstPhaseRanking = compiled.getFirstPhaseRanking(); + secondPhaseRanking = compiled.getSecondPhaseRanking(); + summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures()); + rankFeatures = compiled.getRankFeatures(); + rerankCount = compiled.getRerankCount(); + matchPhaseSettings = compiled.getMatchPhaseSettings(); + numThreadsPerSearch = compiled.getNumThreadsPerSearch(); + minHitsPerThread = compiled.getMinHitsPerThread(); + numSearchPartitions = compiled.getNumSearchPartitions(); + termwiseLimit = compiled.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); + keepRankCount = compiled.getKeepRankCount(); + rankScoreDropLimit = compiled.getRankScoreDropLimit(); + ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); + rankProperties = new ArrayList<>(compiled.getRankProperties()); + + Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); + List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); + Map<String, String> functionProperties = new LinkedHashMap<>(); + SerializationContext functionSerializationContext = new SerializationContext(functionExpressions); + + if (firstPhaseRanking != null) { + functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); + } + if (secondPhaseRanking != null) { + functionProperties.putAll(secondPhaseRanking.getRankProperties(functionSerializationContext)); + } + + derivePropertiesAndSummaryFeaturesFromFunctions(functions, functionProperties, functionSerializationContext); + deriveOnnxModelFunctionsAndSummaryFeatures(compiled); + deriveRankTypeSetting(compiled, attributeFields); deriveFilterFields(compiled); deriveWeightProperties(compiled); @@ -184,44 +200,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer { filterFields.addAll(rp.allFilterFields()); } - private void deriveRankingFeatures(RankProfile rankProfile, ModelContext.Properties deployProperties) { - firstPhaseRanking = rankProfile.getFirstPhaseRanking(); - secondPhaseRanking = rankProfile.getSecondPhaseRanking(); - summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures()); - rankFeatures = rankProfile.getRankFeatures(); - rerankCount = rankProfile.getRerankCount(); - matchPhaseSettings = rankProfile.getMatchPhaseSettings(); - numThreadsPerSearch = rankProfile.getNumThreadsPerSearch(); - minHitsPerThread = rankProfile.getMinHitsPerThread(); - numSearchPartitions = rankProfile.getNumSearchPartitions(); - termwiseLimit = rankProfile.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); - keepRankCount = rankProfile.getKeepRankCount(); - rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); - ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); - rankProperties = new ArrayList<>(rankProfile.getRankProperties()); - derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions()); - deriveOnnxModelFunctionsAndSummaryFeatures(rankProfile); - } - - private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) { + private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions, + Map<String, String> functionProperties, + SerializationContext functionContext) { if (functions.isEmpty()) return; - List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); - Map<String, String> functionProperties = new LinkedHashMap<>(); - - if (firstPhaseRanking != null) { - functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions)); - } - if (secondPhaseRanking != null) { - functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions)); - } - - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); - replaceFunctionSummaryFeatures(context); + replaceFunctionSummaryFeatures(functionContext); // First phase, second phase and summary features should add all required functions to the context. // However, we need to add any functions not referenced in those anyway for model-evaluation. - deriveFunctionProperties(functions, functionExpressions, functionProperties); + deriveFunctionProperties(functions, functionProperties, functionContext); for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); @@ -229,15 +217,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, - List<ExpressionFunction> functionExpressions, - Map<String, String> functionProperties) { - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); + Map<String, String> functionProperties, + SerializationContext context) { for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { - if (useExternalExpressionFiles && rankProfile.getExpressionFile(e.getKey()) != null) continue; String propertyName = RankingExpression.propertyName(e.getKey()); if (context.serializedFunctions().containsKey(propertyName)) continue; - String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + String expressionString = e.getValue().function().getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) @@ -259,7 +245,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { ExpressionFunction function = context.getFunction(referenceNode.getName()); if (function != null) { String propertyName = RankingExpression.propertyName(referenceNode.getName()); - String expressionString = function.getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + String expressionString = function.getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); @@ -355,8 +341,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, rankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE)); - properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, rankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, RankProfile.FIRST_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, RankProfile.SECOND_PHASE)); for (FieldRankSettings settings : fieldRankSettings.values()) { properties.addAll(settings.deriveRankProperties()); } @@ -408,9 +394,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (ignoreDefaultRankFeatures) { properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } - Iterator filterFieldsIterator = filterFields.iterator(); - while (filterFieldsIterator.hasNext()) { - String fieldName = (String) filterFieldsIterator.next(); + for (String fieldName : filterFields) { properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); } for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { @@ -423,7 +407,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return properties; } - private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String fileName, String phase) { + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { List<Pair<String, String>> properties = new ArrayList<>(); if (expression == null) return properties; @@ -431,9 +415,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if ("".equals(name)) name = phase; - if (useExternalExpressionFiles && (fileName != null)) { - properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + rankProfile.getUniqueExpressionName(name) + ")")); - } else if (expression.getRoot() instanceof ReferenceNode) { + if (expression.getRoot() instanceof ReferenceNode) { properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java index d15db6b4a55..800bf73cdbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -18,16 +19,20 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr private final ApplicationId applicationId; private final Zone zone; + private final boolean requireConnectivityCheck; /** * Constructs a new ConfigSentinel for the given host. * * @param host Physical host on which to run. */ - public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone) { + public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone, + ModelContext.FeatureFlags featureFlags) + { super(host, "sentinel"); this.applicationId = applicationId; this.zone = zone; + this.requireConnectivityCheck = featureFlags.requireConnectivityCheck(); portsMeta.on(0).tag("rpc").tag("admin"); portsMeta.on(1).tag("telnet").tag("interactive").tag("http").tag("state"); setProp("clustertype", "hosts"); @@ -75,6 +80,19 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr builder.service(getServiceConfig(s)); } } + builder.connectivity(getConnectivityConfig(requireConnectivityCheck)); + } + + private SentinelConfig.Connectivity.Builder getConnectivityConfig(boolean enable) { + var builder = new SentinelConfig.Connectivity.Builder(); + if (enable) { + builder.maxBadOutPercent(60); + builder.maxBadReverseCount(3); + } else { + builder.maxBadOutPercent(100); + builder.maxBadReverseCount(Integer.MAX_VALUE); + } + return builder; } private SentinelConfig.Application.Builder getApplicationConfig() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 09bbd446803..53f42866d8d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -47,16 +47,15 @@ public class HostSystem extends AbstractConfigProducer<Host> { void checkName(String hostname) { // Give a warning if the host does not exist try { - @SuppressWarnings("unused") - Object ignore = java.net.InetAddress.getByName(hostname); + var inetAddr = java.net.InetAddress.getByName(hostname); + String canonical = inetAddr.getCanonicalHostName(); + if (! hostname.equals(canonical)) { + deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + + "since it differs from its canonical hostname '" + canonical + "' (check DNS and /etc/hosts)."); + } } catch (UnknownHostException e) { deployLogger.logApplicationPackage(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } - if (! hostname.contains(".")) { - deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + - "since it is not a canonical hostname. " + - "Disregard this warning when testing in a Docker container."); - } } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index e080ce43730..a2a6ada9093 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; @@ -242,7 +243,8 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } private void addCommonServices(HostResource host, DeployState deployState) { - addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone()); + addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone(), + deployState.featureFlags()); addLogd(deployState.getDeployLogger(), host); addConfigProxy(deployState.getDeployLogger(), host); addFileDistribution(host); @@ -262,8 +264,10 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } } - private void addConfigSentinel(DeployLogger deployLogger, HostResource host, ApplicationId applicationId, Zone zone) { - ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone); + private void addConfigSentinel(DeployLogger deployLogger, HostResource host, + ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) + { + ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone, featureFlags); addAndInitializeService(deployLogger, host, configSentinel); host.getHost().setConfigSentinel(configSentinel); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 6f467b21535..e2aa325c380 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -19,19 +19,31 @@ public class AutoscalingMetrics { private static MetricSet create() { List<String> metrics = new ArrayList<>(); + metrics.add("cpu.util"); - metrics.add("mem.util"); - metrics.add("disk.util"); + + // Memory util + metrics.add("mem.util"); // node level - default + metrics.add("content.proton.resource_usage.memory.average"); // better for content as it is the basis for blocking + + // Disk util + metrics.add("disk.util"); // node level -default + metrics.add("content.proton.resource_usage.disk.average"); // better for content as it is the basis for blocking + metrics.add("application_generation"); + metrics.add("in_service"); + // Query rate metrics.add("queries.rate"); // container metrics.add("content.proton.documentdb.matching.queries.rate"); // content + // Write rate metrics.add("feed.http-requests.rate"); // container metrics.add("vds.filestor.alldisks.allthreads.put.sum.count.rate"); // content metrics.add("vds.filestor.alldisks.allthreads.remove.sum.count.rate"); // content metrics.add("vds.filestor.alldisks.allthreads.update.sum.count.rate"); // content + return new MetricSet("autoscaling", toMetrics(metrics)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 034bf772ffc..114a3e380ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -256,8 +256,11 @@ public class VespaMetricSet { metrics.add(new Metric("cluster-controller.node-event.count")); metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.last")); + metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.max")); metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.max")); metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.max")); metrics.add(new Metric("cluster-controller.resource_usage.disk_limit.last")); metrics.add(new Metric("cluster-controller.resource_usage.memory_limit.last")); @@ -762,6 +765,15 @@ public class VespaMetricSet { metrics.add(new Metric("vds.bouncer.clock_skew_aborts.count")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.max")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.sum")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.count")); + metrics.add(new Metric("vds.mergethrottler.queuesize.max")); + metrics.add(new Metric("vds.mergethrottler.queuesize.sum")); + metrics.add(new Metric("vds.mergethrottler.queuesize.count")); + metrics.add(new Metric("vds.mergethrottler.bounced_due_to_back_pressure.rate")); + metrics.add(new Metric("vds.mergethrottler.locallyexecutedmerges.ok.rate")); + metrics.add(new Metric("vds.mergethrottler.mergechains.ok.rate")); return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java new file mode 100644 index 00000000000..ffcad391e57 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java @@ -0,0 +1,32 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; + +import java.util.logging.Level; + +/** + * Validates that correct directory is used for schemas + * + * @author hmusum + */ +public class SchemasDirValidator extends Validator { + + public SchemasDirValidator() { + } + + @Override + public void validate(VespaModel model, DeployState deployState) { + ApplicationPackage app = deployState.getApplicationPackage(); + ApplicationFile sdDir = app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR); + if (sdDir.exists() && sdDir.isDirectory()) + deployState.getDeployLogger().logApplicationPackage( + Level.WARNING, + "Directory " + ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative() + + "/ should not be used for schemas, use " + ApplicationPackage.SCHEMAS_DIR.getRelative() + "/ instead"); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index d1c93bcd611..55443d4b260 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -9,7 +9,6 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.container.QrConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.application.validation.change.ClusterSizeReductionValidator; @@ -25,8 +24,6 @@ import com.yahoo.vespa.model.application.validation.change.ResourcesReductionVal import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.Container; import java.time.Instant; import java.util.Arrays; @@ -43,7 +40,6 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; /** * Executor of validators. This defines the right order of validator execution. @@ -63,6 +59,7 @@ public class Validation { new RoutingValidator().validate(model, deployState); new RoutingSelectorValidator().validate(model, deployState); } + new SchemasDirValidator().validate(model, deployState); new ComponentValidator().validate(model, deployState); new SearchDataTypeValidator().validate(model, deployState); new ComplexAttributeFieldsValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index f18f10644ca..fc41da43479 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -35,7 +35,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this(cluster, logType, compressionType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), null, null, isHostedVespa, - capitalize(logType.name()) + "." + clusterName); + capitalize(logType.name()) + "." + clusterName, + queueSize(cluster).orElse(-1), + ((cluster instanceof ApplicationContainerCluster) ? 4*1024*1024 : null)); } private static String capitalize(String name) { @@ -49,7 +51,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL String rotationInterval, Boolean compressOnRotation, boolean isHostedVespa, - String symlinkName) + String symlinkName, + Integer queueSize, + Integer bufferSize) { super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); this.fileNamePattern = fileNamePattern; @@ -58,10 +62,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; this.compressionType = compressionType; - this.queueSize = queueSize(cluster).orElse(-1); - bufferSize = (cluster instanceof ApplicationContainerCluster) - ? 4*1024*1024 - : null; + this.queueSize = (queueSize == null) ? 256 : queueSize; + this.bufferSize = bufferSize; if (fileNamePattern == null) throw new RuntimeException("File name pattern required when configuring access log."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java index 83e06aab703..0b51cd163a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.logging.ConnectionLog; import com.yahoo.container.logging.ConnectionLogConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import java.util.OptionalInt; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index f84d6f0724b..d7812e9b4ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -62,7 +62,9 @@ public class AccessLogBuilder { rotationInterval(spec), compressOnRotation(spec), isHostedVespa, - symlinkName(spec)); + symlinkName(spec), + queueSize(spec), + bufferSize(spec)); } private String symlinkName(Element spec) { @@ -74,6 +76,16 @@ public class AccessLogBuilder { return (compress.isEmpty() ? null : Boolean.parseBoolean(compress)); } + private Integer bufferSize(Element spec) { + String value = spec.getAttribute("bufferSize"); + return (value.isEmpty() ? null : Integer.parseInt(value)); + } + + private Integer queueSize(Element spec) { + String value = spec.getAttribute("queueSize"); + return (value.isEmpty() ? null : Integer.parseInt(value)); + } + private String rotationInterval(Element spec) { return nullIfEmpty(spec.getAttribute("rotationInterval")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 7e0be6e448b..a37d0ef416f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -48,7 +48,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { cluster.addComponent( new AccessLogComponent( cluster, AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, - "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log")); + "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { super.addAccessLogs(deployState, cluster, spec); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index b477587bcac..08ccfe33cd5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -34,6 +34,7 @@ import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -89,6 +90,7 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; import java.net.URI; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -217,7 +219,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if(deployState.isHosted()) { cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath("jdisc-cloud-aws")); } - if (deployState.featureFlags().tenantIamRole()) { + if (deployState.zone().system().isPublic()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); Handler<AbstractConfigProducer<?>> handler = new Handler<>( new ComponentModel("com.yahoo.jdisc.cloud.aws.AwsParameterStoreValidationHandler", null, "jdisc-cloud-aws", null)); @@ -431,6 +433,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // If the deployment contains certificate/private key reference, setup TLS port HostedSslConnectorFactory connectorFactory; + boolean enableHttp2 = deployState.featureFlags().enableJdiscHttp2(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { @@ -444,7 +447,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .orElse(false); connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState)) : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets, enforceHandshakeClientAuth); } else { connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName); @@ -453,6 +456,19 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { server.addConnector(connectorFactory); } + /* + Return trusted certificates as a PEM encoded string containing the concatenation of + trusted certs from the application package and all operator certificates. + */ + String getTlsClientAuthorities(DeployState deployState) { + List<X509Certificate> trustedCertificates = deployState.tlsClientAuthority() + .map(X509CertificateUtils::certificateListFromPem) + .orElse(Collections.emptyList()); + ArrayList<X509Certificate> x509Certificates = new ArrayList<>(trustedCertificates); + x509Certificates.addAll(deployState.getProperties().operatorCertificates()); + return X509CertificateUtils.toPem(x509Certificates); + } + private static boolean isHostedTenantApplication(ConfigModelContext context) { var deployState = context.getDeployState(); boolean isTesterApplication = deployState.getProperties().applicationId().instance().isTester(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 2bd9cb09aa6..ea52f9689ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -177,7 +177,7 @@ public class Content extends ConfigModel { s.setVespaMallocDebugStackTrace(cluster.getRootGroup().getVespaMallocDebugStackTrace().get()); } } - cluster.prepare(deployState); + cluster.prepare(); } private void setCpuSocketAffinity() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 18580249ddc..51949e78838 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -65,7 +65,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; - private final boolean useBucketExecutorForPruneRemoved; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -210,7 +209,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); - useBucketExecutorForPruneRemoved = featureFlags.useBucketExecutorForPruneRemoved(); } public void setVisibilityDelay(double delay) { @@ -427,7 +425,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.indexing.optimize(feedSequencerType); } - builder.pruneremoveddocuments.usebucketexecutor(useBucketExecutorForPruneRemoved); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 830eced56d3..90cca1494b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -70,7 +70,7 @@ import static java.util.stream.Collectors.toList; * @author mostly somebody unknown * @author bratseth */ -public class ContentCluster extends AbstractConfigProducer implements +public class ContentCluster extends AbstractConfigProducer<AbstractConfigProducer<?>> implements DistributionConfig.Producer, StorDistributionConfig.Producer, StorDistributormanagerConfig.Producer, @@ -94,14 +94,6 @@ public class ContentCluster extends AbstractConfigProducer implements private Integer maxNodesPerMerge; private final Zone zone; - /** - * If multitenant or a cluster controller was explicitly configured in this cluster: - * The cluster controller cluster of this particular content cluster. - * - * Otherwise: null - the cluster controller is shared by all content clusters and part of Admin. - */ - private ClusterControllerContainerCluster clusterControllers; - public enum DistributionMode { LEGACY, STRICT, LOOSE } private DistributionMode distributionMode; @@ -413,16 +405,10 @@ public class ContentCluster extends AbstractConfigProducer implements public ClusterSpec.Id id() { return ClusterSpec.Id.from(clusterId); } - public void prepare(DeployState deployState) { + public void prepare() { search.prepare(); - - if (clusterControllers != null) - clusterControllers.prepare(deployState); } - /** Returns cluster controllers if this is multitenant, null otherwise */ - public ClusterControllerContainerCluster getClusterControllers() { return clusterControllers; } - public DistributionMode getDistributionMode() { if (distributionMode != null) return distributionMode; return getPersistence().getDefaultDistributionMode(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 40a634fbfe8..e89d45e8b83 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -6,6 +6,8 @@ import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import java.util.Optional; + /** * Serves config for stor-server for storage clusters (clusters of storage nodes). */ @@ -14,7 +16,7 @@ public class StorServerProducer implements StorServerConfig.Producer { StorServerProducer build(ModelContext.Properties properties, ModelElement element) { ModelElement tuning = element.child("tuning"); - StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element)); + StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element), properties.featureFlags()); if (tuning == null) return producer; ModelElement merges = tuning.child("merges"); @@ -32,11 +34,15 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer bucketDBStripeBits; private StorServerProducer setMaxMergesPerNode(Integer value) { - maxMergesPerNode = value; + if (value != null) { + maxMergesPerNode = value; + } return this; } private StorServerProducer setMaxQueueSize(Integer value) { - queueSize = value; + if (value != null) { + queueSize = value; + } return this; } private StorServerProducer setBucketDBStripeBits(Integer value) { @@ -44,8 +50,10 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } - public StorServerProducer(String clusterName) { + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; + maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); + queueSize = featureFlags.maxMergeQueueSize(); } @Override diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 39df939f78c..3679e53e257 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -81,7 +81,8 @@ AccessLog = element accesslog { attribute compressOnRotation { xsd:boolean }? & attribute symlinkName { string }? & attribute compressionType { string "gzip" | string "zstd" }? & - + attribute queueSize { xsd:nonNegativeInteger }? & + attribute bufferSize { xsd:nonNegativeInteger }? & attribute rotationInterval { string }? } diff --git a/config-model/src/test/cfg/admin/metricconfig/searchdefinitions/music.sd b/config-model/src/test/cfg/admin/metricconfig/schemas/music.sd index 32e8451d8e2..32e8451d8e2 100644 --- a/config-model/src/test/cfg/admin/metricconfig/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/admin/metricconfig/schemas/music.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/bar.expression b/config-model/src/test/cfg/application/app1/schemas/bar.expression index eed496e6aeb..eed496e6aeb 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/bar.expression +++ b/config-model/src/test/cfg/application/app1/schemas/bar.expression diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/foo.expression b/config-model/src/test/cfg/application/app1/schemas/foo.expression index ce26aa75dcb..ce26aa75dcb 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/foo.expression +++ b/config-model/src/test/cfg/application/app1/schemas/foo.expression diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/laptop.sd b/config-model/src/test/cfg/application/app1/schemas/laptop.sd index 21a76ad605c..21a76ad605c 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/laptop.sd +++ b/config-model/src/test/cfg/application/app1/schemas/laptop.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/music.sd b/config-model/src/test/cfg/application/app1/schemas/music.sd index 693afbd308d..693afbd308d 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/app1/schemas/music.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/pc.sd b/config-model/src/test/cfg/application/app1/schemas/pc.sd index bdc90328cdb..bdc90328cdb 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/pc.sd +++ b/config-model/src/test/cfg/application/app1/schemas/pc.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/product.sd b/config-model/src/test/cfg/application/app1/schemas/product.sd index 588a1d544c6..588a1d544c6 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/product.sd +++ b/config-model/src/test/cfg/application/app1/schemas/product.sd diff --git a/config-model/src/test/cfg/application/app1/searchdefinitions/sock.sd b/config-model/src/test/cfg/application/app1/schemas/sock.sd index 9331c0caf54..9331c0caf54 100644 --- a/config-model/src/test/cfg/application/app1/searchdefinitions/sock.sd +++ b/config-model/src/test/cfg/application/app1/schemas/sock.sd diff --git a/config-model/src/test/cfg/application/app_complicated_deployment_spec/searchdefinitions/music.sd b/config-model/src/test/cfg/application/app_complicated_deployment_spec/schemas/music.sd index 693afbd308d..693afbd308d 100644 --- a/config-model/src/test/cfg/application/app_complicated_deployment_spec/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/app_complicated_deployment_spec/schemas/music.sd diff --git a/config-model/src/test/cfg/application/app_genericservices/searchdefinitions/music.sd b/config-model/src/test/cfg/application/app_genericservices/schemas/music.sd index 693afbd308d..693afbd308d 100644 --- a/config-model/src/test/cfg/application/app_genericservices/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/app_genericservices/schemas/music.sd diff --git a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/mail.sd b/config-model/src/test/cfg/application/app_nohosts/schemas/mail.sd index 8b6a6c44bf1..8b6a6c44bf1 100644 --- a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/mail.sd +++ b/config-model/src/test/cfg/application/app_nohosts/schemas/mail.sd diff --git a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/mailbox.sd b/config-model/src/test/cfg/application/app_nohosts/schemas/mailbox.sd index f98017f401e..f98017f401e 100644 --- a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/mailbox.sd +++ b/config-model/src/test/cfg/application/app_nohosts/schemas/mailbox.sd diff --git a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/message.sd b/config-model/src/test/cfg/application/app_nohosts/schemas/message.sd index 6c27917c656..6c27917c656 100644 --- a/config-model/src/test/cfg/application/app_nohosts/searchdefinitions/message.sd +++ b/config-model/src/test/cfg/application/app_nohosts/schemas/message.sd diff --git a/config-model/src/test/cfg/application/app_qrserverandgw/searchdefinitions/message.sd b/config-model/src/test/cfg/application/app_qrserverandgw/schemas/message.sd index c09b2fb2da2..c09b2fb2da2 100644 --- a/config-model/src/test/cfg/application/app_qrserverandgw/searchdefinitions/message.sd +++ b/config-model/src/test/cfg/application/app_qrserverandgw/schemas/message.sd diff --git a/config-model/src/test/cfg/application/deprecated_features_app/hosts.xml b/config-model/src/test/cfg/application/deprecated_features_app/hosts.xml new file mode 100644 index 00000000000..f0c76482b2f --- /dev/null +++ b/config-model/src/test/cfg/application/deprecated_features_app/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node0</alias> + </host> +</hosts> diff --git a/config-model/src/test/cfg/application/deprecated_features_app/searchdefinitions/message.sd b/config-model/src/test/cfg/application/deprecated_features_app/searchdefinitions/message.sd new file mode 100644 index 00000000000..c97c860a129 --- /dev/null +++ b/config-model/src/test/cfg/application/deprecated_features_app/searchdefinitions/message.sd @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search message { + + document message { + field foo type string { + indexing: summary + } + } +} diff --git a/config-model/src/test/cfg/application/deprecated_features_app/services.xml b/config-model/src/test/cfg/application/deprecated_features_app/services.xml new file mode 100644 index 00000000000..daf03d0c4c3 --- /dev/null +++ b/config-model/src/test/cfg/application/deprecated_features_app/services.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + + <container version="1.0"> + <nodes> + <node hostalias="node0"/> + </nodes> + <document-api/> + <search/> + </container> + + <content version="1.0" id="message"> + <redundancy>1</redundancy> + <documents> + <document type="message" mode="index"/> + </documents> + <nodes> + <node hostalias="node0" distribution-key="0" /> + </nodes> + </content> + +</services> diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/schemas/test.sd index cc73f2daff5..cc73f2daff5 100644 --- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/ml_models/schemas/test.sd diff --git a/config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd b/config-model/src/test/cfg/application/onnx/schemas/test.sd index d49782ddf39..d49782ddf39 100644 --- a/config-model/src/test/cfg/application/onnx/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/onnx/schemas/test.sd diff --git a/config-model/src/test/cfg/application/sdfilenametest/searchdefinitions/notmusic.sd b/config-model/src/test/cfg/application/sdfilenametest/schemas/notmusic.sd index 77307c04843..77307c04843 100644 --- a/config-model/src/test/cfg/application/sdfilenametest/searchdefinitions/notmusic.sd +++ b/config-model/src/test/cfg/application/sdfilenametest/schemas/notmusic.sd diff --git a/config-model/src/test/cfg/application/validation/document_references_validation/searchdefinitions/ad.sd b/config-model/src/test/cfg/application/validation/document_references_validation/schemas/ad.sd index 1e85ab70357..1e85ab70357 100644 --- a/config-model/src/test/cfg/application/validation/document_references_validation/searchdefinitions/ad.sd +++ b/config-model/src/test/cfg/application/validation/document_references_validation/schemas/ad.sd diff --git a/config-model/src/test/cfg/application/validation/document_references_validation/searchdefinitions/campaign.sd b/config-model/src/test/cfg/application/validation/document_references_validation/schemas/campaign.sd index 9e208e397b9..9e208e397b9 100644 --- a/config-model/src/test/cfg/application/validation/document_references_validation/searchdefinitions/campaign.sd +++ b/config-model/src/test/cfg/application/validation/document_references_validation/schemas/campaign.sd diff --git a/config-model/src/test/cfg/application/validation/global_distribution_validation/searchdefinitions/parent.sd b/config-model/src/test/cfg/application/validation/global_distribution_validation/schemas/parent.sd index 8decebc6680..8decebc6680 100644 --- a/config-model/src/test/cfg/application/validation/global_distribution_validation/searchdefinitions/parent.sd +++ b/config-model/src/test/cfg/application/validation/global_distribution_validation/schemas/parent.sd diff --git a/config-model/src/test/cfg/application/validation/global_distribution_validation/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/global_distribution_validation/schemas/simple.sd index b901fa6e82a..b901fa6e82a 100644 --- a/config-model/src/test/cfg/application/validation/global_distribution_validation/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/global_distribution_validation/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/index_struct/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/index_struct/schemas/simple.sd index a27a2d074d2..a27a2d074d2 100644 --- a/config-model/src/test/cfg/application/validation/index_struct/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/index_struct/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/prefix/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/prefix/schemas/simple.sd index 60ce940fd7d..60ce940fd7d 100644 --- a/config-model/src/test/cfg/application/validation/prefix/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/prefix/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/prefix_index/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/prefix_index/schemas/simple.sd index bf70a1662c6..bf70a1662c6 100644 --- a/config-model/src/test/cfg/application/validation/prefix_index/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/prefix_index/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/prefix_index_and_attribute/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/prefix_index_and_attribute/schemas/simple.sd index 41412f4ef4f..41412f4ef4f 100644 --- a/config-model/src/test/cfg/application/validation/prefix_index_and_attribute/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/prefix_index_and_attribute/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/prefix_streaming/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/prefix_streaming/schemas/simple.sd index bf70a1662c6..bf70a1662c6 100644 --- a/config-model/src/test/cfg/application/validation/prefix_streaming/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/prefix_streaming/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_fail/schemas/simple.sd index 8b782a01946..8b782a01946 100644 --- a/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_ok/schemas/simple.sd index 8b782a01946..8b782a01946 100644 --- a/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/parent.sd b/config-model/src/test/cfg/application/validation/search_alltypes/schemas/parent.sd index 8decebc6680..8decebc6680 100644 --- a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/parent.sd +++ b/config-model/src/test/cfg/application/validation/search_alltypes/schemas/parent.sd diff --git a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/search_alltypes/schemas/simple.sd index 242a7493df3..242a7493df3 100644 --- a/config-model/src/test/cfg/application/validation/search_alltypes/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/search_alltypes/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/search_empty_content/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/search_empty_content/schemas/simple.sd index 51512363fbc..51512363fbc 100644 --- a/config-model/src/test/cfg/application/validation/search_empty_content/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/search_empty_content/schemas/simple.sd diff --git a/config-model/src/test/cfg/application/validation/search_struct/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/search_struct/schemas/simple.sd index 0ab072111c5..0ab072111c5 100644 --- a/config-model/src/test/cfg/application/validation/search_struct/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/search_struct/schemas/simple.sd diff --git a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/mobile.sd b/config-model/src/test/cfg/routing/content_two_clusters/schemas/mobile.sd index 3abc9238e8a..3abc9238e8a 100644 --- a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/mobile.sd +++ b/config-model/src/test/cfg/routing/content_two_clusters/schemas/mobile.sd diff --git a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/music.sd b/config-model/src/test/cfg/routing/content_two_clusters/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/routing/content_two_clusters/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/routing/content_two_clusters/schemas/music.sd diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/searchdefinitions/music.sd b/config-model/src/test/cfg/routing/contentsimpleconfig/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/routing/contentsimpleconfig/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/schemas/music.sd diff --git a/config-model/src/test/cfg/routing/replacehop/searchdefinitions/music.sd b/config-model/src/test/cfg/routing/replacehop/schemas/music.sd index f9816d0d392..f9816d0d392 100755 --- a/config-model/src/test/cfg/routing/replacehop/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/routing/replacehop/schemas/music.sd diff --git a/config-model/src/test/cfg/routing/replaceroute/searchdefinitions/music.sd b/config-model/src/test/cfg/routing/replaceroute/schemas/music.sd index f9816d0d392..f9816d0d392 100755 --- a/config-model/src/test/cfg/routing/replaceroute/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/routing/replaceroute/schemas/music.sd diff --git a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTData.sd b/config-model/src/test/cfg/search/data/travel/schemas/TTData.sd index 67013239499..67013239499 100644 --- a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTData.sd +++ b/config-model/src/test/cfg/search/data/travel/schemas/TTData.sd diff --git a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd b/config-model/src/test/cfg/search/data/travel/schemas/TTEdge.sd index e6707345235..e6707345235 100644 --- a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd +++ b/config-model/src/test/cfg/search/data/travel/schemas/TTEdge.sd diff --git a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd b/config-model/src/test/cfg/search/data/travel/schemas/TTPOI.sd index c39ef03add5..c39ef03add5 100644 --- a/config-model/src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd +++ b/config-model/src/test/cfg/search/data/travel/schemas/TTPOI.sd diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/base.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/base.sd index 22944f61997..22944f61997 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/base.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/base.sd diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/left.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/left.sd index c72b349bb4c..c72b349bb4c 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/left.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/left.sd diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/music.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/music.sd diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/right.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/right.sd index eb0a4cf8021..eb0a4cf8021 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/searchdefinitions/right.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/right.sd diff --git a/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/searchdefinitions/music.sd b/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/schemas/music.sd diff --git a/config-model/src/test/cfg/storage/clustercontroller_advanced/searchdefinitions/music.sd b/config-model/src/test/cfg/storage/clustercontroller_advanced/schemas/music.sd index da9e2d14c55..da9e2d14c55 100644 --- a/config-model/src/test/cfg/storage/clustercontroller_advanced/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/storage/clustercontroller_advanced/schemas/music.sd diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg index 9629ad863d4..c6263d658c5 100644 --- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg @@ -124,12 +124,12 @@ rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" rankprofile[].fef.property[].value "1" rankprofile[].name "macros" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + rankBoost)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -141,16 +141,16 @@ rankprofile[].fef.property[].value "fieldMatch(title)" rankprofile[].name "macros2" rankprofile[].fef.property[].name "foo" rankprofile[].fef.property[].value "some, list" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" -rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" +rankprofile[].fef.property[].value "4 * (match + match)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness" +rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" +rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" -rankprofile[].fef.property[].value "4 * (match + match)" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "classicRank" rankprofile[].fef.property[].name "vespa.rank.secondphase" @@ -174,16 +174,16 @@ rankprofile[].fef.property[].value "rankingExpression(matches(title,rankingExpre rankprofile[].name "macros-inherited" rankprofile[].fef.property[].name "foo" rankprofile[].fef.property[].value "some, list" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" -rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + match)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" +rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" +rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -199,16 +199,16 @@ rankprofile[].fef.property[].value "rankingExpression(mysummaryfeature)" rankprofile[].name "macros-inherited2" rankprofile[].fef.property[].name "foo" rankprofile[].fef.property[].value "some, list" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" -rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" -rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" +rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" +rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + match)" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" +rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" +rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -224,12 +224,12 @@ rankprofile[].fef.property[].value "rankingExpression(mysummaryfeature)" rankprofile[].name "macros-inherited3" rankprofile[].fef.property[].name "foo" rankprofile[].fef.property[].value "some, list" -rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" -rankprofile[].fef.property[].value "4 * (var1 + var2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" -rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" +rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" +rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" +rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "vespa.rank.firstphase" @@ -254,20 +254,20 @@ rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "40000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "40000 * rankingExpression(m2) * rankingExpression(m4)" rankprofile[].name "macros-refering-macros-inherited" rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" -rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness" +rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness" rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript" rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "3000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(m4)" rankprofile[].name "macros-refering-macros-inherited2" rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" @@ -285,7 +285,7 @@ rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" -rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness" +rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness" rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript" rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" rankprofile[].fef.property[].name "rankingExpression(m5).rankingScript" @@ -293,4 +293,4 @@ rankprofile[].fef.property[].value "if (isNan(attribute(glmpfw)) == 1, rankingEx rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "3000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(m4)" diff --git a/config-model/src/test/derived/rankexpression/rankexpression.sd b/config-model/src/test/derived/rankexpression/rankexpression.sd index d3e0057cfe1..20f9c7a9160 100644 --- a/config-model/src/test/derived/rankexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankexpression/rankexpression.sd @@ -276,7 +276,7 @@ search rankexpression { second-phase { expression { - 40000 * m2 + 40000 * m2 * m4 } } @@ -291,14 +291,9 @@ search rankexpression { ) } } - macro m4() { - expression { - 701 * fieldMatch(title).completeness - } - } second-phase { expression { - 3000 * m2 + 3000 * m2 * m4 } } } @@ -324,4 +319,3 @@ search rankexpression { } - diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 75cb41be13f..8f751631fb5 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -46,7 +46,7 @@ import static org.junit.Assert.fail; public class ApplicationDeployTest { private static final String TESTDIR = "src/test/cfg/application/"; - private static final String TESTSDDIR = TESTDIR + "app1/searchdefinitions/"; + private static final String TEST_SCHEMAS_DIR = TESTDIR + "app1/schemas/"; @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -54,7 +54,7 @@ public class ApplicationDeployTest { @Test public void testVespaModel() throws SAXException, IOException { ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "app1"); - VespaModel model = new VespaModel(tester.app()); + new VespaModel(tester.app()); List<NamedSchema> schemas = tester.getSchemas(); assertEquals(schemas.size(), 5); for (NamedSchema searchDefinition : schemas) { @@ -73,11 +73,11 @@ public class ApplicationDeployTest { fail(); } } - assertEquals(Set.of(new File(TESTSDDIR + "laptop.sd"), - new File(TESTSDDIR + "music.sd"), - new File(TESTSDDIR + "pc.sd"), - new File(TESTSDDIR + "product.sd"), - new File(TESTSDDIR + "sock.sd")), + assertEquals(Set.of(new File(TEST_SCHEMAS_DIR + "laptop.sd"), + new File(TEST_SCHEMAS_DIR + "music.sd"), + new File(TEST_SCHEMAS_DIR + "pc.sd"), + new File(TEST_SCHEMAS_DIR + "product.sd"), + new File(TEST_SCHEMAS_DIR + "sock.sd")), new HashSet<>(tester.app().getSearchDefinitionFiles())); List<FilesApplicationPackage.Component> components = tester.app().getComponents(); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 2b72420614d..b0ddadf11bd 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -38,7 +38,6 @@ import com.yahoo.yolean.Exceptions; import org.junit.Test; import java.io.StringReader; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -50,6 +49,7 @@ import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; import static com.yahoo.vespa.model.search.NodeResourcesTuning.reservedMemoryGb; +import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -502,7 +502,6 @@ public class ModelProvisioningTest { // Check content clusters ContentCluster cluster = model.getContentClusters().get("bar"); - assertNull("No own cluster controllers when hosted", cluster.getClusterControllers()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(9, cluster.getRootGroup().getSubgroups().size()); assertEquals("0", cluster.getRootGroup().getSubgroups().get(0).getIndex()); @@ -536,7 +535,6 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getConfigId(), is("bar/storage/26")); cluster = model.getContentClusters().get("baz"); - assertNull("No own cluster controllers when hosted", cluster.getClusterControllers()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(27, cluster.getRootGroup().getSubgroups().size()); assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); @@ -730,7 +728,6 @@ public class ModelProvisioningTest { // Check content cluster ContentCluster cluster = model.getContentClusters().get("bar"); - assertNull(cluster.getClusterControllers()); assertEquals(0, cluster.getRootGroup().getNodes().size()); assertEquals(8, cluster.getRootGroup().getSubgroups().size()); assertEquals(8, cluster.distributionBits()); @@ -865,8 +862,6 @@ public class ModelProvisioningTest { assertEquals(7, model.getRoot().hostSystem().getHosts().size()); // Check cluster controllers - assertNull(model.getContentClusters().get("foo").getClusterControllers()); - assertNull(model.getContentClusters().get("bar").getClusterControllers()); ClusterControllerContainerCluster clusterControllers = model.getAdmin().getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("cluster-controllers", clusterControllers.getName()); @@ -2021,7 +2016,7 @@ public class ModelProvisioningTest { } private VespaModel createNonProvisionedModel(boolean multitenant, String hosts, String services) { - VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSearchDefinition("type1")); + VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(hosts, services, generateSchemas("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; DeployState deployState = new DeployState.Builder().applicationPackage(appPkg). properties((new TestProperties()).setMultitenant(multitenant)). @@ -2029,7 +2024,7 @@ public class ModelProvisioningTest { return modelCreatorWithMockPkg.create(false, deployState); } - private int physicalMemoryPercentage(ContainerCluster cluster) { + private int physicalMemoryPercentage(ContainerCluster<?> cluster) { QrStartConfig.Builder b = new QrStartConfig.Builder(); cluster.getConfig(b); return b.build().jvm().heapSizeAsPercentageOfPhysicalMemory(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 125e25470df..370315ad08b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -105,7 +105,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testRankExpression() throws IOException, ParseException { assertCorrectDeriving("rankexpression", null, - new TestProperties().useExternalRankExpression(false), new TestableDeployLogger()); + new TestProperties().useExternalRankExpression(true), new TestableDeployLogger()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index ff17a211e17..85ef70132b5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -52,22 +52,22 @@ public class RankingExpressionsTestCase extends SchemaTestCase { RawRankProfile rawRankProfile = new RawRankProfile(functionsRankProfile, new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(search), deployProperties); List<Pair<String, String>> rankProperties = rawRankProfile.configProperties(); - assertEquals(5, rankProperties.size()); + assertEquals(6, rankProperties.size()); - assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); - assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); + assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); - assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); - assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); + assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); + assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); - assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(2).getFirst()); - assertEquals("78 + closeness(distance)", rankProperties.get(2).getSecond()); + assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); + assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); - assertEquals("vespa.rank.firstphase", rankProperties.get(3).getFirst()); - assertEquals("rankingExpression(firstphase)", rankProperties.get(3).getSecond()); + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); + assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); - assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(4).getFirst()); - assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(4).getSecond()); + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index 91152648b10..8f9aa33ea52 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -49,9 +49,9 @@ public class DocumentModelBuilderTestCase extends SchemaTestCase { @Test public void testMultipleInheritanceArray() throws IOException, ParseException { SearchBuilder search = new SearchBuilder(); - search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTData.sd"); - search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd"); - search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd"); + search.importFile("src/test/cfg/search/data/travel/schemas/TTData.sd"); + search.importFile("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); + search.importFile("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); search.build(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java index 76f34cf4a81..341a90c6618 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java @@ -15,6 +15,7 @@ import org.junit.rules.ExpectedException; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -99,17 +100,17 @@ public class ComplexAttributeFieldsValidatorTestCase { "}")); } - private static void createModelAndValidate(String searchDefinition) throws IOException, SAXException { - DeployState deployState = createDeployState(servicesXml(), searchDefinition); + private static void createModelAndValidate(String schema) throws IOException, SAXException { + DeployState deployState = createDeployState(servicesXml(), schema); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationParameters validationParameters = new ValidationParameters(CheckRouting.FALSE); Validation.validate(model, validationParameters, deployState); } - private static DeployState createDeployState(String servicesXml, String searchDefinition) { + private static DeployState createDeployState(String servicesXml, String schema) { ApplicationPackage app = new MockApplicationPackage.Builder() .withServices(servicesXml) - .withSearchDefinition(searchDefinition) + .withSchemas(List.of(schema)) .build(); return new DeployState.Builder().applicationPackage(app).build(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index ad4603e5c6b..7bff4890b7e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -824,24 +824,6 @@ public class ContentBuilderTest extends DomBuilderTest { verifyThatFeatureFlagControlsVisibilityDelayDefault(0.6, 0.6); } - private void verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(boolean flag) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForPruneRemoved(flag)); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(singleNodeContentXml()) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(flag, config.pruneremoveddocuments().usebucketexecutor()); - } - - - @Test - public void verifyUseBucketExecutorForPruneRemoved() { - verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(true); - verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(false); - } - @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index f3199f6a46f..80aafdd4ec7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -69,7 +69,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { " <accesslog type='vespa' ", " fileNamePattern='pattern' rotationInterval='interval' />", " <accesslog type='json' ", - " fileNamePattern='pattern' rotationInterval='interval' />", + " fileNamePattern='pattern' rotationInterval='interval' queueSize='17' bufferSize='65536'/>", nodesXml, "</container>" ); @@ -85,8 +85,8 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); - assertEquals(10000, fileHandlerConfig.queueSize()); - assertEquals(4*1024*1024, fileHandlerConfig.bufferSize()); + assertEquals(256, fileHandlerConfig.queueSize()); + assertEquals(256*1024, fileHandlerConfig.bufferSize()); } { // json @@ -97,8 +97,8 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); - assertEquals(10000, fileHandlerConfig.queueSize()); - assertEquals(4*1024*1024, fileHandlerConfig.bufferSize()); + assertEquals(17, fileHandlerConfig.queueSize()); + assertEquals(65536, fileHandlerConfig.bufferSize()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 7f862afa1b0..543318f9224 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -40,6 +40,11 @@ import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.prelude.cluster.QrMonitorConfig; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -53,6 +58,7 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; import org.hamcrest.core.IsEqual; import org.junit.Rule; @@ -61,8 +67,15 @@ import org.junit.rules.TemporaryFolder; import org.w3c.dom.Element; import org.xml.sax.SAXException; +import javax.security.auth.x500.X500Principal; import java.io.IOException; import java.io.StringReader; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -82,6 +95,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -818,6 +832,48 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void operator_certificates_are_joined_with_clients_pem() { + var applicationPackage = new MockApplicationPackage.Builder() + .withRoot(applicationFolder.getRoot()) + .build(); + + var applicationTrustCert = X509CertificateUtils.toPem( + X509CertificateUtils.createSelfSigned("CN=application", Duration.ofDays(1)).certificate()); + var operatorCert = X509CertificateUtils.createSelfSigned("CN=operator", Duration.ofDays(1)).certificate(); + + applicationPackage.getFile(Path.fromString("security")).createDirectory(); + applicationPackage.getFile(Path.fromString("security/clients.pem")).writeFile(new StringReader(applicationTrustCert)); + + var deployState = new DeployState.Builder().properties( + new TestProperties() + .setOperatorCertificates(List.of(operatorCert)) + .setHostedVespa(true) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) + .zone(new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName())) + .applicationPackage(applicationPackage) + .build(); + + Element clusterElem = DomBuilderTest.parse("<container version='1.0' />"); + + createModel(root, deployState, null, clusterElem); + + ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); + List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories(); + ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow(); + + ConnectorConfig.Builder builder = new ConnectorConfig.Builder(); + tlsPort.getConfig(builder); + + ConnectorConfig connectorConfig = new ConnectorConfig(builder); + var caCerts = X509CertificateUtils.certificateListFromPem(connectorConfig.ssl().caCertificate()); + assertEquals(2, caCerts.size()); + List<String> certnames = caCerts.stream() + .map(cert -> cert.getSubjectX500Principal().getName()) + .collect(Collectors.toList()); + assertThat(certnames, containsInAnyOrder("CN=operator", "CN=application")); + } + + @Test public void environment_vars_are_honoured() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 953c42243a6..13d02fc1fb8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; @@ -46,7 +45,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static java.util.stream.Collectors.toList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -533,6 +531,7 @@ public class ContentClusterTest extends ContentBaseTest { ContentCluster stagingNot16Bits = createWithZone(xml, new Zone(Environment.staging, RegionName.from("us-east-3"))); assertDistributionBitsInConfig(stagingNot16Bits, 8); } + @Test public void testGenerateSearchNodes() { @@ -1076,7 +1075,6 @@ public class ContentClusterTest extends ContentBaseTest { " </documents>" + " </content>" + " </services>"); - assertNull("No own cluster controller for content", oneContentModel.getContentClusters().get("storage").getClusterControllers()); assertNotNull("Shared cluster controller with content", oneContentModel.getAdmin().getClusterControllers()); String twoContentServices = "<?xml version='1.0' encoding='UTF-8' ?>" + @@ -1108,8 +1106,6 @@ public class ContentClusterTest extends ContentBaseTest { VespaModel twoContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) .setMultitenant(true), twoContentServices); - assertNull("No own cluster controller for content", twoContentModel.getContentClusters().get("storage").getClusterControllers()); - assertNull("No own cluster controller for content", twoContentModel.getContentClusters().get("dev-null").getClusterControllers()); assertNotNull("Shared cluster controller with content", twoContentModel.getAdmin().getClusterControllers()); ClusterControllerContainerCluster clusterControllers = twoContentModel.getAdmin().getClusterControllers(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 5cf57430f91..9a681003293 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.config.content.PersistenceConfig; import com.yahoo.config.model.test.MockRoot; import com.yahoo.documentmodel.NewDocumentType; import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static com.yahoo.config.model.test.TestUtil.joinLines; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; @@ -44,10 +45,17 @@ public class StorageClusterTest { return parse(xml, root); } - StorageCluster parse(String xml) { - MockRoot root = new MockRoot(); + StorageCluster parse(String xml, ModelContext.Properties properties) { + MockRoot root = new MockRoot("", + new DeployState.Builder() + .properties(properties) + .applicationPackage(new MockApplicationPackage.Builder().build()) + .build()); return parse(xml, root); } + StorageCluster parse(String xml) { + return parse(xml, new TestProperties()); + } StorageCluster parse(String xml, MockRoot root) { root.getDeployState().getDocumentModel().getDocumentManager().add( new NewDocumentType(new NewDocumentType.Name("music")) @@ -61,13 +69,23 @@ public class StorageClusterTest { return cluster.getStorageNodes(); } + private static String group() { + return joinLines( + "<group>", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + "</group>"); + } + private static String cluster(String clusterName, String insert) { + return joinLines( + "<content id=\"" + clusterName + "\">", + "<documents/>", + insert, + group(), + "</content>"); + } @Test public void testBasics() { - StorageCluster storage = parse("<content id=\"foofighters\"><documents/>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</content>\n"); + StorageCluster storage = parse(cluster("foofighters", "")); assertEquals(1, storage.getChildren().size()); StorServerConfig.Builder builder = new StorServerConfig.Builder(); @@ -79,11 +97,7 @@ public class StorageClusterTest { } @Test public void testCommunicationManagerDefaults() { - StorageCluster storage = parse("<content id=\"foofighters\"><documents/>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</content>\n"); + StorageCluster storage = parse(cluster("foofighters", "")); StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder(); storage.getChildren().get("0").getConfig(builder); StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); @@ -97,40 +111,49 @@ public class StorageClusterTest { } @Test + public void testMergeDefaults() { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + parse(cluster("foofighters", "")).getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + assertEquals(16, config.max_merges_per_node()); + assertEquals(1024, config.max_merge_queue_size()); + } + + @Test public void testMerges() { StorServerConfig.Builder builder = new StorServerConfig.Builder(); - parse("" + - "<content id=\"foofighters\">\n" + - " <documents/>" + - " <tuning>" + - " <merges max-per-node=\"1K\" max-queue-size=\"10K\"/>\n" + - " </tuning>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</content>" + parse(cluster("foofighters", joinLines( + "<tuning>", + " <merges max-per-node=\"1K\" max-queue-size=\"10K\"/>", + "</tuning>")), + new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37) ).getConfig(builder); StorServerConfig config = new StorServerConfig(builder); assertEquals(1024, config.max_merges_per_node()); assertEquals(1024*10, config.max_merge_queue_size()); } + @Test + public void testMergeFeatureFlags() { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + parse(cluster("foofighters", ""), new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)).getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + assertEquals(37, config.max_merges_per_node()); + assertEquals(1919, config.max_merge_queue_size()); + } @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); - parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " <visitors thread-count=\"7\" max-queue-size=\"1000\">\n" + - " <max-concurrent fixed=\"42\" variable=\"100\"/>\n" + - " </visitors>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>" + parse(cluster("bees", + joinLines( + "<tuning>", + " <visitors thread-count=\"7\" max-queue-size=\"1000\">", + " <max-concurrent fixed=\"42\" variable=\"100\"/>", + " </visitors>", + "</tuning>")) ).getConfig(builder); StorVisitorConfig config = new StorVisitorConfig(builder); @@ -143,16 +166,10 @@ public class StorageClusterTest { @Test public void testPersistenceThreads() { - StorageCluster stc = parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " <persistence-threads count=\"7\"/>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + StorageCluster stc = parse(cluster("bees",joinLines( + "<tuning>", + " <persistence-threads count=\"7\"/>", + "</tuning>")), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); @@ -178,16 +195,10 @@ public class StorageClusterTest { @Test public void testResponseThreads() { - StorageCluster stc = parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " <persistence-threads count=\"7\"/>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + StorageCluster stc = parse(cluster("bees",joinLines( + "<tuning>", + " <persistence-threads count=\"7\"/>", + "</tuning>")), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); @@ -201,20 +212,14 @@ public class StorageClusterTest { @Test public void testPersistenceThreadsOld() { - StorageCluster stc = parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " <persistence-threads>\n" + - " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + - " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + - " <thread count=\"1\"/>\n" + - " </persistence-threads>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + StorageCluster stc = parse(cluster("bees", joinLines( + "<tuning>", + " <persistence-threads>", + " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>", + " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>", + " <thread count=\"1\"/>", + " </persistence-threads>", + "</tuning>")), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); @@ -238,15 +243,7 @@ public class StorageClusterTest { @Test public void testNoPersistenceThreads() { - StorageCluster stc = parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <tuning>\n" + - " </tuning>\n" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + StorageCluster stc = parse(cluster("bees", ""), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); @@ -267,13 +264,7 @@ public class StorageClusterTest { } private StorageCluster simpleCluster(ModelContext.Properties properties) { - return parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>", + return parse(cluster("bees", ""), new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()), properties); } @@ -302,14 +293,7 @@ public class StorageClusterTest { @Test public void integrity_checker_explicitly_disabled_when_not_running_with_vds_provider() { StorIntegritycheckerConfig.Builder builder = new StorIntegritycheckerConfig.Builder(); - parse( - "<cluster id=\"bees\">\n" + - " <documents/>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + - " </group>" + - "</cluster>" - ).getConfig(builder); + parse(cluster("bees", "")).getConfig(builder); StorIntegritycheckerConfig config = new StorIntegritycheckerConfig(builder); // '-' --> don't run on the given week day assertEquals("-------", config.weeklycycle()); @@ -317,15 +301,15 @@ public class StorageClusterTest { @Test public void testCapacity() { - String xml = - "<cluster id=\"storage\">\n" + - " <documents/>" + - " <group>\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\" capacity=\"1.5\"/>\n" + - " <node distribution-key=\"2\" hostalias=\"mockhost\" capacity=\"2.0\"/>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <documents/>", + " <group>", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " <node distribution-key=\"1\" hostalias=\"mockhost\" capacity=\"1.5\"/>", + " <node distribution-key=\"2\" hostalias=\"mockhost\" capacity=\"2.0\"/>", + " </group>", + "</cluster>"); ContentCluster cluster = ContentClusterUtils.createCluster(xml, new MockRoot()); @@ -341,15 +325,7 @@ public class StorageClusterTest { @Test public void testRootFolder() { - String xml = - "<cluster id=\"storage\">\n" + - " <documents/>" + - " <group>\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - "</cluster>"; - - ContentCluster cluster = ContentClusterUtils.createCluster(xml, new MockRoot()); + ContentCluster cluster = ContentClusterUtils.createCluster(cluster("storage", ""), new MockRoot()); StorageNode node = cluster.getStorageNodes().getChildren().get("0"); @@ -372,18 +348,18 @@ public class StorageClusterTest { @Test public void testGenericPersistenceTuning() { - String xml = - "<cluster id=\"storage\">\n" + - "<documents/>" + - "<engine>\n" + - " <fail-partition-on-error>true</fail-partition-on-error>\n" + - " <revert-time>34m</revert-time>\n" + - " <recovery-time>5d</recovery-time>\n" + - "</engine>" + - " <group>\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <documents/>", + " <engine>", + " <fail-partition-on-error>true</fail-partition-on-error>", + " <revert-time>34m</revert-time>", + " <recovery-time>5d</recovery-time>", + " </engine>", + " <group>", + " node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + "</cluster>"); ContentCluster cluster = ContentClusterUtils.createCluster(xml, new MockRoot()); @@ -398,21 +374,21 @@ public class StorageClusterTest { @Test public void requireThatUserDoesNotSpecifyBothGroupAndNodes() { - String xml = - "<cluster id=\"storage\">\n" + - "<documents/>\n" + - "<engine>\n" + - " <fail-partition-on-error>true</fail-partition-on-error>\n" + - " <revert-time>34m</revert-time>\n" + - " <recovery-time>5d</recovery-time>\n" + - "</engine>" + - " <group>\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " <nodes>\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + - " </nodes>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <documents/>", + " <engine>", + " <fail-partition-on-error>true</fail-partition-on-error>", + " <revert-time>34m</revert-time>", + " <recovery-time>5d</recovery-time>", + " </engine>", + " <group>", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + " <nodes>", + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>", + " </nodes>", + "</cluster>"); try { final MockRoot root = new MockRoot(); @@ -429,20 +405,20 @@ public class StorageClusterTest { @Test public void requireThatGroupNamesMustBeUniqueAmongstSiblings() { - String xml = - "<cluster id=\"storage\">\n" + - " <redundancy>2</redundancy>" + - " <documents/>\n" + - " <group>\n" + - " <distribution partitions=\"*\"/>\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <redundancy>2</redundancy>", + " <documents/>", + " <group>", + " <distribution partitions=\"*\"/>", + " <group distribution-key=\"0\" name=\"bar\">", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + " <group distribution-key=\"0\" name=\"bar\">", + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>", + " </group>", + " </group>", + "</cluster>"); try { ContentClusterUtils.createCluster(xml, new MockRoot()); @@ -455,24 +431,24 @@ public class StorageClusterTest { @Test public void requireThatGroupNamesCanBeDuplicatedAcrossLevels() { - String xml = - "<cluster id=\"storage\">\n" + - " <redundancy>2</redundancy>" + - "<documents/>\n" + - " <group>\n" + - " <distribution partitions=\"*\"/>\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <group distribution-key=\"0\" name=\"foo\">\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " </group>\n" + - " <group distribution-key=\"0\" name=\"foo\">\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " </group>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <redundancy>2</redundancy>", + " <documents/>", + " <group>", + " <distribution partitions=\"*\"/>", + " <group distribution-key=\"0\" name=\"bar\">", + " <group distribution-key=\"0\" name=\"foo\">", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + " </group>", + " <group distribution-key=\"0\" name=\"foo\">", + " <group distribution-key=\"0\" name=\"bar\">", + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>", + " </group>", + " </group>", + " </group>", + "</cluster>"); // Should not throw. ContentClusterUtils.createCluster(xml, new MockRoot()); @@ -480,18 +456,18 @@ public class StorageClusterTest { @Test public void requireThatNestedGroupsRequireDistribution() { - String xml = - "<cluster id=\"storage\">\n" + - "<documents/>\n" + - " <group>\n" + - " <group distribution-key=\"0\" name=\"bar\">\n" + - " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " <group distribution-key=\"0\" name=\"baz\">\n" + - " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + - " </group>\n" + - " </group>\n" + - "</cluster>"; + String xml = joinLines( + "<cluster id=\"storage\">", + " <documents/>", + " <group>", + " <group distribution-key=\"0\" name=\"bar\">", + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>", + " </group>", + " <group distribution-key=\"0\" name=\"baz\">", + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>", + " </group>", + " </group>", + "</cluster>"); try { ContentClusterUtils.createCluster(xml, new MockRoot()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 7c93b4ef02b..afeffbbc875 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -15,9 +15,7 @@ import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -179,7 +177,7 @@ public class ClusterTest { " </tuning>", " </content>", "</services>")) - .withSchemas(ApplicationPackageUtils.generateSearchDefinition("my_document")) + .withSchemas(ApplicationPackageUtils.generateSchemas("my_document")) .build(); List<Content> contents = new TestDriver().buildModel(app).getConfigModels(Content.class); assertEquals(1, contents.size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index 854c4d32a64..9e87b5509bc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -32,8 +32,8 @@ public class MlModelsTest { IOUtils.copy(appDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - IOUtils.copyDirectory(appDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile(), - storedAppDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile()); + IOUtils.copyDirectory(appDir.append(ApplicationPackage.SCHEMAS_DIR).toFile(), + storedAppDir.append(ApplicationPackage.SCHEMAS_DIR).toFile()); ImportedModelTester storedTester = new ImportedModelTester("ml_models", storedAppDir); verify(storedTester.createVespaModel()); } @@ -61,7 +61,7 @@ public class MlModelsTest { assertEquals(testProfile, b.toString()); } - private final String testProfile = + private static final String testProfile = "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + "rankingExpression(Placeholder).type: tensor<float>(d0[1],d1[784])\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index 5dea4a04229..5c64dd9f8cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -4,17 +4,13 @@ package com.yahoo.vespa.model.ml; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; -import ai.vespa.models.evaluation.RankProfilesConfigImporter; -import ai.vespa.models.handler.ModelsEvaluationHandler; import com.yahoo.component.ComponentId; -import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; @@ -31,7 +27,6 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -55,8 +50,8 @@ public class StatelessOnnxEvaluationTest { IOUtils.copy(appDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - IOUtils.copyDirectory(appDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile(), - storedAppDir.append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).toFile()); + IOUtils.copyDirectory(appDir.append(ApplicationPackage.SCHEMAS_DIR).toFile(), + storedAppDir.append(ApplicationPackage.SCHEMAS_DIR).toFile()); ImportedModelTester storedTester = new ImportedModelTester("onnx_rt", storedAppDir); assertModelEvaluation(storedTester.createVespaModel(), appDir); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index f5250343afe..2b36bfc47b2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -192,7 +192,7 @@ public class VespaModelTestCase { " </documents>" + "</content>" + "</services>", - ApplicationPackageUtils.generateSearchDefinition("music")) + ApplicationPackageUtils.generateSchemas("music")) .create(); MessagebusConfig.Builder mBusB = new MessagebusConfig.Builder(); model.getConfig(mBusB, "client"); @@ -216,7 +216,7 @@ public class VespaModelTestCase { "</hosts>"); } - class MyLogger implements DeployLogger { + static class MyLogger implements DeployLogger { List<Pair<Level, String>> msgs = new ArrayList<>(); @Override public void log(Level level, String message) { @@ -305,4 +305,29 @@ public class VespaModelTestCase { assertThat(model.getContainerClusters().size(), is(1)); } + @Test + public void testThatDeployLogContainsWarningWhenUsingSearchdefinitionsDir() throws IOException, SAXException { + ApplicationPackage app = FilesApplicationPackage.fromFile( + new File("src/test/cfg/application/deprecated_features_app/")); + MyLogger logger = new MyLogger(); + DeployState deployState = new DeployState.Builder() + .applicationPackage(app) + .deployLogger(logger) + .build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + Validation.validate(model, new ValidationParameters(), deployState); + assertContainsWarning(logger.msgs, "Directory searchdefinitions/ should not be used for schemas, use schemas/ instead"); + } + + private void assertContainsWarning(List<Pair<Level,String>> msgs, String text) { + boolean foundCorrectWarning = false; + for (var msg : msgs) + if (msg.getFirst().getName().equals("WARNING") && msg.getSecond().equals(text)) { + foundCorrectWarning = true; + } + if (! foundCorrectWarning) for (var msg : msgs) System.err.println("MSG: "+msg); + assertTrue(msgs.size() > 0); + assertTrue(foundCorrectWarning); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index ba975e52d1a..7e34e9efbbf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -21,7 +21,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; @@ -31,6 +30,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; + /** * Helper class which sets up a system with multiple hosts. * Usage: @@ -168,7 +169,7 @@ public class VespaModelTester { boolean alwaysReturnOneNode, int startIndexForClusters, Optional<VespaModel> previousModel, DeployState.Builder deployStatebuilder, String ... retiredHostNames) { - VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); + VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, generateSchemas("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; provisioner = hosted ? new ProvisionerAdapter(new InMemoryProvisioner(hostsByResources, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index df62a3bff07..1f7deaf1991 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.test.utils; import java.util.ArrayList; @@ -43,10 +43,6 @@ public class ApplicationPackageUtils { "}"; } - public static List<String> generateSearchDefinition(String name) { - return generateSchemas(name); - } - public static List<String> generateSchemas(String ... sdNames) { return generateSchemas(Arrays.asList(sdNames)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java index 8147d2e00ca..eea3da0e096 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -24,9 +24,8 @@ import java.io.IOException; */ public class VespaModelCreatorWithFilePkg { - private FilesApplicationPackage applicationPkg; - - private ConfigModelRegistry configModelRegistry; + private final FilesApplicationPackage applicationPkg; + private final ConfigModelRegistry configModelRegistry; public VespaModelCreatorWithFilePkg(String directoryName) { this(new File(directoryName)); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index db1e6c29586..4ad7865c93b 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -147,7 +147,7 @@ </http> <accesslog type='json' - fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S' compressOnRotation='true' compressionType='zstd'/> + fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S' compressOnRotation='true' compressionType='zstd' queueSize='13' bufferSize='65536'/> <accesslog type='vespa' fileNamePattern='logs/vespa/qrs/access-vespa.%Y%m%d%H%M%S' /> |