diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
23 files changed, 300 insertions, 240 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 061ad42e028..364dd1742ae 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -65,8 +65,8 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra * Creates and initializes a new Vespa from the service config file * in the given application directory. * - * @param parent The parent, usually VespaModel - * @param name The name, used as configId + * @param parent the parent, usually VespaModel + * @param name the name, used as configId * @param documentModel DocumentModel to serve global document config from. */ public ApplicationConfigProducerRoot(AbstractConfigProducer parent, String name, DocumentModel documentModel, Version vespaVersion, ApplicationId applicationId) { diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 557df4bd106..100b5ec5a24 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -11,6 +11,16 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.model.graph.ModelNode; import com.yahoo.config.model.provision.HostsXmlProvisioner; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; @@ -29,7 +39,6 @@ import java.io.IOException; import java.io.Reader; import java.io.Serializable; import java.io.StringReader; -import java.util.*; import java.util.logging.Logger; /** diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 1d0541b67d1..68924dde3e1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -22,7 +22,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; -import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.model.provision .SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; @@ -460,7 +460,7 @@ public class DeployState implements ConfigDefinitionStore { private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles, ValidationParameters validationParameters) { - Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); + Collection<NamedReader> readers = applicationPackage.getSchemas(); Map<String, String> names = new LinkedHashMap<>(); SearchBuilder builder = new SearchBuilder(applicationPackage, logger, properties, rankProfileRegistry, queryProfiles.getRegistry()); for (NamedReader reader : readers) { @@ -470,14 +470,14 @@ public class DeployState implements ConfigDefinitionStore { String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); names.put(topLevelName, sdName); if ( ! sdName.equals(topLevelName)) { - throw new IllegalArgumentException("Schema definition file name ('" + sdName + "') and name of " + + throw new IllegalArgumentException("Schema file name ('" + sdName + "') and name of " + "top level element ('" + topLevelName + "') are not equal for file '" + readerName + "'"); } } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse sd file '" + reader.getName() + "'", e); + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); } catch (IOException e) { - throw new IllegalArgumentException("Could not read sd file '" + reader.getName() + "'", e); + throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); } finally { closeIgnoreException(reader.getReader()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2a530b78b86..2927df57bc1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import java.net.URI; +import java.security.cert.X509Certificate; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -50,7 +51,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useAccessControlTlsHandshakeClientAuth; private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean useBucketExecutorForPruneRemoved; private boolean enableFeedBlockInDistributor = true; private boolean useExternalRankExpression = false; private int clusterControllerMaxHeapSizeInMb = 128; @@ -59,7 +59,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int numDistributorStripes = 0; + private int maxConcurrentMergesPerNode = 16; + private int maxMergeQueueSize = 1024; private boolean allowDisableMtls = true; + private List<X509Certificate> operatorCertificates = Collections.emptyList(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -90,7 +93,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; } @@ -99,8 +101,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public int numDistributorStripes() { return numDistributorStripes; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } + @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; } @Override public boolean distributeExternalRankExpressions() { return useExternalRankExpression; } + @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } + @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } public TestProperties useExternalRankExpression(boolean value) { useExternalRankExpression = value; @@ -133,6 +138,15 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMaxConcurrentMergesPerNode(int maxConcurrentMergesPerNode) { + this.maxConcurrentMergesPerNode = maxConcurrentMergesPerNode; + return this; + } + public TestProperties setMaxMergeQueueSize(int maxMergeQueueSize) { + this.maxMergeQueueSize = maxMergeQueueSize; + return this; + } + public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -198,11 +212,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties useBucketExecutorForPruneRemoved(boolean enabled) { - useBucketExecutorForPruneRemoved = enabled; - return this; - } - public TestProperties enableFeedBlockInDistributor(boolean enabled) { enableFeedBlockInDistributor = enabled; return this; @@ -243,6 +252,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setOperatorCertificates(List<X509Certificate> operatorCertificates) { + this.operatorCertificates = List.copyOf(operatorCertificates); + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index e9fe0824f30..411a37bb70a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -115,7 +115,7 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> getSearchDefinitions() { + public List<NamedReader> getSchemas() { ArrayList<NamedReader> readers = new ArrayList<>(); SearchBuilder searchBuilder = new SearchBuilder(this, new BaseDeployLogger(), @@ -134,11 +134,6 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> searchDefinitionContents() { - return new ArrayList<>(); - } - - @Override public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { return Collections.emptyMap(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 39ab443554c..68e03ecb188 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -73,12 +73,10 @@ public class RankProfile implements Cloneable { protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); /** The ranking expression to be used for first phase */ - private RankingExpression firstPhaseRanking = null; + private RankingExpressionFunction firstPhaseRanking = null; /** The ranking expression to be used for second phase */ - private RankingExpression secondPhaseRanking = null; - - private Set<String> externalFileExpressions = new HashSet<>(); + private RankingExpressionFunction secondPhaseRanking = null; /** Number of hits to be reranked in second phase, -1 means use default */ private int rerankCount = -1; @@ -128,8 +126,8 @@ public class RankProfile implements Cloneable { /** * Creates a new rank profile for a particular search definition * - * @param name the name of the new profile - * @param search the search definition owning this profile + * @param name the name of the new profile + * @param search the search definition owning this profile * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing * and looking up rank profiles. */ @@ -169,10 +167,6 @@ public class RankProfile implements Cloneable { return search != null ? search.rankingConstants() : model.rankingConstants(); } - public RankExpressionFiles rankExpressionFiles() { - return search != null ? search.rankExpressionFiles() : model.rankExpressionFiles(); - } - public Map<String, OnnxModel> onnxModels() { return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); } @@ -198,9 +192,7 @@ public class RankProfile implements Cloneable { } /** Returns the name of the profile this one inherits, or null if none is inherited */ - public String getInheritedName() { - return inheritedName; - } + public String getInheritedName() { return inheritedName; } /** Returns the inherited rank profile, or null if there is none */ public RankProfile getInherited() { @@ -243,7 +235,7 @@ public class RankProfile implements Cloneable { public MatchPhaseSettings getMatchPhaseSettings() { MatchPhaseSettings settings = this.matchPhaseSettings; - if (settings != null ) return settings; + if (settings != null) return settings; if (getInherited() != null) return getInherited().getMatchPhaseSettings(); return null; } @@ -264,7 +256,7 @@ public class RankProfile implements Cloneable { * @return the rank setting found, or null. */ RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { - for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext();) { + for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext(); ) { RankSetting setting = i.next(); if (setting.getFieldName().equals(field) && setting.getType().equals(type)) { @@ -369,55 +361,26 @@ public class RankProfile implements Cloneable { * Returns null if no expression is set. */ public RankingExpression getFirstPhaseRanking() { - if (firstPhaseRanking != null) return firstPhaseRanking; - if (getInherited() != null) return getInherited().getFirstPhaseRanking(); - return null; - } - - void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking = rankingExpression; - } - - public String getUniqueExpressionName(String name) { - return getName() + "_" + name; - } - public String getFirstPhaseFile() { - String name = FIRST_PHASE; - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if ((firstPhaseRanking == null) && (getInherited() != null)) { - return getInherited().getFirstPhaseFile(); - } - return null; + RankingExpressionFunction function = getFirstPhase(); + if (function == null) return null; + return function.function.getBody(); } - public String getSecondPhaseFile() { - String name = SECOND_PHASE; - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if ((secondPhaseRanking == null) && (getInherited() != null)) { - return getInherited().getSecondPhaseFile(); - } + public RankingExpressionFunction getFirstPhase() { + if (firstPhaseRanking != null) return firstPhaseRanking; + RankProfile inherited = getInherited(); + if (inherited != null) return inherited.getFirstPhase(); return null; } - public String getExpressionFile(String name) { - if (externalFileExpressions.contains(name)) { - return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); - } - if (getInherited() != null) { - return getInherited().getExpressionFile(name); - } - return null; + void setFirstPhaseRanking(RankingExpression rankingExpression) { + this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, Collections.emptyList(), rankingExpression), false); } public void setFirstPhaseRanking(String expression) { try { - this.firstPhaseRanking = parseRankingExpression(FIRST_PHASE, expression); - } - catch (ParseException e) { + firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, Collections.emptyList(), expression), false); + } catch (ParseException e) { throw new IllegalArgumentException("Illegal first phase ranking function", e); } } @@ -427,14 +390,21 @@ public class RankProfile implements Cloneable { * Returns null if no expression is set. */ public RankingExpression getSecondPhaseRanking() { + RankingExpressionFunction function = getSecondPhase(); + if (function == null) return null; + return function.function().getBody(); + } + + public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - if (getInherited() != null) return getInherited().getSecondPhaseRanking(); + RankProfile inherited = getInherited(); + if (inherited != null) return inherited.getSecondPhase(); return null; } public void setSecondPhaseRanking(String expression) { try { - this.secondPhaseRanking = parseRankingExpression(SECOND_PHASE, expression); + secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, Collections.emptyList(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal second phase ranking function", e); @@ -603,7 +573,7 @@ public class RankProfile implements Cloneable { /** Adds a function */ public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { - addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); + addFunction(parseRankingExpression(name, arguments, expression), inline); } catch (ParseException e) { throw new IllegalArgumentException("Could not parse function '" + name + "'", e); @@ -691,20 +661,20 @@ public class RankProfile implements Cloneable { return retval; } - private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException { + private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { if (expression.trim().length() == 0) - throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + "."); + throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + name + "."); - try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) { - return new RankingExpression(expressionName, rankingExpressionReader); + try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { + return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + - "' in " + getName()+ ", " + expressionName + "."); + "' in " + getName()+ ", " + name + "."); throw (ParseException)exception.initCause(e); } catch (IOException e) { - throw new RuntimeException("IOException parsing ranking expression '" + expressionName + "'"); + throw new RuntimeException("IOException parsing ranking expression '" + name + "'"); } } @@ -725,10 +695,6 @@ public class RankProfile implements Cloneable { throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file + "' in subdirectory, which is not supported."); - if (search.getDeployProperties().featureFlags().distributeExternalRankExpressions()) { - rankExpressionFiles().add(new RankExpressionFile(getUniqueExpressionName(expName), fileName), search.getDeployLogger()); - externalFileExpressions.add(expName); - } return search.getRankingExpression(fileName); } @@ -777,17 +743,17 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> inlineFunctions = compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); // Function compiling second pass: compile all functions and insert previously compiled inline functions + // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) { - if (constants.get(functionEntry.getKey()) != null) + if (constants.containsKey(functionEntry.getKey())) throw new IllegalArgumentException("Cannot have both a constant and function named '" + functionEntry.getKey() + "'"); } @@ -811,15 +777,15 @@ public class RankProfile implements Cloneable { // A straightforward iteration will either miss those functions, or may cause a ConcurrentModificationException while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) { RankingExpressionFunction rankingExpressionFunction = entry.getValue(); - RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, featureTypes, + RankingExpressionFunction compiled = compile(rankingExpressionFunction, queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withExpression(compiled)); + compiledFunctions.put(entry.getKey(), compiled); } return compiledFunctions; } - private Map.Entry<String, RankingExpressionFunction> findUncompiledFunction(Map<String, RankingExpressionFunction> functions, - Set<String> compiledFunctionNames) { + private static Map.Entry<String, RankingExpressionFunction> findUncompiledFunction(Map<String, RankingExpressionFunction> functions, + Set<String> compiledFunctionNames) { for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) { if ( ! compiledFunctionNames.contains(entry.getKey())) return entry; @@ -827,25 +793,25 @@ public class RankProfile implements Cloneable { return null; } - private RankingExpression compile(RankingExpression expression, + private RankingExpressionFunction compile(RankingExpressionFunction function, QueryProfileRegistry queryProfiles, Map<Reference, TensorType> featureTypes, ImportedMlModels importedModels, Map<String, Value> constants, Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { - if (expression == null) return null; + if (function == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, featureTypes, importedModels, constants, inlineFunctions); - expression = expressionTransforms.transform(expression, context); + RankingExpression expression = expressionTransforms.transform(function.function().getBody(), context); for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); } - return expression; + return function.withExpression(expression); } /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 41ac1e17d93..6b589a22de5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -55,15 +55,18 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, + AttributeFields attributeFields, ModelContext.Properties deployProperties) { this.name = rankProfile.getName(); - compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields, deployProperties).derive()); + compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), + attributeFields, deployProperties).derive()); } /** * Only for testing */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, + ImportedMlModels importedModels, AttributeFields attributeFields) { this(rankProfile, queryProfiles, importedModels, attributeFields, new TestProperties()); } @@ -120,61 +123,74 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private static class Deriver { - /** - * The field rank settings of this profile - */ - private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); - - private final RankProfile rankProfile; - private RankingExpression firstPhaseRanking = null; - private RankingExpression secondPhaseRanking = null; - - private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>(); - - private Set<ReferenceNode> rankFeatures = new LinkedHashSet<>(); - - private List<RankProfile.RankProperty> rankProperties = new ArrayList<>(); + private final Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); + private final Set<ReferenceNode> summaryFeatures; + private final Set<ReferenceNode> rankFeatures; + private final List<RankProfile.RankProperty> rankProperties; /** * Rank properties for weight settings to make these available to feature executors */ - private List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); - - private boolean ignoreDefaultRankFeatures = false; - - private RankProfile.MatchPhaseSettings matchPhaseSettings = null; - - private int rerankCount = -1; - private int keepRankCount = -1; - private int numThreadsPerSearch = -1; - private int minHitsPerThread = -1; - private int numSearchPartitions = -1; - private double termwiseLimit = 1.0; - private double rankScoreDropLimit = -Double.MAX_VALUE; + private final List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); + + private final boolean ignoreDefaultRankFeatures; + private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final int rerankCount; + private final int keepRankCount; + private final int numThreadsPerSearch; + private final int minHitsPerThread; + private final int numSearchPartitions; + private final double termwiseLimit; + private final double rankScoreDropLimit; /** * The rank type definitions used to derive settings for the native rank features */ private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); - private final Map<String, String> attributeTypes; private final Map<String, String> queryFeatureTypes; - private final boolean useExternalExpressionFiles; + private final Set<String> filterFields = new java.util.LinkedHashSet<>(); - private Set<String> filterFields = new java.util.LinkedHashSet<>(); + private RankingExpression firstPhaseRanking; + private RankingExpression secondPhaseRanking; /** * Creates a raw rank profile from the given rank profile */ - Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - AttributeFields attributeFields, ModelContext.Properties deployProperties) + Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.rankProfile = rankProfile; - RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); - useExternalExpressionFiles = deployProperties.featureFlags().useExternalRankExpressions(); - deriveRankingFeatures(compiled, deployProperties); + firstPhaseRanking = compiled.getFirstPhaseRanking(); + secondPhaseRanking = compiled.getSecondPhaseRanking(); + summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures()); + rankFeatures = compiled.getRankFeatures(); + rerankCount = compiled.getRerankCount(); + matchPhaseSettings = compiled.getMatchPhaseSettings(); + numThreadsPerSearch = compiled.getNumThreadsPerSearch(); + minHitsPerThread = compiled.getMinHitsPerThread(); + numSearchPartitions = compiled.getNumSearchPartitions(); + termwiseLimit = compiled.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); + keepRankCount = compiled.getKeepRankCount(); + rankScoreDropLimit = compiled.getRankScoreDropLimit(); + ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); + rankProperties = new ArrayList<>(compiled.getRankProperties()); + + Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); + List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); + Map<String, String> functionProperties = new LinkedHashMap<>(); + SerializationContext functionSerializationContext = new SerializationContext(functionExpressions); + + if (firstPhaseRanking != null) { + functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); + } + if (secondPhaseRanking != null) { + functionProperties.putAll(secondPhaseRanking.getRankProperties(functionSerializationContext)); + } + + derivePropertiesAndSummaryFeaturesFromFunctions(functions, functionProperties, functionSerializationContext); + deriveOnnxModelFunctionsAndSummaryFeatures(compiled); + deriveRankTypeSetting(compiled, attributeFields); deriveFilterFields(compiled); deriveWeightProperties(compiled); @@ -184,44 +200,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer { filterFields.addAll(rp.allFilterFields()); } - private void deriveRankingFeatures(RankProfile rankProfile, ModelContext.Properties deployProperties) { - firstPhaseRanking = rankProfile.getFirstPhaseRanking(); - secondPhaseRanking = rankProfile.getSecondPhaseRanking(); - summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures()); - rankFeatures = rankProfile.getRankFeatures(); - rerankCount = rankProfile.getRerankCount(); - matchPhaseSettings = rankProfile.getMatchPhaseSettings(); - numThreadsPerSearch = rankProfile.getNumThreadsPerSearch(); - minHitsPerThread = rankProfile.getMinHitsPerThread(); - numSearchPartitions = rankProfile.getNumSearchPartitions(); - termwiseLimit = rankProfile.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); - keepRankCount = rankProfile.getKeepRankCount(); - rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); - ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); - rankProperties = new ArrayList<>(rankProfile.getRankProperties()); - derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions()); - deriveOnnxModelFunctionsAndSummaryFeatures(rankProfile); - } - - private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) { + private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions, + Map<String, String> functionProperties, + SerializationContext functionContext) { if (functions.isEmpty()) return; - List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); - Map<String, String> functionProperties = new LinkedHashMap<>(); - - if (firstPhaseRanking != null) { - functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions)); - } - if (secondPhaseRanking != null) { - functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions)); - } - - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); - replaceFunctionSummaryFeatures(context); + replaceFunctionSummaryFeatures(functionContext); // First phase, second phase and summary features should add all required functions to the context. // However, we need to add any functions not referenced in those anyway for model-evaluation. - deriveFunctionProperties(functions, functionExpressions, functionProperties); + deriveFunctionProperties(functions, functionProperties, functionContext); for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); @@ -229,15 +217,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, - List<ExpressionFunction> functionExpressions, - Map<String, String> functionProperties) { - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); + Map<String, String> functionProperties, + SerializationContext context) { for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { - if (useExternalExpressionFiles && rankProfile.getExpressionFile(e.getKey()) != null) continue; String propertyName = RankingExpression.propertyName(e.getKey()); if (context.serializedFunctions().containsKey(propertyName)) continue; - String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + String expressionString = e.getValue().function().getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) @@ -259,7 +245,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { ExpressionFunction function = context.getFunction(referenceNode.getName()); if (function != null) { String propertyName = RankingExpression.propertyName(referenceNode.getName()); - String expressionString = function.getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + String expressionString = function.getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); @@ -355,8 +341,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, rankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE)); - properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, rankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, RankProfile.FIRST_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, RankProfile.SECOND_PHASE)); for (FieldRankSettings settings : fieldRankSettings.values()) { properties.addAll(settings.deriveRankProperties()); } @@ -408,9 +394,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (ignoreDefaultRankFeatures) { properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } - Iterator filterFieldsIterator = filterFields.iterator(); - while (filterFieldsIterator.hasNext()) { - String fieldName = (String) filterFieldsIterator.next(); + for (String fieldName : filterFields) { properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); } for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { @@ -423,7 +407,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return properties; } - private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String fileName, String phase) { + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { List<Pair<String, String>> properties = new ArrayList<>(); if (expression == null) return properties; @@ -431,9 +415,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if ("".equals(name)) name = phase; - if (useExternalExpressionFiles && (fileName != null)) { - properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + rankProfile.getUniqueExpressionName(name) + ")")); - } else if (expression.getRoot() instanceof ReferenceNode) { + if (expression.getRoot() instanceof ReferenceNode) { properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java index d15db6b4a55..800bf73cdbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -18,16 +19,20 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr private final ApplicationId applicationId; private final Zone zone; + private final boolean requireConnectivityCheck; /** * Constructs a new ConfigSentinel for the given host. * * @param host Physical host on which to run. */ - public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone) { + public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone, + ModelContext.FeatureFlags featureFlags) + { super(host, "sentinel"); this.applicationId = applicationId; this.zone = zone; + this.requireConnectivityCheck = featureFlags.requireConnectivityCheck(); portsMeta.on(0).tag("rpc").tag("admin"); portsMeta.on(1).tag("telnet").tag("interactive").tag("http").tag("state"); setProp("clustertype", "hosts"); @@ -75,6 +80,19 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr builder.service(getServiceConfig(s)); } } + builder.connectivity(getConnectivityConfig(requireConnectivityCheck)); + } + + private SentinelConfig.Connectivity.Builder getConnectivityConfig(boolean enable) { + var builder = new SentinelConfig.Connectivity.Builder(); + if (enable) { + builder.maxBadOutPercent(60); + builder.maxBadReverseCount(3); + } else { + builder.maxBadOutPercent(100); + builder.maxBadReverseCount(Integer.MAX_VALUE); + } + return builder; } private SentinelConfig.Application.Builder getApplicationConfig() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 09bbd446803..53f42866d8d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -47,16 +47,15 @@ public class HostSystem extends AbstractConfigProducer<Host> { void checkName(String hostname) { // Give a warning if the host does not exist try { - @SuppressWarnings("unused") - Object ignore = java.net.InetAddress.getByName(hostname); + var inetAddr = java.net.InetAddress.getByName(hostname); + String canonical = inetAddr.getCanonicalHostName(); + if (! hostname.equals(canonical)) { + deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + + "since it differs from its canonical hostname '" + canonical + "' (check DNS and /etc/hosts)."); + } } catch (UnknownHostException e) { deployLogger.logApplicationPackage(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } - if (! hostname.contains(".")) { - deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + - "since it is not a canonical hostname. " + - "Disregard this warning when testing in a Docker container."); - } } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index e080ce43730..a2a6ada9093 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; @@ -242,7 +243,8 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } private void addCommonServices(HostResource host, DeployState deployState) { - addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone()); + addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone(), + deployState.featureFlags()); addLogd(deployState.getDeployLogger(), host); addConfigProxy(deployState.getDeployLogger(), host); addFileDistribution(host); @@ -262,8 +264,10 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } } - private void addConfigSentinel(DeployLogger deployLogger, HostResource host, ApplicationId applicationId, Zone zone) { - ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone); + private void addConfigSentinel(DeployLogger deployLogger, HostResource host, + ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) + { + ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone, featureFlags); addAndInitializeService(deployLogger, host, configSentinel); host.getHost().setConfigSentinel(configSentinel); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 6f467b21535..e2aa325c380 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -19,19 +19,31 @@ public class AutoscalingMetrics { private static MetricSet create() { List<String> metrics = new ArrayList<>(); + metrics.add("cpu.util"); - metrics.add("mem.util"); - metrics.add("disk.util"); + + // Memory util + metrics.add("mem.util"); // node level - default + metrics.add("content.proton.resource_usage.memory.average"); // better for content as it is the basis for blocking + + // Disk util + metrics.add("disk.util"); // node level -default + metrics.add("content.proton.resource_usage.disk.average"); // better for content as it is the basis for blocking + metrics.add("application_generation"); + metrics.add("in_service"); + // Query rate metrics.add("queries.rate"); // container metrics.add("content.proton.documentdb.matching.queries.rate"); // content + // Write rate metrics.add("feed.http-requests.rate"); // container metrics.add("vds.filestor.alldisks.allthreads.put.sum.count.rate"); // content metrics.add("vds.filestor.alldisks.allthreads.remove.sum.count.rate"); // content metrics.add("vds.filestor.alldisks.allthreads.update.sum.count.rate"); // content + return new MetricSet("autoscaling", toMetrics(metrics)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 034bf772ffc..114a3e380ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -256,8 +256,11 @@ public class VespaMetricSet { metrics.add(new Metric("cluster-controller.node-event.count")); metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.last")); + metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.max")); metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.max")); metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.max")); metrics.add(new Metric("cluster-controller.resource_usage.disk_limit.last")); metrics.add(new Metric("cluster-controller.resource_usage.memory_limit.last")); @@ -762,6 +765,15 @@ public class VespaMetricSet { metrics.add(new Metric("vds.bouncer.clock_skew_aborts.count")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.max")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.sum")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.count")); + metrics.add(new Metric("vds.mergethrottler.queuesize.max")); + metrics.add(new Metric("vds.mergethrottler.queuesize.sum")); + metrics.add(new Metric("vds.mergethrottler.queuesize.count")); + metrics.add(new Metric("vds.mergethrottler.bounced_due_to_back_pressure.rate")); + metrics.add(new Metric("vds.mergethrottler.locallyexecutedmerges.ok.rate")); + metrics.add(new Metric("vds.mergethrottler.mergechains.ok.rate")); return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java new file mode 100644 index 00000000000..ffcad391e57 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SchemasDirValidator.java @@ -0,0 +1,32 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; + +import java.util.logging.Level; + +/** + * Validates that correct directory is used for schemas + * + * @author hmusum + */ +public class SchemasDirValidator extends Validator { + + public SchemasDirValidator() { + } + + @Override + public void validate(VespaModel model, DeployState deployState) { + ApplicationPackage app = deployState.getApplicationPackage(); + ApplicationFile sdDir = app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR); + if (sdDir.exists() && sdDir.isDirectory()) + deployState.getDeployLogger().logApplicationPackage( + Level.WARNING, + "Directory " + ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative() + + "/ should not be used for schemas, use " + ApplicationPackage.SCHEMAS_DIR.getRelative() + "/ instead"); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index d1c93bcd611..55443d4b260 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -9,7 +9,6 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.container.QrConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.application.validation.change.ClusterSizeReductionValidator; @@ -25,8 +24,6 @@ import com.yahoo.vespa.model.application.validation.change.ResourcesReductionVal import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.Container; import java.time.Instant; import java.util.Arrays; @@ -43,7 +40,6 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; /** * Executor of validators. This defines the right order of validator execution. @@ -63,6 +59,7 @@ public class Validation { new RoutingValidator().validate(model, deployState); new RoutingSelectorValidator().validate(model, deployState); } + new SchemasDirValidator().validate(model, deployState); new ComponentValidator().validate(model, deployState); new SearchDataTypeValidator().validate(model, deployState); new ComplexAttributeFieldsValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index f18f10644ca..fc41da43479 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -35,7 +35,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this(cluster, logType, compressionType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), null, null, isHostedVespa, - capitalize(logType.name()) + "." + clusterName); + capitalize(logType.name()) + "." + clusterName, + queueSize(cluster).orElse(-1), + ((cluster instanceof ApplicationContainerCluster) ? 4*1024*1024 : null)); } private static String capitalize(String name) { @@ -49,7 +51,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL String rotationInterval, Boolean compressOnRotation, boolean isHostedVespa, - String symlinkName) + String symlinkName, + Integer queueSize, + Integer bufferSize) { super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); this.fileNamePattern = fileNamePattern; @@ -58,10 +62,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; this.compressionType = compressionType; - this.queueSize = queueSize(cluster).orElse(-1); - bufferSize = (cluster instanceof ApplicationContainerCluster) - ? 4*1024*1024 - : null; + this.queueSize = (queueSize == null) ? 256 : queueSize; + this.bufferSize = bufferSize; if (fileNamePattern == null) throw new RuntimeException("File name pattern required when configuring access log."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java index 83e06aab703..0b51cd163a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.logging.ConnectionLog; import com.yahoo.container.logging.ConnectionLogConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import java.util.OptionalInt; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index f84d6f0724b..d7812e9b4ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -62,7 +62,9 @@ public class AccessLogBuilder { rotationInterval(spec), compressOnRotation(spec), isHostedVespa, - symlinkName(spec)); + symlinkName(spec), + queueSize(spec), + bufferSize(spec)); } private String symlinkName(Element spec) { @@ -74,6 +76,16 @@ public class AccessLogBuilder { return (compress.isEmpty() ? null : Boolean.parseBoolean(compress)); } + private Integer bufferSize(Element spec) { + String value = spec.getAttribute("bufferSize"); + return (value.isEmpty() ? null : Integer.parseInt(value)); + } + + private Integer queueSize(Element spec) { + String value = spec.getAttribute("queueSize"); + return (value.isEmpty() ? null : Integer.parseInt(value)); + } + private String rotationInterval(Element spec) { return nullIfEmpty(spec.getAttribute("rotationInterval")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 7e0be6e448b..a37d0ef416f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -48,7 +48,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { cluster.addComponent( new AccessLogComponent( cluster, AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, - "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log")); + "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log", 1024,256*1024)); cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); } else { super.addAccessLogs(deployState, cluster, spec); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index b477587bcac..08ccfe33cd5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -34,6 +34,7 @@ import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -89,6 +90,7 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; import java.net.URI; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -217,7 +219,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if(deployState.isHosted()) { cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath("jdisc-cloud-aws")); } - if (deployState.featureFlags().tenantIamRole()) { + if (deployState.zone().system().isPublic()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); Handler<AbstractConfigProducer<?>> handler = new Handler<>( new ComponentModel("com.yahoo.jdisc.cloud.aws.AwsParameterStoreValidationHandler", null, "jdisc-cloud-aws", null)); @@ -431,6 +433,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // If the deployment contains certificate/private key reference, setup TLS port HostedSslConnectorFactory connectorFactory; + boolean enableHttp2 = deployState.featureFlags().enableJdiscHttp2(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { @@ -444,7 +447,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .orElse(false); connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState)) : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets, enforceHandshakeClientAuth); } else { connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName); @@ -453,6 +456,19 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { server.addConnector(connectorFactory); } + /* + Return trusted certificates as a PEM encoded string containing the concatenation of + trusted certs from the application package and all operator certificates. + */ + String getTlsClientAuthorities(DeployState deployState) { + List<X509Certificate> trustedCertificates = deployState.tlsClientAuthority() + .map(X509CertificateUtils::certificateListFromPem) + .orElse(Collections.emptyList()); + ArrayList<X509Certificate> x509Certificates = new ArrayList<>(trustedCertificates); + x509Certificates.addAll(deployState.getProperties().operatorCertificates()); + return X509CertificateUtils.toPem(x509Certificates); + } + private static boolean isHostedTenantApplication(ConfigModelContext context) { var deployState = context.getDeployState(); boolean isTesterApplication = deployState.getProperties().applicationId().instance().isTester(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 2bd9cb09aa6..ea52f9689ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -177,7 +177,7 @@ public class Content extends ConfigModel { s.setVespaMallocDebugStackTrace(cluster.getRootGroup().getVespaMallocDebugStackTrace().get()); } } - cluster.prepare(deployState); + cluster.prepare(); } private void setCpuSocketAffinity() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 18580249ddc..51949e78838 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -65,7 +65,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; - private final boolean useBucketExecutorForPruneRemoved; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -210,7 +209,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); - useBucketExecutorForPruneRemoved = featureFlags.useBucketExecutorForPruneRemoved(); } public void setVisibilityDelay(double delay) { @@ -427,7 +425,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.indexing.optimize(feedSequencerType); } - builder.pruneremoveddocuments.usebucketexecutor(useBucketExecutorForPruneRemoved); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 830eced56d3..90cca1494b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -70,7 +70,7 @@ import static java.util.stream.Collectors.toList; * @author mostly somebody unknown * @author bratseth */ -public class ContentCluster extends AbstractConfigProducer implements +public class ContentCluster extends AbstractConfigProducer<AbstractConfigProducer<?>> implements DistributionConfig.Producer, StorDistributionConfig.Producer, StorDistributormanagerConfig.Producer, @@ -94,14 +94,6 @@ public class ContentCluster extends AbstractConfigProducer implements private Integer maxNodesPerMerge; private final Zone zone; - /** - * If multitenant or a cluster controller was explicitly configured in this cluster: - * The cluster controller cluster of this particular content cluster. - * - * Otherwise: null - the cluster controller is shared by all content clusters and part of Admin. - */ - private ClusterControllerContainerCluster clusterControllers; - public enum DistributionMode { LEGACY, STRICT, LOOSE } private DistributionMode distributionMode; @@ -413,16 +405,10 @@ public class ContentCluster extends AbstractConfigProducer implements public ClusterSpec.Id id() { return ClusterSpec.Id.from(clusterId); } - public void prepare(DeployState deployState) { + public void prepare() { search.prepare(); - - if (clusterControllers != null) - clusterControllers.prepare(deployState); } - /** Returns cluster controllers if this is multitenant, null otherwise */ - public ClusterControllerContainerCluster getClusterControllers() { return clusterControllers; } - public DistributionMode getDistributionMode() { if (distributionMode != null) return distributionMode; return getPersistence().getDefaultDistributionMode(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 40a634fbfe8..e89d45e8b83 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -6,6 +6,8 @@ import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import java.util.Optional; + /** * Serves config for stor-server for storage clusters (clusters of storage nodes). */ @@ -14,7 +16,7 @@ public class StorServerProducer implements StorServerConfig.Producer { StorServerProducer build(ModelContext.Properties properties, ModelElement element) { ModelElement tuning = element.child("tuning"); - StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element)); + StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element), properties.featureFlags()); if (tuning == null) return producer; ModelElement merges = tuning.child("merges"); @@ -32,11 +34,15 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer bucketDBStripeBits; private StorServerProducer setMaxMergesPerNode(Integer value) { - maxMergesPerNode = value; + if (value != null) { + maxMergesPerNode = value; + } return this; } private StorServerProducer setMaxQueueSize(Integer value) { - queueSize = value; + if (value != null) { + queueSize = value; + } return this; } private StorServerProducer setBucketDBStripeBits(Integer value) { @@ -44,8 +50,10 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } - public StorServerProducer(String clusterName) { + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; + maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); + queueSize = featureFlags.maxMergeQueueSize(); } @Override |