diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
40 files changed, 336 insertions, 353 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index 3d3ebc90890..23ea5775caa 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -84,7 +84,6 @@ public class AdminModel extends ConfigModel { AbstractConfigProducer<?> parent = modelContext.getParentProducer(); ModelContext.Properties properties = modelContext.getDeployState().getProperties(); DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), - modelContext.getDeployState().getFileRegistry().fileSourceHost(), properties.multitenant(), properties.configServerSpecs()); model.admin = domBuilder.build(modelContext.getDeployState(), parent, adminElement); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 220599b6d1c..249ca71117a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -5,6 +5,7 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; @@ -55,6 +56,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutorService; import java.util.logging.Level; /** @@ -87,6 +89,7 @@ public class DeployState implements ConfigDefinitionStore { private final HostProvisioner provisioner; private final Provisioned provisioned; private final Reindexing reindexing; + private final ExecutorService executor; public static DeployState createTestState() { return new Builder().build(); @@ -104,6 +107,7 @@ public class DeployState implements ConfigDefinitionStore { SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, FileRegistry fileRegistry, + ExecutorService executor, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, Provisioned provisioned, @@ -124,6 +128,7 @@ public class DeployState implements ConfigDefinitionStore { Reindexing reindexing) { this.logger = deployLogger; this.fileRegistry = fileRegistry; + this.executor = executor; this.rankProfileRegistry = rankProfileRegistry; this.applicationPackage = applicationPackage; this.properties = properties; @@ -282,6 +287,8 @@ public class DeployState implements ConfigDefinitionStore { /** The (machine learned) models imported from the models/ directory, as an unmodifiable map indexed by model name */ public ImportedMlModels getImportedModels() { return importedModels; } + public ExecutorService getExecutor() { return executor; } + public Version getWantedNodeVespaVersion() { return wantedNodeVespaVersion; } public Optional<DockerImage> getWantedDockerImageRepo() { return wantedDockerImageRepo; } @@ -312,6 +319,7 @@ public class DeployState implements ConfigDefinitionStore { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); private FileRegistry fileRegistry = new MockFileRegistry(); + private ExecutorService executor = new InThreadExecutorService(); private DeployLogger logger = new BaseDeployLogger(); private Optional<HostProvisioner> hostProvisioner = Optional.empty(); private Provisioned provisioned = new Provisioned(); @@ -329,6 +337,8 @@ public class DeployState implements ConfigDefinitionStore { private Optional<DockerImage> wantedDockerImageRepo = Optional.empty(); private Reindexing reindexing = null; + public Builder() {} + public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; return this; @@ -339,6 +349,11 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder executor(ExecutorService executor) { + this.executor = executor; + return this; + } + public Builder deployLogger(DeployLogger logger) { this.logger = logger; return this; @@ -433,6 +448,7 @@ public class DeployState implements ConfigDefinitionStore { searchDocumentModel, rankProfileRegistry, fileRegistry, + executor, logger, hostProvisioner, provisioned, diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 7d8932b9d00..f4f3ab53bc6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -47,19 +47,18 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int responseNumThreads = 2; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; - private ApplicationRoles applicationRoles; private Quota quota = Quota.unlimited(); private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; private boolean enableFeedBlockInDistributor = true; - private boolean useExternalRankExpression = false; + private boolean useExternalRankExpression = true; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int numDistributorStripes = 0; private int maxConcurrentMergesPerNode = 16; private int maxMergeQueueSize = 1024; - private int largeRankExpressionLimit = 0x10000; + private int largeRankExpressionLimit = 8192; private boolean allowDisableMtls = true; private boolean dryRunOnnxOnSetup = false; private List<X509Certificate> operatorCertificates = Collections.emptyList(); @@ -86,7 +85,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; } @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } - @Override public Optional<ApplicationRoles> applicationRoles() { return Optional.ofNullable(applicationRoles); } @Override public String responseSequencerType() { return responseSequencerType; } @Override public int defaultNumResponseThreads() { return responseNumThreads; } @Override public boolean skipCommunicationManagerThread() { return false; } @@ -111,6 +109,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } + @Override public int metricsproxyNumThreads() { return 1; } public TestProperties setDryRunOnnxOnSetup(boolean value) { dryRunOnnxOnSetup = value; @@ -214,11 +213,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setApplicationRoles(ApplicationRoles applicationRoles) { - this.applicationRoles = applicationRoles; - return this; - } - public TestProperties setQuota(Quota quota) { this.quota = quota; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 12b6f902770..753db89af1b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.test; import com.yahoo.config.ConfigInstance; @@ -17,7 +17,7 @@ import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; -import com.yahoo.vespa.model.filedistribution.FileDistributor; +import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -43,7 +43,7 @@ public class MockRoot extends AbstractConfigProducerRoot { private final HostSystem hostSystem; private final DeployState deployState; - private final FileDistributor fileDistributor; + private final FileReferencesRepository fileReferencesRepository; private Admin admin; public MockRoot() { @@ -66,7 +66,7 @@ public class MockRoot extends AbstractConfigProducerRoot { super(rootConfigId); hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner(), deployState.getDeployLogger()); this.deployState = deployState; - fileDistributor = new FileDistributor(deployState.getFileRegistry().fileSourceHost()); + fileReferencesRepository = new FileReferencesRepository(); } public FileDistributionConfigProducer getFileDistributionConfigProducer() { @@ -119,15 +119,11 @@ public class MockRoot extends AbstractConfigProducerRoot { return deployState; } - public FileDistributor getFileDistributor() { - return fileDistributor; - } + public FileReferencesRepository fileReferencesRepository() { return fileReferencesRepository; } - public HostSystem hostSystem() { - return hostSystem; - } + public HostSystem hostSystem() { return hostSystem; } - public void addDescendant(String configId, AbstractConfigProducer descendant) { + public void addDescendant(String configId, AbstractConfigProducer<?> descendant) { if (id2producer.containsKey(configId)) { throw new RuntimeException ("Config ID '" + configId + "' cannot be reserved by an instance of class '" + @@ -138,7 +134,7 @@ public class MockRoot extends AbstractConfigProducerRoot { } @Override - public void addChild(AbstractConfigProducer abstractConfigProducer) { + public void addChild(AbstractConfigProducer<?> abstractConfigProducer) { super.addChild(abstractConfigProducer); } @@ -149,8 +145,7 @@ public class MockRoot extends AbstractConfigProducerRoot { try { Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); - setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry().fileSourceHost(), - false, new ArrayList<>()). + setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, false, new ArrayList<>()). build(deployState, this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); } catch (SAXException | IOException e) { throw new RuntimeException(e); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index e31d90c2800..1719ea72cb0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -38,6 +38,7 @@ public class DistributableResource { Objects.requireNonNull(blob, "Blob cannot be null"); this.name = name; this.blob = blob; + path = name + ".lz4"; pathType = PathType.BLOB; } @@ -87,7 +88,7 @@ public class DistributableResource { fileReference = fileRegistry.addUri(path); break; case BLOB: - fileReference = fileRegistry.addBlob(blob); + fileReference = fileRegistry.addBlob(path, blob); break; default: throw new IllegalArgumentException("Unknown path type " + pathType); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index 0d10932c333..41d7519b784 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -4,7 +4,6 @@ package com.yahoo.searchdefinition; import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Optional; -import java.util.regex.Pattern; /** * Utility methods for query, document and constant rank feature names @@ -12,9 +11,6 @@ import java.util.regex.Pattern; * @author bratseth */ public class FeatureNames { - - private static final Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); - public static Reference asConstantFeature(String constantName) { return Reference.simple("constant", quoteIfNecessary(constantName)); } @@ -46,7 +42,7 @@ public class FeatureNames { */ public static Optional<String> argumentOf(String feature) { Optional<Reference> reference = Reference.simple(feature); - if ( ! reference.isPresent()) return Optional.empty(); + if ( reference.isEmpty()) return Optional.empty(); if ( ! ( reference.get().name().equals("attribute") || reference.get().name().equals("constant") || reference.get().name().equals("query"))) @@ -56,10 +52,28 @@ public class FeatureNames { } private static String quoteIfNecessary(String s) { - if (identifierRegexp.matcher(s).matches()) + if (notNeedQuotes(s)) return s; else return "\"" + s + "\""; } + static boolean notNeedQuotes(String s) { + // Faster version of the regexp [A-Za-z0-9_][A-Za-z0-9_-]* + if (s.isEmpty()) return false; + if ( ! isValidFirst(s.charAt(0))) return false; + for (int i = 1; i < s.length(); i++) { + if (!isValidAny(s.charAt(i))) return false; + } + return true; + } + private static boolean isValidFirst(char c) { + // [A-Za-z0-9_] + return (c == '_') || ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || ((c >= '0') && (c <= '9')); + } + private static boolean isValidAny(char c) { + // [A-Za-z0-9_-]* + return c == '-' || isValidFirst(c); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java index 9d8e8b90ceb..90dc40e7b93 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java @@ -21,11 +21,14 @@ public class LargeRankExpressions { expression.register(fileRegistry); String name = expression.getName(); if (expressions.containsKey(name)) { - throw new IllegalArgumentException("Rank expression '" + name + - "' defined twice. Previous blob with " + expressions.get(name).getBlob().remaining() + - " bytes, while current has " + expression.getBlob().remaining() + " bytes"); + if ( ! expressions.get(name).getBlob().equals(expression.getBlob())) { + throw new IllegalArgumentException("Rank expression '" + name + + "' defined twice. Previous blob with " + expressions.get(name).getBlob().remaining() + + " bytes, while current has " + expression.getBlob().remaining() + " bytes"); + } + } else { + expressions.put(name, expression); } - expressions.put(name, expression); } /** Returns the ranking constant with the given name, or null if not present */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 3c1f00f1252..47d95cba516 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -104,6 +104,8 @@ public class RankProfile implements Cloneable { private Boolean ignoreDefaultRankFeatures = null; private Map<String, RankingExpressionFunction> functions = new LinkedHashMap<>(); + // This cache must be invalidated every time modifications are done to 'functions'. + private Map<String, RankingExpressionFunction> allFunctionsCached = null; private Map<Reference, TensorType> inputFeatures = new LinkedHashMap<>(); @@ -120,6 +122,9 @@ public class RankProfile implements Cloneable { private List<ImmutableSDField> allFieldsList; + /** Global onnx models not tied to a search definition */ + private final OnnxModels onnxModels; + /** * Creates a new rank profile for a particular search definition * @@ -132,6 +137,7 @@ public class RankProfile implements Cloneable { this.name = Objects.requireNonNull(name, "name cannot be null"); this.search = Objects.requireNonNull(search, "search cannot be null"); this.model = null; + this.onnxModels = null; this.rankProfileRegistry = rankProfileRegistry; } @@ -141,11 +147,12 @@ public class RankProfile implements Cloneable { * @param name the name of the new profile * @param model the model owning this profile */ - public RankProfile(String name, VespaModel model, RankProfileRegistry rankProfileRegistry) { + public RankProfile(String name, VespaModel model, RankProfileRegistry rankProfileRegistry, OnnxModels onnxModels) { this.name = Objects.requireNonNull(name, "name cannot be null"); this.search = null; this.model = Objects.requireNonNull(model, "model cannot be null"); this.rankProfileRegistry = rankProfileRegistry; + this.onnxModels = onnxModels; } public String getName() { return name; } @@ -164,7 +171,7 @@ public class RankProfile implements Cloneable { } public Map<String, OnnxModel> onnxModels() { - return search != null ? search.onnxModels().asMap() : model.onnxModels().asMap(); + return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); } private Stream<ImmutableSDField> allFields() { @@ -580,6 +587,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); functions.put(function.getName(), rankingExpressionFunction); + allFunctionsCached = null; return rankingExpressionFunction; } @@ -608,6 +616,12 @@ public class RankProfile implements Cloneable { } /** Returns an unmodifiable snapshot of the functions in this */ public Map<String, RankingExpressionFunction> getFunctions() { + if (needToUpdateFunctionCache()) { + allFunctionsCached = gatherAllFunctions(); + } + return allFunctionsCached; + } + private Map<String, RankingExpressionFunction> gatherAllFunctions() { if (functions.isEmpty() && getInherited() == null) return Collections.emptyMap(); if (functions.isEmpty()) return getInherited().getFunctions(); if (getInherited() == null) return Collections.unmodifiableMap(new LinkedHashMap<>(functions)); @@ -618,6 +632,12 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableMap(allFunctions); } + private boolean needToUpdateFunctionCache() { + if (getInherited() != null) + return (allFunctionsCached == null) || getInherited().needToUpdateFunctionCache(); + return allFunctionsCached == null; + } + public int getKeepRankCount() { if (keepRankCount >= 0) return keepRankCount; if (getInherited() != null) return getInherited().getKeepRankCount(); @@ -706,6 +726,7 @@ public class RankProfile implements Cloneable { clone.rankProperties = new LinkedHashMap<>(this.rankProperties); clone.inputFeatures = new LinkedHashMap<>(this.inputFeatures); clone.functions = new LinkedHashMap<>(this.functions); + clone.allFunctionsCached = null; clone.filterFields = new HashSet<>(this.filterFields); clone.constants = new HashMap<>(this.constants); return clone; @@ -745,6 +766,7 @@ public class RankProfile implements Cloneable { // Function compiling second pass: compile all functions and insert previously compiled inline functions // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); + allFunctionsCached = null; } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 625c584f79a..f49a477c66b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -527,5 +527,7 @@ public class SearchBuilder { public QueryProfileRegistry getQueryProfileRegistry() { return queryProfileRegistry; } + public ModelContext.Properties getProperties() { return properties; } + public DeployLogger getDeployLogger() { return deployLogger; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 16de8fd4b9e..9764d63a804 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -19,7 +20,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import java.io.IOException; import java.io.Writer; -import java.util.logging.Level; +import java.util.concurrent.ExecutorService; /** * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when @@ -52,8 +53,12 @@ public class DerivedConfiguration { * modified. * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} */ - DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { - this(search, new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfiles, importedModels); + public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry) { + this(search, rankProfileRegistry, new QueryProfileRegistry()); + } + + DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { + this(search, new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfiles, new ImportedMlModels(), new InThreadExecutorService()); } /** @@ -72,7 +77,8 @@ public class DerivedConfiguration { ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels) { + ImportedMlModels importedModels, + ExecutorService executor) { Validator.ensureNotNull("Search definition", search); this.search = search; this.queryProfiles = queryProfiles; @@ -87,7 +93,7 @@ public class DerivedConfiguration { juniperrc = new Juniperrc(search); rankProfileList = new RankProfileList(search, search.rankingConstants(), search.rankExpressionFiles(), search.onnxModels(), attributeFields, rankProfileRegistry, - queryProfiles, importedModels, deployProperties); + queryProfiles, importedModels, deployProperties, executor); indexingScript = new IndexingScript(search); indexInfo = new IndexInfo(search); indexSchema = new IndexSchema(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 7f19fcd24d4..6528c9195d4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -3,7 +3,6 @@ package com.yahoo.searchdefinition.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; @@ -19,8 +18,13 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.AbstractService; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.logging.Logger; /** @@ -61,13 +65,14 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - ModelContext.Properties deployProperties) { + ModelContext.Properties deployProperties, + ExecutorService executor) { setName(search == null ? "default" : search.getName()); this.rankingConstants = rankingConstants; this.largeRankExpressions = largeRankExpressions; this.onnxModels = onnxModels; // as ONNX models come from parsing rank expressions dryRunOnnxOnSetup = deployProperties.featureFlags().dryRunOnnxOnSetup(); - deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties); + deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties, executor); } private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, @@ -75,12 +80,12 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ ImportedMlModels importedModels, Search search, AttributeFields attributeFields, - ModelContext.Properties deployProperties) { + ModelContext.Properties deployProperties, + ExecutorService executor) { + List<Future<RawRankProfile>> futureRawRankProfiles = new ArrayList<>(); if (search != null) { // profiles belonging to a search have a default profile - RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.get(search, "default"), - largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties); - rankProfiles.put(defaultProfile.getName(), defaultProfile); + futureRawRankProfiles.add(executor.submit(() -> new RawRankProfile(rankProfileRegistry.get(search, "default"), + largeRankExpressions, queryProfiles, importedModels, attributeFields, deployProperties))); } for (RankProfile rank : rankProfileRegistry.rankProfilesOf(search)) { @@ -89,9 +94,16 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ this.onnxModels.add(rank.onnxModels()); } - RawRankProfile rawRank = new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties); - rankProfiles.put(rawRank.getName(), rawRank); + futureRawRankProfiles.add(executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties))); + } + try { + for (Future<RawRankProfile> rawFuture : futureRawRankProfiles) { + RawRankProfile rawRank = rawFuture.get(); + rankProfiles.put(rawRank.getName(), rawRank); + } + } catch (InterruptedException | ExecutionException e) { + throw new IllegalStateException(e); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index fee73a03ae6..792965314c5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -6,7 +6,6 @@ import com.google.common.collect.ImmutableList; import com.yahoo.collections.Pair; import com.yahoo.compress.Compressor; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.LargeRankExpressions; @@ -324,7 +323,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } /** Derives the properties this produces */ - public List<Pair<String, String>> derive(LargeRankExpressions largeRankExpressions) { + public List<Pair<String, String>> derive(LargeRankExpressions largeRankExpressions) { List<Pair<String, String>> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { if (RankingExpression.propertyName(RankProfile.FIRST_PHASE).equals(property.getName())) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 549f99f9577..e47fd339505 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -459,10 +459,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * @param reference file reference (hash) */ public void send(FileReference reference) { - Host host = null; - if (getHost() != null) // false when running application tests without hosts - host = getHost().getHost(); - getRoot().getFileDistributor().sendFileReference(reference, host); + getRoot().fileReferencesRepository().add(reference); } /** The service HTTP port for health status */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java index b30aa5aeca6..e9b587c0eb9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java @@ -1,10 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.filedistribution.FileDistributor; +import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; import java.util.Set; @@ -22,7 +22,7 @@ public interface ConfigProducerRoot extends ConfigProducer { * @param id string id of descendant * @param descendant the producer to add to this root node */ - void addDescendant(String id, AbstractConfigProducer descendant); + void addDescendant(String id, AbstractConfigProducer<?> descendant); /** * @return an unmodifiable copy of the set of configIds in this root. @@ -39,7 +39,7 @@ public interface ConfigProducerRoot extends ConfigProducer { */ public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId); - FileDistributor getFileDistributor(); + FileReferencesRepository fileReferencesRepository(); Admin getAdmin(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 15377a95e3c..e6687ec8ac8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -55,7 +55,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; -import com.yahoo.vespa.model.filedistribution.FileDistributor; +import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; import com.yahoo.vespa.model.generic.service.ServiceCluster; import com.yahoo.vespa.model.ml.ConvertedModel; import com.yahoo.vespa.model.ml.ModelName; @@ -130,13 +130,12 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** Large rank expression files of this */ private final LargeRankExpressions largeRankExpressions; - /** Large rank expression files of this */ - private final OnnxModels onnxModels; + private final FileRegistry fileRegistry; /** The validation overrides of this. This is never null. */ private final ValidationOverrides validationOverrides; - private final FileDistributor fileDistributor; + private final FileReferencesRepository fileReferencesRepository = new FileReferencesRepository(); private final Provisioned provisioned; @@ -169,16 +168,16 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * @param deployState the global deploy state to use for this model. */ public VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState) throws IOException, SAXException { - this(configModelRegistry, deployState, true, null); + this(configModelRegistry, deployState, true); } - private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor) + private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete) throws IOException, SAXException { super("vespamodel"); version = deployState.getVespaVersion(); + fileRegistry = deployState.getFileRegistry(); largeRankExpressions = new LargeRankExpressions(deployState.getFileRegistry()); rankingConstants = new RankingConstants(deployState.getFileRegistry()); - onnxModels = new OnnxModels(deployState.getFileRegistry()); validationOverrides = deployState.validationOverrides(); applicationPackage = deployState.getApplicationPackage(); provisioned = deployState.provisioned(); @@ -190,19 +189,19 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri rankProfileList = new RankProfileList(null, // null search -> global rankingConstants, largeRankExpressions, - onnxModels, + new OnnxModels(deployState.getFileRegistry()), AttributeFields.empty, deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), deployState.getImportedModels(), - deployState.getProperties()); + deployState.getProperties(), + deployState.getExecutor()); HostSystem hostSystem = root.hostSystem(); if (complete) { // create a completed, frozen model configModelRepo.readConfigModels(deployState, this, builder, root, new VespaConfigModelRegistry(configModelRegistry)); addServiceClusters(deployState, builder); setupRouting(deployState); - this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor(); getAdmin().addPerHostServices(hostSystem.getHosts(), deployState); freezeModelTopology(); root.prepare(configModelRepo); @@ -210,14 +209,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri validateWrapExceptions(); hostSystem.dumpPortAllocations(); propagateRestartOnDeploy(); - // must happen after stuff above - this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); - } - else { // create a model with no services instantiated and the given file distributor - this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); - this.fileDistributor = fileDistributor; } + // else: create a model with no services instantiated (no-op) + // must be done last + this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); } @Override @@ -271,12 +267,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; } - public OnnxModels onnxModels() { return onnxModels; } - /** Creates a mutable model with no services instantiated */ public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { - return new VespaModel(new NullConfigModelRegistry(), deployState, false, - new FileDistributor(deployState.getFileRegistry().fileSourceHost())); + return new VespaModel(new NullConfigModelRegistry(), deployState, false); } private void validateWrapExceptions() { @@ -303,8 +296,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri QueryProfiles queryProfiles) { if ( ! importedModels.all().isEmpty()) { // models/ directory is available for (ImportedMlModel model : importedModels.all()) { - onnxModelInfoFromSource(model); - RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry); + // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. + OnnxModels onnxModels = onnxModelInfoFromSource(model); + RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry, onnxModels); rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()), model.name(), profile, queryProfiles.getRegistry(), model); @@ -316,8 +310,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri for (ApplicationFile generatedModelDir : generatedModelsDir.listFiles()) { String modelName = generatedModelDir.getPath().last(); if (modelName.contains(".")) continue; // Name space: Not a global profile - onnxModelInfoFromStore(modelName); - RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry); + // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. + OnnxModels onnxModels = onnxModelInfoFromStore(modelName); + RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry, onnxModels); rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false)); @@ -326,7 +321,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri new Processing().processRankProfiles(deployLogger, rankProfileRegistry, queryProfiles, true, false); } - private void onnxModelInfoFromSource(ImportedMlModel model) { + private OnnxModels onnxModelInfoFromSource(ImportedMlModel model) { + OnnxModels onnxModels = new OnnxModels(fileRegistry); if (model.modelType().equals(ImportedMlModel.ModelType.ONNX)) { String path = model.source(); String applicationPath = this.applicationPackage.getFileReference(Path.fromString("")).toString(); @@ -335,11 +331,14 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } loadOnnxModelInfo(onnxModels, model.name(), path); } + return onnxModels; } - private void onnxModelInfoFromStore(String modelName) { + private OnnxModels onnxModelInfoFromStore(String modelName) { String path = ApplicationPackage.MODELS_DIR.append(modelName + ".onnx").toString(); + OnnxModels onnxModels = new OnnxModels(fileRegistry); loadOnnxModelInfo(onnxModels, modelName, path); + return onnxModels; } private void loadOnnxModelInfo(OnnxModels onnxModels, String name, String path) { @@ -378,14 +377,10 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri .collect(Collectors.toCollection(LinkedHashSet::new)); } - public FileDistributor getFileDistributor() { - return fileDistributor; - } - @Override - public Set<FileReference> fileReferences() { - return fileDistributor.allFilesToSend(); - } + public FileReferencesRepository fileReferencesRepository() { return fileReferencesRepository; } + + public Set<FileReference> fileReferences() { return fileReferencesRepository.allFileReferences(); } /** Returns this models Vespa instance */ public ApplicationConfigProducerRoot getVespa() { return root; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index a2a6ada9093..e7d15ad9742 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; import com.yahoo.config.model.api.ModelContext; @@ -26,7 +26,6 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; -import com.yahoo.vespa.model.filedistribution.FileDistributor; import java.io.Serializable; import java.util.ArrayList; @@ -63,7 +62,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable private LogForwarder.Config logForwarderConfig = null; private boolean logForwarderIncludeAdmin = false; - private ApplicationType applicationType = ApplicationType.DEFAULT; + private final ApplicationType applicationType; public void setLogForwarderConfig(LogForwarder.Config cfg, boolean includeAdmin) { this.logForwarderConfig = cfg; @@ -87,14 +86,15 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable Monitoring monitoring, Metrics metrics, boolean multitenant, - FileDistributionConfigProducer fileDistributionConfigProducer, - boolean isHostedVespa) { + boolean isHostedVespa, + ApplicationType applicationType) { super(parent, "admin"); this.isHostedVespa = isHostedVespa; this.monitoring = monitoring; this.metrics = metrics; this.multitenant = multitenant; - this.fileDistribution = fileDistributionConfigProducer; + this.fileDistribution = new FileDistributionConfigProducer(parent); + this.applicationType = applicationType; } public Configserver getConfigserver() { return defaultConfigserver; } @@ -290,17 +290,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } private void addFileDistribution(HostResource host) { - FileDistributor fileDistributor = fileDistribution.getFileDistributor(); - HostResource hostResource = hostSystem().getHostByHostname(fileDistributor.fileSourceHost()); - if (hostResource == null && ! multitenant) - throw new IllegalArgumentException("Could not find " + fileDistributor.fileSourceHost() + - " in the application's " + hostSystem()); - - FileDistributionConfigProvider configProvider = - new FileDistributionConfigProvider(fileDistribution, - fileDistributor, - host == hostResource, - host.getHost()); + FileDistributionConfigProvider configProvider = new FileDistributionConfigProvider(fileDistribution, host.getHost()); fileDistribution.addFileDistributionConfigProducer(host.getHost(), configProvider); } @@ -336,10 +326,6 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable return multitenant; } - public void setApplicationType(ApplicationType applicationType) { - this.applicationType = applicationType; - } - public ApplicationType getApplicationType() { return applicationType; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMonitoring.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMonitoring.java index e53f92822b5..0f15790aecf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMonitoring.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMonitoring.java @@ -1,7 +1,8 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; import java.util.Objects; +import java.util.Optional; /** * Monitoring service properties @@ -10,14 +11,21 @@ import java.util.Objects; */ public class DefaultMonitoring implements Monitoring { + public static final int DEFAULT_MONITORING_INTERVAL = 1; // in minutes + private static final String DEFAULT_MONITORING_CLUSTER_NAME = "vespa"; + private final Integer interval; private final String clustername; - public DefaultMonitoring(String clustername, Integer interval) { + public DefaultMonitoring() { + this(DEFAULT_MONITORING_CLUSTER_NAME, Optional.empty()); + } + + public DefaultMonitoring(String clustername, Optional<Integer> interval) { Objects.requireNonNull(clustername); Objects.requireNonNull(interval); this.clustername = clustername; - this.interval = interval; + this.interval = interval.orElse(DEFAULT_MONITORING_INTERVAL); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 2d2cfec84f8..a876ba7c3a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -471,6 +471,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); metrics.add(new Metric("content.proton.resource_usage.malloc_arena.max")); + metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.address_space.max")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.multi_value.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); // TODO: Remove in Vespa 8 diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index d22affaf5a3..8e0357c1c05 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -93,7 +93,8 @@ public class QuotaValidator extends Validator { private void throwIfBudgetExceeded(double spend, BigDecimal budget, SystemName systemName) { if (budget.doubleValue() < spend) { - throwBudgetException("Please free up some capacity! This deployment's quota use ($%.2f) exceeds reserved quota ($%.2f)!", spend, budget, systemName); + throw new IllegalArgumentException((systemName.equals(SystemName.Public) ? "" : systemName.value() + ": ") + + "Deployment would make your tenant exceed its quota and has been blocked! Please contact support to update your plan."); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 52dccbe96b5..87d3753ca2e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -1,34 +1,34 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.collections.Pair; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.io.IOUtils; import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; -import com.yahoo.path.Path; +import com.yahoo.searchdefinition.DistributableResource; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.RankExpressionBody; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.yolean.Exceptions; import com.yahoo.system.ProcessExecuter; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.collections.Pair; -import com.yahoo.config.ConfigInstance; import com.yahoo.vespa.config.search.ImportedFieldsConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -36,13 +36,14 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; -import java.util.logging.Logger; -import java.util.logging.Level; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; /** - * Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done + * Validates rank setup for all content clusters (rank-profiles, index-schema, attributes configs), validation is done * by running the binary 'vespa-verify-ranksetup-bin' * * @author vegardh @@ -69,17 +70,17 @@ public class RankSetupValidator extends Validator { for (AbstractSearchCluster cluster : model.getSearchClusters()) { // Skipping rank expression checking for streaming clusters, not implemented yet - if (cluster.isRealtime()) { - IndexedSearchCluster sc = (IndexedSearchCluster) cluster; - String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/"; - for (DocumentDatabase docDb : sc.getDocumentDbs()) { - final String name = docDb.getDerivedConfiguration().getSearch().getName(); - String searchDir = clusterDir + name + "/"; - writeConfigs(searchDir, docDb); - writeExtraVerifyRanksetupConfig(searchDir, docDb); - if ( ! validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) { - return; - } + if (cluster.isStreaming()) continue; + + IndexedSearchCluster sc = (IndexedSearchCluster) cluster; + String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/"; + for (DocumentDatabase docDb : sc.getDocumentDbs()) { + final String name = docDb.getDerivedConfiguration().getSearch().getName(); + String searchDir = clusterDir + name + "/"; + writeConfigs(searchDir, docDb); + writeExtraVerifyRanksetupConfig(searchDir, docDb); + if (!validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) { + return; } } } @@ -95,16 +96,13 @@ public class RankSetupValidator extends Validator { private boolean validate(String configId, SearchCluster searchCluster, String sdName, DeployLogger deployLogger, File tempDir) { Instant start = Instant.now(); try { + log.log(Level.FINE, () -> String.format("Validating schema '%s' for %s with config id %s", sdName, searchCluster, configId)); boolean ret = execValidate(configId, searchCluster, sdName, deployLogger); if (!ret) { // Give up, don't say same error msg repeatedly deleteTempDir(tempDir); } - log.log(Level.FINE, () -> String.format("Validating %s for %s, %s took %s ms", - sdName, - searchCluster, - configId, - Duration.between(start, Instant.now()).toMillis())); + log.log(Level.FINE, () -> String.format("Validation took %s ms", Duration.between(start, Instant.now()).toMillis())); return ret; } catch (IllegalArgumentException e) { deleteTempDir(tempDir); @@ -119,68 +117,58 @@ public class RankSetupValidator extends Validator { private void writeConfigs(String dir, AbstractConfigProducer<?> producer) throws IOException { RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder(); ((RankProfilesConfig.Producer) producer).getConfig(rpcb); - RankProfilesConfig rpc = new RankProfilesConfig(rpcb); - writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc); + writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpcb.build()); IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder(); ((IndexschemaConfig.Producer) producer).getConfig(iscb); - IndexschemaConfig isc = new IndexschemaConfig(iscb); - writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc); + writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", iscb.build()); AttributesConfig.Builder acb = new AttributesConfig.Builder(); ((AttributesConfig.Producer) producer).getConfig(acb); - AttributesConfig ac = new AttributesConfig(acb); - writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac); + writeConfig(dir, AttributesConfig.getDefName() + ".cfg", acb.build()); RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder(); ((RankingConstantsConfig.Producer) producer).getConfig(rccb); - RankingConstantsConfig rcc = new RankingConstantsConfig(rccb); - writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc); + writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rccb.build()); RankingExpressionsConfig.Builder recb = new RankingExpressionsConfig.Builder(); ((RankingExpressionsConfig.Producer) producer).getConfig(recb); - RankingExpressionsConfig rec = new RankingExpressionsConfig(recb); - writeConfig(dir, RankingExpressionsConfig.getDefName() + ".cfg", rec); + writeConfig(dir, RankingExpressionsConfig.getDefName() + ".cfg", recb.build()); OnnxModelsConfig.Builder omcb = new OnnxModelsConfig.Builder(); ((OnnxModelsConfig.Producer) producer).getConfig(omcb); - OnnxModelsConfig omc = new OnnxModelsConfig(omcb); - writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omc); + writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omcb.build()); ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder(); ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb); - ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb); - writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc); + writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifcb.build()); } - private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { - String configName = "verify-ranksetup.cfg"; - String configContent = ""; + private void writeExtraVerifyRanksetupConfig(List<String> config, Collection<? extends DistributableResource> resources) { + for (DistributableResource model : resources) { + String modelPath = getFileRepositoryPath(model.getFilePath().getName(), model.getFileReference()); + int index = config.size() / 2; + config.add(String.format("file[%d].ref \"%s\"", index, model.getFileReference())); + config.add(String.format("file[%d].path \"%s\"", index, modelPath)); + log.log(Level.FINE, index + ": " + model.getPathType() + " -> " + model.getName() + " -> " + modelPath + " -> " + model.getFileReference()); + } + } + private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { List<String> config = new ArrayList<>(); - // Assist verify-ranksetup in finding the actual ONNX model files - for (OnnxModel model : db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()) { - String modelPath = getFileRepositoryPath(model.getFilePath(), model.getFileReference()); - config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, model.getFileReference())); - config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); - } - for (RankExpressionBody expr : db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()) { - String modelPath = getFileRepositoryPath(expr.getFilePath(), expr.getFileReference()); - config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, expr.getFileReference())); - config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); - } + // Assist verify-ranksetup in finding the actual ONNX model files + writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()); + writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()); - if ( ! config.isEmpty() ) { - configContent = StringUtilities.implodeMultiline(config); - } - IOUtils.writeFile(dir + configName, configContent, false); + String configContent = config.isEmpty() ? "" : StringUtilities.implodeMultiline(config); + IOUtils.writeFile(dir + "verify-ranksetup.cfg", configContent, false); } - public static String getFileRepositoryPath(Path path, String fileReference) { + public static String getFileRepositoryPath(String name, String fileReference) { ConfigserverConfig cfg = new ConfigserverConfig(new ConfigserverConfig.Builder()); // assume defaults String fileRefDir = Defaults.getDefaults().underVespaHome(cfg.fileReferencesDir()); - return Paths.get(fileRefDir, fileReference, path.getName()).toString(); + return Paths.get(fileRefDir, fileReference, name).toString(); } private static void writeConfig(String dir, String configName, ConfigInstance config) throws IOException { @@ -203,14 +191,15 @@ public class RankSetupValidator extends Validator { } private void validateWarn(Exception e, DeployLogger deployLogger) { - String msg = "Unable to execute '"+ binaryName + "', validation of rank expressions will only take place when you start Vespa: " + - Exceptions.toMessageString(e); + String msg = "Unable to execute '" + binaryName + + "', validation of rank expressions will only take place when you start Vespa: " + + Exceptions.toMessageString(e); deployLogger.logApplicationPackage(Level.WARNING, msg); } private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) { - StringBuilder errMsg = new StringBuilder("For search cluster '").append(sc.getClusterName()).append("', ") - .append("search definition '").append(sdName).append("': error in rank setup. Details:\n"); + StringBuilder errMsg = new StringBuilder("Error in rank setup in schema '").append(sdName) + .append("' for content cluster '").append(sc.getClusterName()).append("'.").append(" Details:\n"); for (String line : output.split("\n")) { // Remove debug lines from start script if (line.startsWith("debug\t")) continue; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index c2c8b8e86cb..e59c944233d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.application.api.DeployLogger; @@ -19,11 +19,11 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets; import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder; -import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import org.w3c.dom.Element; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * A base class for admin model builders, to support common functionality across versions. @@ -33,20 +33,14 @@ import java.util.List; */ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProducerBuilder<Admin> { - private static final int DEFAULT_INTERVAL = 1; // in minutes - private static final String DEFAULT_CLUSTER_NAME = "vespa"; - private final ApplicationType applicationType; protected final List<ConfigServerSpec> configServerSpecs; - private final String fileSourceHost; protected final boolean multitenant; DomAdminBuilderBase(ApplicationType applicationType, - String fileSourceHost, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { this.applicationType = applicationType; - this.fileSourceHost = fileSourceHost; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; } @@ -71,44 +65,37 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu Monitoring monitoring = getMonitoring(XML.getChild(adminElement,"monitoring"), deployState.isHosted()); Metrics metrics = new MetricsBuilder(applicationType, PredefinedMetricSets.get()) .buildMetrics(XML.getChild(adminElement, "metrics")); - FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent); - - Admin admin = new Admin(parent, monitoring, metrics, multitenant, fileDistributionConfigProducer, deployState.isHosted()); - admin.setApplicationType(applicationType); + Admin admin = new Admin(parent, monitoring, metrics, multitenant, deployState.isHosted(), applicationType); doBuildAdmin(deployState, admin, adminElement); new ModelConfigProvider(admin); return admin; } - private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer<?> parent) { - return new FileDistributionConfigProducer(parent, fileSourceHost); - } - protected abstract void doBuildAdmin(DeployState deployState, Admin admin, Element adminE); private Monitoring getMonitoring(Element monitoringElement, boolean isHosted) { - if (monitoringElement == null) return new DefaultMonitoring(DEFAULT_CLUSTER_NAME, DEFAULT_INTERVAL); + if (monitoringElement == null) return new DefaultMonitoring(); + if (isHosted && applicationType.equals(ApplicationType.DEFAULT)) throw new IllegalArgumentException("The 'monitoring' element cannot be used on hosted Vespa."); - Integer minutes = getMonitoringInterval(monitoringElement); - if (minutes == null) - minutes = DEFAULT_INTERVAL; + Optional<Integer> minutes = getMonitoringInterval(monitoringElement); return new DefaultMonitoring(monitoringElement.getAttribute("systemname"), minutes); } - private Integer getMonitoringInterval(Element monitoringE) { - Integer minutes = null; + private Optional<Integer> getMonitoringInterval(Element monitoringE) { String seconds = monitoringE.getAttribute("interval").trim(); if ( ! seconds.isEmpty()) { - minutes = Integer.parseInt(seconds) / 60; - if (!(minutes == 1 || minutes == 5)) { - throw new IllegalArgumentException("The only allowed values for 'interval' attribute in '" + monitoringE.getTagName() + - "' element is 60 or 300."); + int m = Integer.parseInt(seconds) / 60; + if ( ! List.of(1, 5).contains(m)) { + throw new IllegalArgumentException("The only allowed values for 'interval' attribute in '" + + monitoringE.getTagName() + "' element is 60 or 300."); } + return Optional.of(m); } - return minutes; + + return Optional.empty(); } void addLogForwarders(ModelElement logForwardingElement, Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index ad24a324bbb..04c7db8e701 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -32,10 +32,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private static final String ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK = "standalone-zookeeper"; public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, - String fileSourceHost, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { - super(applicationType, fileSourceHost, multitenant, configServerSpecs); + super(applicationType, multitenant, configServerSpecs); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index ae54e9fecc4..c836c5557ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -36,8 +36,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) { - super(context.getApplicationType(), context.getDeployState().getFileRegistry().fileSourceHost(), multitenant, - configServerSpecs); + super(context.getApplicationType(), multitenant, configServerSpecs); this.containerModels = containerModels; this.context = context; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 02a0f606603..792fa3f1884 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.model.container.component.SimpleComponent; /** * A container that is typically used by container clusters set up from the user application. @@ -30,6 +31,10 @@ public final class ApplicationContainer extends Container implements public ApplicationContainer(AbstractConfigProducer<?> parent, String name, boolean retired, int index, DeployState deployState) { super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); + + addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); + addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); + addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.SessionCache")); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index d0bcfde3c0c..1203497f59f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -71,8 +71,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH); private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH + "/*"); - public static final int heapSizePercentageOfTotalNodeMemory = 60; - public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 17; + public static final int heapSizePercentageOfTotalNodeMemory = 70; + public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 18; private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); @@ -107,6 +107,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.SystemInfoProvider"); addSimpleComponent(com.yahoo.container.core.documentapi.DocumentAccessProvider.class.getName()); + addMetricsHandlers(); addTestrunnerComponentsIfTester(deployState); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index d130b08d083..4bd615cebcf 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -174,7 +174,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); - addComponent(new DefaultThreadpoolProvider(this)); + addComponent(new DefaultThreadpoolProvider(this, deployState.featureFlags().metricsproxyNumThreads())); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent(SecurityFilterInvoker.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java index dc01fcc65b1..4d665b0bb58 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -16,19 +16,21 @@ import com.yahoo.vespa.model.container.component.SimpleComponent; class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolConfig.Producer { private final ContainerCluster<?> cluster; + private final int metricsproxyNumThreads; - DefaultThreadpoolProvider(ContainerCluster<?> cluster) { + DefaultThreadpoolProvider(ContainerCluster<?> cluster, int metricsproxyNumThreads) { super(new ComponentModel( BundleInstantiationSpecification.getFromStrings( "default-threadpool", ThreadPoolProvider.class.getName(), null))); this.cluster = cluster; + this.metricsproxyNumThreads = metricsproxyNumThreads; } private int defaultThreadsByClusterType() { if (cluster instanceof MetricsProxyContainerCluster) { - return 2; + return metricsproxyNumThreads; } return 10; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java index c06f5f4c441..b98cabb6f33 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java @@ -19,6 +19,7 @@ import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.VespaModel; import org.xml.sax.SAXException; @@ -69,10 +70,11 @@ public class ModelsEvaluatorTester { RankProfilesConfig rankProfilesConfig = getRankProfilesConfig(rankProfileList); RankingConstantsConfig rankingConstantsConfig = getRankingConstantConfig(rankProfileList); + RankingExpressionsConfig rankingExpressionsConfig = getRankingExpressionsConfig(rankProfileList); OnnxModelsConfig onnxModelsConfig = getOnnxModelsConfig(rankProfileList); FileAcquirer files = createFileAcquirer(rankingConstantsConfig, onnxModelsConfig, temporaryApplicationDir); - return new ModelsEvaluator(rankProfilesConfig, rankingConstantsConfig, onnxModelsConfig, files); + return new ModelsEvaluator(rankProfilesConfig, rankingConstantsConfig, rankingExpressionsConfig, onnxModelsConfig, files); } catch (IOException | SAXException e) { throw new RuntimeException(e); @@ -104,19 +106,25 @@ public class ModelsEvaluatorTester { private static RankProfilesConfig getRankProfilesConfig(RankProfileList rankProfileList) { RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); rankProfileList.getConfig(builder); - return new RankProfilesConfig(builder); + return builder.build(); } private static RankingConstantsConfig getRankingConstantConfig(RankProfileList rankProfileList) { RankingConstantsConfig.Builder builder = new RankingConstantsConfig.Builder(); rankProfileList.getConfig(builder); - return new RankingConstantsConfig(builder); + return builder.build(); + } + + private static RankingExpressionsConfig getRankingExpressionsConfig(RankProfileList rankProfileList) { + RankingExpressionsConfig.Builder builder = new RankingExpressionsConfig.Builder(); + rankProfileList.getConfig(builder); + return builder.build(); } private static OnnxModelsConfig getOnnxModelsConfig(RankProfileList rankProfileList) { OnnxModelsConfig.Builder builder = new OnnxModelsConfig.Builder(); rankProfileList.getConfig(builder); - return new OnnxModelsConfig(builder); + return builder.build(); } private static FileAcquirer createFileAcquirer(RankingConstantsConfig constantsConfig, OnnxModelsConfig onnxModelsConfig, File appDir) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index efb47e97ccb..5b2b4cc6cba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -153,7 +153,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> String clusterName, ContentSearchCluster search) { List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, deployState); + IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0); isc.setRoutingSelector(clusterElem.childAsString("documents.selection")); Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java index ec677911da6..f0a2e52f6bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; +import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; @@ -14,9 +15,10 @@ import org.w3c.dom.Element; /** * Represents specific configuration for a given distributor node. */ -public class Distributor extends ContentNode { +public class Distributor extends ContentNode implements StorDistributormanagerConfig.Producer { PersistenceEngine provider; + private final int numDistributorStripesFlag; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<Distributor> { ModelElement clusterXml; @@ -39,12 +41,27 @@ public class Distributor extends ContentNode { StorageNode.rootFolder + parent.getClusterName() + "/distributor/" + distributionKey, distributionKey); this.provider = provider; + this.numDistributorStripesFlag = properties.featureFlags().numDistributorStripes(); if (distributorBasePort != null) { setBasePort(distributorBasePort); } } + private int tuneNumDistributorStripes() { + if ((numDistributorStripesFlag == -1) && (getHostResource() != null)) { + int cores = (int)getHostResource().realResources().vcpu(); + if (cores <= 16) { + return 1; + } else if (cores <= 64) { + return 2; + } else { + return 4; + } + } + return numDistributorStripesFlag; + } + @Override public void getConfig(StorServerConfig.Builder builder) { super.getConfig(builder); @@ -59,6 +76,11 @@ public class Distributor extends ContentNode { } @Override + public void getConfig(StorDistributormanagerConfig.Builder builder) { + builder.num_distributor_stripes(tuneNumDistributorStripes()); + } + + @Override public String getStartupCommand() { return "exec sbin/vespa-distributord -c $VESPA_CONFIG_ID"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 390c939c4b4..0e3fee5a749 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -43,7 +43,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean hasIndexedDocumentType; private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; - private final int numDistributorStripes; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -106,20 +105,18 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); - int numDistributorStripes = deployState.getProperties().featureFlags().numDistributorStripes(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, numDistributorStripes); + maxInhibitedGroups); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, - int maxActivationInhibitedOutOfSyncGroups, - int numDistributorStripes) + int maxActivationInhibitedOutOfSyncGroups) { super(parent, "distributor"); this.parent = parent; @@ -128,7 +125,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.hasIndexedDocumentType = hasIndexedDocumentType; this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; - this.numDistributorStripes = numDistributorStripes; } @Override @@ -142,7 +138,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.disable_bucket_activation(hasIndexedDocumentType == false); builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); - builder.num_distributor_stripes(numDistributorStripes); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java index a12e183b409..cd9a0c89bf8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java @@ -37,8 +37,7 @@ public class ResourceLimits implements FleetcontrollerConfig.Producer, ProtonCon // storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp builder.cluster_feed_block_limit.put("memory", memoryLimit.orElse(0.8)); builder.cluster_feed_block_limit.put("disk", diskLimit.orElse(0.8)); - builder.cluster_feed_block_limit.put("attribute-enum-store", 0.89); - builder.cluster_feed_block_limit.put("attribute-multi-value", 0.89); + builder.cluster_feed_block_limit.put("attribute-address-space", 0.89); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index c764b5ab449..0997f29729f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -241,7 +241,8 @@ public class StorageGroup { int minNodesPerGroup = (int)Math.ceil((double)nodesSpec.minResources().nodes() / nodesSpec.minResources().groups()); if (minNodesPerGroup < redundancy) { // TODO: Fail on this on Vespa 8, and simplify - context.getDeployLogger().logApplicationPackage(Level.WARNING, + context.getDeployLogger() + .logApplicationPackage(Level.WARNING, "Cluster '" + clusterElement.stringAttribute("id") + "' " + "specifies redundancy " + redundancy + " but cannot be higher than " + "the minimum nodes per group, which is " + minNodesPerGroup); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index fabcbbcc9ec..ad8948153cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -325,13 +325,13 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce } } - public static final NodeResources clusterControllerResources = new NodeResources(0.5, 2, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); + public static final NodeResources clusterControllerResources = new NodeResources(0.25, 1, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private ClusterControllerContainerCluster getDedicatedSharedControllers(ModelElement contentElement, Admin admin, ConfigModelContext context, DeployState deployState) { if (admin.getClusterControllers() == null) { NodesSpecification spec = NodesSpecification.requiredFromSharedParents(deployState.zone().environment().isProduction() ? 3 : 1, - deployState.featureFlags().dedicatedClusterControllerFlavor().orElse(clusterControllerResources), + clusterControllerResources, contentElement, context); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java index d29a2611ac7..2e1902cb1ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.filedistribution; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -15,19 +15,9 @@ import java.util.Map; public class FileDistributionConfigProducer extends AbstractConfigProducer<AbstractConfigProducer<?>> { private final Map<Host, FileDistributionConfigProvider> fileDistributionConfigProviders = new IdentityHashMap<>(); - private final FileDistributor fileDistributor; - public FileDistributionConfigProducer(AbstractConfigProducer<?> ancestor, String fileSourceHost) { - this(ancestor, new FileDistributor(fileSourceHost)); - } - - private FileDistributionConfigProducer(AbstractConfigProducer<?> parent, FileDistributor fileDistributor) { + public FileDistributionConfigProducer(AbstractConfigProducer<?> parent) { super(parent, "filedistribution"); - this.fileDistributor = fileDistributor; - } - - public FileDistributor getFileDistributor() { - return fileDistributor; } public void addFileDistributionConfigProducer(Host host, FileDistributionConfigProvider fileDistributionConfigProvider) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java index 47ee546181a..314c69a6e09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java @@ -1,29 +1,17 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.filedistribution; import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; -import com.yahoo.cloud.config.filedistribution.FilereferencesConfig; -import com.yahoo.config.FileReference; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.ConfigProxy; import com.yahoo.vespa.model.Host; -import java.util.Collection; +public class FileDistributionConfigProvider extends AbstractConfigProducer<AbstractConfigProducer<?>> implements FiledistributorrpcConfig.Producer { -public class FileDistributionConfigProvider extends AbstractConfigProducer - implements FiledistributorrpcConfig.Producer, FilereferencesConfig.Producer { - - private final FileDistributor fileDistributor; - private final boolean sendAllFiles; private final Host host; - public FileDistributionConfigProvider(AbstractConfigProducer parent, - FileDistributor fileDistributor, - boolean sendAllFiles, - Host host) { + public FileDistributionConfigProvider(AbstractConfigProducer<?> parent, Host host) { super(parent, host.getHostname()); - this.fileDistributor = fileDistributor; - this.sendAllFiles = sendAllFiles; this.host = host; } @@ -32,18 +20,4 @@ public class FileDistributionConfigProvider extends AbstractConfigProducer builder.connectionspec("tcp/" + host.getHostname() + ":" + ConfigProxy.BASEPORT); } - @Override - public void getConfig(FilereferencesConfig.Builder builder) { - for (FileReference reference : getFileReferences()) { - builder.filereferences(reference.value()); - } - } - - private Collection<FileReference> getFileReferences() { - if (sendAllFiles) { - return fileDistributor.allFilesToSend(); - } else { - return fileDistributor.filesToSendToHost(host); - } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java deleted file mode 100644 index b2e9cfeec85..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.filedistribution; - -import com.yahoo.config.FileReference; -import com.yahoo.vespa.model.Host; - -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - - -/** - * Sends RPC requests to hosts (tenant hosts and config servers) asking them to start download of files. This is used - * during prepare of an application. Services themselves will also request files, the methods in this class are used - * so that hosts can start downloading files before services gets new config that needs these files. It also tries - * to make sure that all config servers (not just the one where the application was deployed) have the files available. - * - * @author Tony Vaagenes - */ -public class FileDistributor { - - private final String fileSourceHost; - - /** A map from file reference to the hosts to which that file reference should be distributed */ - private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>(); - - public FileDistributor(String fileSourceHost) { - this.fileSourceHost = fileSourceHost; - } - - public void sendFileReference(FileReference reference, Host host) { - filesToHosts.computeIfAbsent(reference, k -> new HashSet<>()).add(host); - } - - /** Returns the files which has been marked for distribution to the given host */ - public Set<FileReference> filesToSendToHost(Host host) { - Set<FileReference> files = new HashSet<>(); - - for (Map.Entry<FileReference,Set<Host>> e : filesToHosts.entrySet()) { - if (e.getValue().contains(host)) { - files.add(e.getKey()); - } - } - return files; - } - - public Set<Host> getTargetHosts() { - Set<Host> hosts = new HashSet<>(); - for (Set<Host> hostSubset: filesToHosts.values()) - hosts.addAll(hostSubset); - return hosts; - } - - /** Returns the host which is the source of the files */ - public String fileSourceHost() { - return fileSourceHost; - } - - public Set<FileReference> allFilesToSend() { - return Set.copyOf(filesToHosts.keySet()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java new file mode 100644 index 00000000000..bdda9cdf582 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.config.FileReference; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Keeps track of what files to send with file distribution + * + * @author Tony Vaagenes + * @author hmusum + */ +public class FileReferencesRepository { + + /** A set of file references that should be distributed */ + private final Set<FileReference> fileReferences = new LinkedHashSet<>(); + + public FileReferencesRepository() { } + + public void add(FileReference reference) { fileReferences.add(reference); } + + public Set<FileReference> allFileReferences() { return Set.copyOf(fileReferences); } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index c99549e82e9..95021dca736 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.content.DispatchSpec; import com.yahoo.vespa.model.content.SearchCoverage; -import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -63,7 +62,7 @@ public class IndexedSearchCluster extends SearchCluster return routingSelector; } - public IndexedSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, int index, DeployState deployState) { + public IndexedSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, int index) { super(parent, clusterName, index); unionCfg = new UnionConfiguration(this, documentDbs); rootDispatch = new DispatchGroup(this); @@ -204,7 +203,8 @@ public class IndexedSearchCluster extends SearchCluster deployState.getProperties(), deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels())); + deployState.getImportedModels(), + deployState.getExecutor())); // TODO: remove explicit adding of user configs when the complete content model is built using builders. db.mergeUserConfigs(spec.getUserConfigs()); documentDbs.add(db); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 890e42eb60c..330c7768321 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -12,8 +12,6 @@ import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; -import java.io.File; -import java.io.IOException; import java.util.List; /** @@ -99,7 +97,8 @@ public class StreamingSearchCluster extends SearchCluster implements deployState.getProperties(), deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels()); + deployState.getImportedModels(), + deployState.getExecutor()); } @Override public DerivedConfiguration getSdConfig() { |