diff options
Diffstat (limited to 'config-model')
74 files changed, 757 insertions, 493 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index 3d3ebc90890..23ea5775caa 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -84,7 +84,6 @@ public class AdminModel extends ConfigModel { AbstractConfigProducer<?> parent = modelContext.getParentProducer(); ModelContext.Properties properties = modelContext.getDeployState().getProperties(); DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), - modelContext.getDeployState().getFileRegistry().fileSourceHost(), properties.multitenant(), properties.configServerSpecs()); model.admin = domBuilder.build(modelContext.getDeployState(), parent, adminElement); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 220599b6d1c..249ca71117a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -5,6 +5,7 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; @@ -55,6 +56,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutorService; import java.util.logging.Level; /** @@ -87,6 +89,7 @@ public class DeployState implements ConfigDefinitionStore { private final HostProvisioner provisioner; private final Provisioned provisioned; private final Reindexing reindexing; + private final ExecutorService executor; public static DeployState createTestState() { return new Builder().build(); @@ -104,6 +107,7 @@ public class DeployState implements ConfigDefinitionStore { SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, FileRegistry fileRegistry, + ExecutorService executor, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, Provisioned provisioned, @@ -124,6 +128,7 @@ public class DeployState implements ConfigDefinitionStore { Reindexing reindexing) { this.logger = deployLogger; this.fileRegistry = fileRegistry; + this.executor = executor; this.rankProfileRegistry = rankProfileRegistry; this.applicationPackage = applicationPackage; this.properties = properties; @@ -282,6 +287,8 @@ public class DeployState implements ConfigDefinitionStore { /** The (machine learned) models imported from the models/ directory, as an unmodifiable map indexed by model name */ public ImportedMlModels getImportedModels() { return importedModels; } + public ExecutorService getExecutor() { return executor; } + public Version getWantedNodeVespaVersion() { return wantedNodeVespaVersion; } public Optional<DockerImage> getWantedDockerImageRepo() { return wantedDockerImageRepo; } @@ -312,6 +319,7 @@ public class DeployState implements ConfigDefinitionStore { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); private FileRegistry fileRegistry = new MockFileRegistry(); + private ExecutorService executor = new InThreadExecutorService(); private DeployLogger logger = new BaseDeployLogger(); private Optional<HostProvisioner> hostProvisioner = Optional.empty(); private Provisioned provisioned = new Provisioned(); @@ -329,6 +337,8 @@ public class DeployState implements ConfigDefinitionStore { private Optional<DockerImage> wantedDockerImageRepo = Optional.empty(); private Reindexing reindexing = null; + public Builder() {} + public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; return this; @@ -339,6 +349,11 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder executor(ExecutorService executor) { + this.executor = executor; + return this; + } + public Builder deployLogger(DeployLogger logger) { this.logger = logger; return this; @@ -433,6 +448,7 @@ public class DeployState implements ConfigDefinitionStore { searchDocumentModel, rankProfileRegistry, fileRegistry, + executor, logger, hostProvisioner, provisioned, diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 7d8932b9d00..f4f3ab53bc6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -47,19 +47,18 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int responseNumThreads = 2; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; - private ApplicationRoles applicationRoles; private Quota quota = Quota.unlimited(); private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; private boolean enableFeedBlockInDistributor = true; - private boolean useExternalRankExpression = false; + private boolean useExternalRankExpression = true; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int numDistributorStripes = 0; private int maxConcurrentMergesPerNode = 16; private int maxMergeQueueSize = 1024; - private int largeRankExpressionLimit = 0x10000; + private int largeRankExpressionLimit = 8192; private boolean allowDisableMtls = true; private boolean dryRunOnnxOnSetup = false; private List<X509Certificate> operatorCertificates = Collections.emptyList(); @@ -86,7 +85,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; } @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } - @Override public Optional<ApplicationRoles> applicationRoles() { return Optional.ofNullable(applicationRoles); } @Override public String responseSequencerType() { return responseSequencerType; } @Override public int defaultNumResponseThreads() { return responseNumThreads; } @Override public boolean skipCommunicationManagerThread() { return false; } @@ -111,6 +109,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } + @Override public int metricsproxyNumThreads() { return 1; } public TestProperties setDryRunOnnxOnSetup(boolean value) { dryRunOnnxOnSetup = value; @@ -214,11 +213,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setApplicationRoles(ApplicationRoles applicationRoles) { - this.applicationRoles = applicationRoles; - return this; - } - public TestProperties setQuota(Quota quota) { this.quota = quota; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 12b6f902770..753db89af1b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.test; import com.yahoo.config.ConfigInstance; @@ -17,7 +17,7 @@ import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; -import com.yahoo.vespa.model.filedistribution.FileDistributor; +import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -43,7 +43,7 @@ public class MockRoot extends AbstractConfigProducerRoot { private final HostSystem hostSystem; private final DeployState deployState; - private final FileDistributor fileDistributor; + private final FileReferencesRepository fileReferencesRepository; private Admin admin; public MockRoot() { @@ -66,7 +66,7 @@ public class MockRoot extends AbstractConfigProducerRoot { super(rootConfigId); hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner(), deployState.getDeployLogger()); this.deployState = deployState; - fileDistributor = new FileDistributor(deployState.getFileRegistry().fileSourceHost()); + fileReferencesRepository = new FileReferencesRepository(); } public FileDistributionConfigProducer getFileDistributionConfigProducer() { @@ -119,15 +119,11 @@ public class MockRoot extends AbstractConfigProducerRoot { return deployState; } - public FileDistributor getFileDistributor() { - return fileDistributor; - } + public FileReferencesRepository fileReferencesRepository() { return fileReferencesRepository; } - public HostSystem hostSystem() { - return hostSystem; - } + public HostSystem hostSystem() { return hostSystem; } - public void addDescendant(String configId, AbstractConfigProducer descendant) { + public void addDescendant(String configId, AbstractConfigProducer<?> descendant) { if (id2producer.containsKey(configId)) { throw new RuntimeException ("Config ID '" + configId + "' cannot be reserved by an instance of class '" + @@ -138,7 +134,7 @@ public class MockRoot extends AbstractConfigProducerRoot { } @Override - public void addChild(AbstractConfigProducer abstractConfigProducer) { + public void addChild(AbstractConfigProducer<?> abstractConfigProducer) { super.addChild(abstractConfigProducer); } @@ -149,8 +145,7 @@ public class MockRoot extends AbstractConfigProducerRoot { try { Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); - setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry().fileSourceHost(), - false, new ArrayList<>()). + setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, false, new ArrayList<>()). build(deployState, this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); } catch (SAXException | IOException e) { throw new RuntimeException(e); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index e31d90c2800..1719ea72cb0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -38,6 +38,7 @@ public class DistributableResource { Objects.requireNonNull(blob, "Blob cannot be null"); this.name = name; this.blob = blob; + path = name + ".lz4"; pathType = PathType.BLOB; } @@ -87,7 +88,7 @@ public class DistributableResource { fileReference = fileRegistry.addUri(path); break; case BLOB: - fileReference = fileRegistry.addBlob(blob); + fileReference = fileRegistry.addBlob(path, blob); break; default: throw new IllegalArgumentException("Unknown path type " + pathType); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index 0d10932c333..41d7519b784 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -4,7 +4,6 @@ package com.yahoo.searchdefinition; import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Optional; -import java.util.regex.Pattern; /** * Utility methods for query, document and constant rank feature names @@ -12,9 +11,6 @@ import java.util.regex.Pattern; * @author bratseth */ public class FeatureNames { - - private static final Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); - public static Reference asConstantFeature(String constantName) { return Reference.simple("constant", quoteIfNecessary(constantName)); } @@ -46,7 +42,7 @@ public class FeatureNames { */ public static Optional<String> argumentOf(String feature) { Optional<Reference> reference = Reference.simple(feature); - if ( ! reference.isPresent()) return Optional.empty(); + if ( reference.isEmpty()) return Optional.empty(); if ( ! ( reference.get().name().equals("attribute") || reference.get().name().equals("constant") || reference.get().name().equals("query"))) @@ -56,10 +52,28 @@ public class FeatureNames { } private static String quoteIfNecessary(String s) { - if (identifierRegexp.matcher(s).matches()) + if (notNeedQuotes(s)) return s; else return "\"" + s + "\""; } + static boolean notNeedQuotes(String s) { + // Faster version of the regexp [A-Za-z0-9_][A-Za-z0-9_-]* + if (s.isEmpty()) return false; + if ( ! isValidFirst(s.charAt(0))) return false; + for (int i = 1; i < s.length(); i++) { + if (!isValidAny(s.charAt(i))) return false; + } + return true; + } + private static boolean isValidFirst(char c) { + // [A-Za-z0-9_] + return (c == '_') || ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || ((c >= '0') && (c <= '9')); + } + private static boolean isValidAny(char c) { + // [A-Za-z0-9_-]* + return c == '-' || isValidFirst(c); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java index 9d8e8b90ceb..90dc40e7b93 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java @@ -21,11 +21,14 @@ public class LargeRankExpressions { expression.register(fileRegistry); String name = expression.getName(); if (expressions.containsKey(name)) { - throw new IllegalArgumentException("Rank expression '" + name + - "' defined twice. Previous blob with " + expressions.get(name).getBlob().remaining() + - " bytes, while current has " + expression.getBlob().remaining() + " bytes"); + if ( ! expressions.get(name).getBlob().equals(expression.getBlob())) { + throw new IllegalArgumentException("Rank expression '" + name + + "' defined twice. Previous blob with " + expressions.get(name).getBlob().remaining() + + " bytes, while current has " + expression.getBlob().remaining() + " bytes"); + } + } else { + expressions.put(name, expression); } - expressions.put(name, expression); } /** Returns the ranking constant with the given name, or null if not present */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 3c1f00f1252..47d95cba516 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -104,6 +104,8 @@ public class RankProfile implements Cloneable { private Boolean ignoreDefaultRankFeatures = null; private Map<String, RankingExpressionFunction> functions = new LinkedHashMap<>(); + // This cache must be invalidated every time modifications are done to 'functions'. + private Map<String, RankingExpressionFunction> allFunctionsCached = null; private Map<Reference, TensorType> inputFeatures = new LinkedHashMap<>(); @@ -120,6 +122,9 @@ public class RankProfile implements Cloneable { private List<ImmutableSDField> allFieldsList; + /** Global onnx models not tied to a search definition */ + private final OnnxModels onnxModels; + /** * Creates a new rank profile for a particular search definition * @@ -132,6 +137,7 @@ public class RankProfile implements Cloneable { this.name = Objects.requireNonNull(name, "name cannot be null"); this.search = Objects.requireNonNull(search, "search cannot be null"); this.model = null; + this.onnxModels = null; this.rankProfileRegistry = rankProfileRegistry; } @@ -141,11 +147,12 @@ public class RankProfile implements Cloneable { * @param name the name of the new profile * @param model the model owning this profile */ - public RankProfile(String name, VespaModel model, RankProfileRegistry rankProfileRegistry) { + public RankProfile(String name, VespaModel model, RankProfileRegistry rankProfileRegistry, OnnxModels onnxModels) { this.name = Objects.requireNonNull(name, "name cannot be null"); this.search = null; this.model = Objects.requireNonNull(model, "model cannot be null"); this.rankProfileRegistry = rankProfileRegistry; + this.onnxModels = onnxModels; } public String getName() { return name; } @@ -164,7 +171,7 @@ public class RankProfile implements Cloneable { } public Map<String, OnnxModel> onnxModels() { - return search != null ? search.onnxModels().asMap() : model.onnxModels().asMap(); + return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); } private Stream<ImmutableSDField> allFields() { @@ -580,6 +587,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); functions.put(function.getName(), rankingExpressionFunction); + allFunctionsCached = null; return rankingExpressionFunction; } @@ -608,6 +616,12 @@ public class RankProfile implements Cloneable { } /** Returns an unmodifiable snapshot of the functions in this */ public Map<String, RankingExpressionFunction> getFunctions() { + if (needToUpdateFunctionCache()) { + allFunctionsCached = gatherAllFunctions(); + } + return allFunctionsCached; + } + private Map<String, RankingExpressionFunction> gatherAllFunctions() { if (functions.isEmpty() && getInherited() == null) return Collections.emptyMap(); if (functions.isEmpty()) return getInherited().getFunctions(); if (getInherited() == null) return Collections.unmodifiableMap(new LinkedHashMap<>(functions)); @@ -618,6 +632,12 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableMap(allFunctions); } + private boolean needToUpdateFunctionCache() { + if (getInherited() != null) + return (allFunctionsCached == null) || getInherited().needToUpdateFunctionCache(); + return allFunctionsCached == null; + } + public int getKeepRankCount() { if (keepRankCount >= 0) return keepRankCount; if (getInherited() != null) return getInherited().getKeepRankCount(); @@ -706,6 +726,7 @@ public class RankProfile implements Cloneable { clone.rankProperties = new LinkedHashMap<>(this.rankProperties); clone.inputFeatures = new LinkedHashMap<>(this.inputFeatures); clone.functions = new LinkedHashMap<>(this.functions); + clone.allFunctionsCached = null; clone.filterFields = new HashSet<>(this.filterFields); clone.constants = new HashMap<>(this.constants); return clone; @@ -745,6 +766,7 @@ public class RankProfile implements Cloneable { // Function compiling second pass: compile all functions and insert previously compiled inline functions // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); + allFunctionsCached = null; } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 625c584f79a..f49a477c66b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -527,5 +527,7 @@ public class SearchBuilder { public QueryProfileRegistry getQueryProfileRegistry() { return queryProfileRegistry; } + public ModelContext.Properties getProperties() { return properties; } + public DeployLogger getDeployLogger() { return deployLogger; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 16de8fd4b9e..9764d63a804 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -19,7 +20,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import java.io.IOException; import java.io.Writer; -import java.util.logging.Level; +import java.util.concurrent.ExecutorService; /** * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when @@ -52,8 +53,12 @@ public class DerivedConfiguration { * modified. * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} */ - DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { - this(search, new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfiles, importedModels); + public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry) { + this(search, rankProfileRegistry, new QueryProfileRegistry()); + } + + DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { + this(search, new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfiles, new ImportedMlModels(), new InThreadExecutorService()); } /** @@ -72,7 +77,8 @@ public class DerivedConfiguration { ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels) { + ImportedMlModels importedModels, + ExecutorService executor) { Validator.ensureNotNull("Search definition", search); this.search = search; this.queryProfiles = queryProfiles; @@ -87,7 +93,7 @@ public class DerivedConfiguration { juniperrc = new Juniperrc(search); rankProfileList = new RankProfileList(search, search.rankingConstants(), search.rankExpressionFiles(), search.onnxModels(), attributeFields, rankProfileRegistry, - queryProfiles, importedModels, deployProperties); + queryProfiles, importedModels, deployProperties, executor); indexingScript = new IndexingScript(search); indexInfo = new IndexInfo(search); indexSchema = new IndexSchema(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 7f19fcd24d4..6528c9195d4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -3,7 +3,6 @@ package com.yahoo.searchdefinition.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; @@ -19,8 +18,13 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.AbstractService; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.logging.Logger; /** @@ -61,13 +65,14 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - ModelContext.Properties deployProperties) { + ModelContext.Properties deployProperties, + ExecutorService executor) { setName(search == null ? "default" : search.getName()); this.rankingConstants = rankingConstants; this.largeRankExpressions = largeRankExpressions; this.onnxModels = onnxModels; // as ONNX models come from parsing rank expressions dryRunOnnxOnSetup = deployProperties.featureFlags().dryRunOnnxOnSetup(); - deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties); + deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties, executor); } private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, @@ -75,12 +80,12 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ ImportedMlModels importedModels, Search search, AttributeFields attributeFields, - ModelContext.Properties deployProperties) { + ModelContext.Properties deployProperties, + ExecutorService executor) { + List<Future<RawRankProfile>> futureRawRankProfiles = new ArrayList<>(); if (search != null) { // profiles belonging to a search have a default profile - RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.get(search, "default"), - largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties); - rankProfiles.put(defaultProfile.getName(), defaultProfile); + futureRawRankProfiles.add(executor.submit(() -> new RawRankProfile(rankProfileRegistry.get(search, "default"), + largeRankExpressions, queryProfiles, importedModels, attributeFields, deployProperties))); } for (RankProfile rank : rankProfileRegistry.rankProfilesOf(search)) { @@ -89,9 +94,16 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ this.onnxModels.add(rank.onnxModels()); } - RawRankProfile rawRank = new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties); - rankProfiles.put(rawRank.getName(), rawRank); + futureRawRankProfiles.add(executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties))); + } + try { + for (Future<RawRankProfile> rawFuture : futureRawRankProfiles) { + RawRankProfile rawRank = rawFuture.get(); + rankProfiles.put(rawRank.getName(), rawRank); + } + } catch (InterruptedException | ExecutionException e) { + throw new IllegalStateException(e); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index fee73a03ae6..792965314c5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -6,7 +6,6 @@ import com.google.common.collect.ImmutableList; import com.yahoo.collections.Pair; import com.yahoo.compress.Compressor; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.LargeRankExpressions; @@ -324,7 +323,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } /** Derives the properties this produces */ - public List<Pair<String, String>> derive(LargeRankExpressions largeRankExpressions) { + public List<Pair<String, String>> derive(LargeRankExpressions largeRankExpressions) { List<Pair<String, String>> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { if (RankingExpression.propertyName(RankProfile.FIRST_PHASE).equals(property.getName())) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 549f99f9577..e47fd339505 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -459,10 +459,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * @param reference file reference (hash) */ public void send(FileReference reference) { - Host host = null; - if (getHost() != null) // false when running application tests without hosts - host = getHost().getHost(); - getRoot().getFileDistributor().sendFileReference(reference, host); + getRoot().fileReferencesRepository().add(reference); } /** The service HTTP port for health status */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java index b30aa5aeca6..e9b587c0eb9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java @@ -1,10 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.filedistribution.FileDistributor; +import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; import java.util.Set; @@ -22,7 +22,7 @@ public interface ConfigProducerRoot extends ConfigProducer { * @param id string id of descendant * @param descendant the producer to add to this root node */ - void addDescendant(String id, AbstractConfigProducer descendant); + void addDescendant(String id, AbstractConfigProducer<?> descendant); /** * @return an unmodifiable copy of the set of configIds in this root. @@ -39,7 +39,7 @@ public interface ConfigProducerRoot extends ConfigProducer { */ public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId); - FileDistributor getFileDistributor(); + FileReferencesRepository fileReferencesRepository(); Admin getAdmin(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 15377a95e3c..e6687ec8ac8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -55,7 +55,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; -import com.yahoo.vespa.model.filedistribution.FileDistributor; +import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; import com.yahoo.vespa.model.generic.service.ServiceCluster; import com.yahoo.vespa.model.ml.ConvertedModel; import com.yahoo.vespa.model.ml.ModelName; @@ -130,13 +130,12 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** Large rank expression files of this */ private final LargeRankExpressions largeRankExpressions; - /** Large rank expression files of this */ - private final OnnxModels onnxModels; + private final FileRegistry fileRegistry; /** The validation overrides of this. This is never null. */ private final ValidationOverrides validationOverrides; - private final FileDistributor fileDistributor; + private final FileReferencesRepository fileReferencesRepository = new FileReferencesRepository(); private final Provisioned provisioned; @@ -169,16 +168,16 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * @param deployState the global deploy state to use for this model. */ public VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState) throws IOException, SAXException { - this(configModelRegistry, deployState, true, null); + this(configModelRegistry, deployState, true); } - private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor) + private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete) throws IOException, SAXException { super("vespamodel"); version = deployState.getVespaVersion(); + fileRegistry = deployState.getFileRegistry(); largeRankExpressions = new LargeRankExpressions(deployState.getFileRegistry()); rankingConstants = new RankingConstants(deployState.getFileRegistry()); - onnxModels = new OnnxModels(deployState.getFileRegistry()); validationOverrides = deployState.validationOverrides(); applicationPackage = deployState.getApplicationPackage(); provisioned = deployState.provisioned(); @@ -190,19 +189,19 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri rankProfileList = new RankProfileList(null, // null search -> global rankingConstants, largeRankExpressions, - onnxModels, + new OnnxModels(deployState.getFileRegistry()), AttributeFields.empty, deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), deployState.getImportedModels(), - deployState.getProperties()); + deployState.getProperties(), + deployState.getExecutor()); HostSystem hostSystem = root.hostSystem(); if (complete) { // create a completed, frozen model configModelRepo.readConfigModels(deployState, this, builder, root, new VespaConfigModelRegistry(configModelRegistry)); addServiceClusters(deployState, builder); setupRouting(deployState); - this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor(); getAdmin().addPerHostServices(hostSystem.getHosts(), deployState); freezeModelTopology(); root.prepare(configModelRepo); @@ -210,14 +209,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri validateWrapExceptions(); hostSystem.dumpPortAllocations(); propagateRestartOnDeploy(); - // must happen after stuff above - this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); - } - else { // create a model with no services instantiated and the given file distributor - this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); - this.fileDistributor = fileDistributor; } + // else: create a model with no services instantiated (no-op) + // must be done last + this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); } @Override @@ -271,12 +267,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; } - public OnnxModels onnxModels() { return onnxModels; } - /** Creates a mutable model with no services instantiated */ public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { - return new VespaModel(new NullConfigModelRegistry(), deployState, false, - new FileDistributor(deployState.getFileRegistry().fileSourceHost())); + return new VespaModel(new NullConfigModelRegistry(), deployState, false); } private void validateWrapExceptions() { @@ -303,8 +296,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri QueryProfiles queryProfiles) { if ( ! importedModels.all().isEmpty()) { // models/ directory is available for (ImportedMlModel model : importedModels.all()) { - onnxModelInfoFromSource(model); - RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry); + // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. + OnnxModels onnxModels = onnxModelInfoFromSource(model); + RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry, onnxModels); rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()), model.name(), profile, queryProfiles.getRegistry(), model); @@ -316,8 +310,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri for (ApplicationFile generatedModelDir : generatedModelsDir.listFiles()) { String modelName = generatedModelDir.getPath().last(); if (modelName.contains(".")) continue; // Name space: Not a global profile - onnxModelInfoFromStore(modelName); - RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry); + // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. + OnnxModels onnxModels = onnxModelInfoFromStore(modelName); + RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry, onnxModels); rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false)); @@ -326,7 +321,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri new Processing().processRankProfiles(deployLogger, rankProfileRegistry, queryProfiles, true, false); } - private void onnxModelInfoFromSource(ImportedMlModel model) { + private OnnxModels onnxModelInfoFromSource(ImportedMlModel model) { + OnnxModels onnxModels = new OnnxModels(fileRegistry); if (model.modelType().equals(ImportedMlModel.ModelType.ONNX)) { String path = model.source(); String applicationPath = this.applicationPackage.getFileReference(Path.fromString("")).toString(); @@ -335,11 +331,14 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } loadOnnxModelInfo(onnxModels, model.name(), path); } + return onnxModels; } - private void onnxModelInfoFromStore(String modelName) { + private OnnxModels onnxModelInfoFromStore(String modelName) { String path = ApplicationPackage.MODELS_DIR.append(modelName + ".onnx").toString(); + OnnxModels onnxModels = new OnnxModels(fileRegistry); loadOnnxModelInfo(onnxModels, modelName, path); + return onnxModels; } private void loadOnnxModelInfo(OnnxModels onnxModels, String name, String path) { @@ -378,14 +377,10 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri .collect(Collectors.toCollection(LinkedHashSet::new)); } - public FileDistributor getFileDistributor() { - return fileDistributor; - } - @Override - public Set<FileReference> fileReferences() { - return fileDistributor.allFilesToSend(); - } + public FileReferencesRepository fileReferencesRepository() { return fileReferencesRepository; } + + public Set<FileReference> fileReferences() { return fileReferencesRepository.allFileReferences(); } /** Returns this models Vespa instance */ public ApplicationConfigProducerRoot getVespa() { return root; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index a2a6ada9093..e7d15ad9742 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; import com.yahoo.config.model.api.ModelContext; @@ -26,7 +26,6 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; -import com.yahoo.vespa.model.filedistribution.FileDistributor; import java.io.Serializable; import java.util.ArrayList; @@ -63,7 +62,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable private LogForwarder.Config logForwarderConfig = null; private boolean logForwarderIncludeAdmin = false; - private ApplicationType applicationType = ApplicationType.DEFAULT; + private final ApplicationType applicationType; public void setLogForwarderConfig(LogForwarder.Config cfg, boolean includeAdmin) { this.logForwarderConfig = cfg; @@ -87,14 +86,15 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable Monitoring monitoring, Metrics metrics, boolean multitenant, - FileDistributionConfigProducer fileDistributionConfigProducer, - boolean isHostedVespa) { + boolean isHostedVespa, + ApplicationType applicationType) { super(parent, "admin"); this.isHostedVespa = isHostedVespa; this.monitoring = monitoring; this.metrics = metrics; this.multitenant = multitenant; - this.fileDistribution = fileDistributionConfigProducer; + this.fileDistribution = new FileDistributionConfigProducer(parent); + this.applicationType = applicationType; } public Configserver getConfigserver() { return defaultConfigserver; } @@ -290,17 +290,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } private void addFileDistribution(HostResource host) { - FileDistributor fileDistributor = fileDistribution.getFileDistributor(); - HostResource hostResource = hostSystem().getHostByHostname(fileDistributor.fileSourceHost()); - if (hostResource == null && ! multitenant) - throw new IllegalArgumentException("Could not find " + fileDistributor.fileSourceHost() + - " in the application's " + hostSystem()); - - FileDistributionConfigProvider configProvider = - new FileDistributionConfigProvider(fileDistribution, - fileDistributor, - host == hostResource, - host.getHost()); + FileDistributionConfigProvider configProvider = new FileDistributionConfigProvider(fileDistribution, host.getHost()); fileDistribution.addFileDistributionConfigProducer(host.getHost(), configProvider); } @@ -336,10 +326,6 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable return multitenant; } - public void setApplicationType(ApplicationType applicationType) { - this.applicationType = applicationType; - } - public ApplicationType getApplicationType() { return applicationType; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMonitoring.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMonitoring.java index e53f92822b5..0f15790aecf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMonitoring.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMonitoring.java @@ -1,7 +1,8 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; import java.util.Objects; +import java.util.Optional; /** * Monitoring service properties @@ -10,14 +11,21 @@ import java.util.Objects; */ public class DefaultMonitoring implements Monitoring { + public static final int DEFAULT_MONITORING_INTERVAL = 1; // in minutes + private static final String DEFAULT_MONITORING_CLUSTER_NAME = "vespa"; + private final Integer interval; private final String clustername; - public DefaultMonitoring(String clustername, Integer interval) { + public DefaultMonitoring() { + this(DEFAULT_MONITORING_CLUSTER_NAME, Optional.empty()); + } + + public DefaultMonitoring(String clustername, Optional<Integer> interval) { Objects.requireNonNull(clustername); Objects.requireNonNull(interval); this.clustername = clustername; - this.interval = interval; + this.interval = interval.orElse(DEFAULT_MONITORING_INTERVAL); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 2d2cfec84f8..a876ba7c3a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -471,6 +471,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); metrics.add(new Metric("content.proton.resource_usage.malloc_arena.max")); + metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.address_space.max")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.multi_value.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); // TODO: Remove in Vespa 8 diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index d22affaf5a3..8e0357c1c05 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -93,7 +93,8 @@ public class QuotaValidator extends Validator { private void throwIfBudgetExceeded(double spend, BigDecimal budget, SystemName systemName) { if (budget.doubleValue() < spend) { - throwBudgetException("Please free up some capacity! This deployment's quota use ($%.2f) exceeds reserved quota ($%.2f)!", spend, budget, systemName); + throw new IllegalArgumentException((systemName.equals(SystemName.Public) ? "" : systemName.value() + ": ") + + "Deployment would make your tenant exceed its quota and has been blocked! Please contact support to update your plan."); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 52dccbe96b5..87d3753ca2e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -1,34 +1,34 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.collections.Pair; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.io.IOUtils; import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; -import com.yahoo.path.Path; +import com.yahoo.searchdefinition.DistributableResource; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.RankExpressionBody; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.yolean.Exceptions; import com.yahoo.system.ProcessExecuter; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.collections.Pair; -import com.yahoo.config.ConfigInstance; import com.yahoo.vespa.config.search.ImportedFieldsConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -36,13 +36,14 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; -import java.util.logging.Logger; -import java.util.logging.Level; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; /** - * Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done + * Validates rank setup for all content clusters (rank-profiles, index-schema, attributes configs), validation is done * by running the binary 'vespa-verify-ranksetup-bin' * * @author vegardh @@ -69,17 +70,17 @@ public class RankSetupValidator extends Validator { for (AbstractSearchCluster cluster : model.getSearchClusters()) { // Skipping rank expression checking for streaming clusters, not implemented yet - if (cluster.isRealtime()) { - IndexedSearchCluster sc = (IndexedSearchCluster) cluster; - String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/"; - for (DocumentDatabase docDb : sc.getDocumentDbs()) { - final String name = docDb.getDerivedConfiguration().getSearch().getName(); - String searchDir = clusterDir + name + "/"; - writeConfigs(searchDir, docDb); - writeExtraVerifyRanksetupConfig(searchDir, docDb); - if ( ! validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) { - return; - } + if (cluster.isStreaming()) continue; + + IndexedSearchCluster sc = (IndexedSearchCluster) cluster; + String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/"; + for (DocumentDatabase docDb : sc.getDocumentDbs()) { + final String name = docDb.getDerivedConfiguration().getSearch().getName(); + String searchDir = clusterDir + name + "/"; + writeConfigs(searchDir, docDb); + writeExtraVerifyRanksetupConfig(searchDir, docDb); + if (!validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) { + return; } } } @@ -95,16 +96,13 @@ public class RankSetupValidator extends Validator { private boolean validate(String configId, SearchCluster searchCluster, String sdName, DeployLogger deployLogger, File tempDir) { Instant start = Instant.now(); try { + log.log(Level.FINE, () -> String.format("Validating schema '%s' for %s with config id %s", sdName, searchCluster, configId)); boolean ret = execValidate(configId, searchCluster, sdName, deployLogger); if (!ret) { // Give up, don't say same error msg repeatedly deleteTempDir(tempDir); } - log.log(Level.FINE, () -> String.format("Validating %s for %s, %s took %s ms", - sdName, - searchCluster, - configId, - Duration.between(start, Instant.now()).toMillis())); + log.log(Level.FINE, () -> String.format("Validation took %s ms", Duration.between(start, Instant.now()).toMillis())); return ret; } catch (IllegalArgumentException e) { deleteTempDir(tempDir); @@ -119,68 +117,58 @@ public class RankSetupValidator extends Validator { private void writeConfigs(String dir, AbstractConfigProducer<?> producer) throws IOException { RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder(); ((RankProfilesConfig.Producer) producer).getConfig(rpcb); - RankProfilesConfig rpc = new RankProfilesConfig(rpcb); - writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc); + writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpcb.build()); IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder(); ((IndexschemaConfig.Producer) producer).getConfig(iscb); - IndexschemaConfig isc = new IndexschemaConfig(iscb); - writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc); + writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", iscb.build()); AttributesConfig.Builder acb = new AttributesConfig.Builder(); ((AttributesConfig.Producer) producer).getConfig(acb); - AttributesConfig ac = new AttributesConfig(acb); - writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac); + writeConfig(dir, AttributesConfig.getDefName() + ".cfg", acb.build()); RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder(); ((RankingConstantsConfig.Producer) producer).getConfig(rccb); - RankingConstantsConfig rcc = new RankingConstantsConfig(rccb); - writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc); + writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rccb.build()); RankingExpressionsConfig.Builder recb = new RankingExpressionsConfig.Builder(); ((RankingExpressionsConfig.Producer) producer).getConfig(recb); - RankingExpressionsConfig rec = new RankingExpressionsConfig(recb); - writeConfig(dir, RankingExpressionsConfig.getDefName() + ".cfg", rec); + writeConfig(dir, RankingExpressionsConfig.getDefName() + ".cfg", recb.build()); OnnxModelsConfig.Builder omcb = new OnnxModelsConfig.Builder(); ((OnnxModelsConfig.Producer) producer).getConfig(omcb); - OnnxModelsConfig omc = new OnnxModelsConfig(omcb); - writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omc); + writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omcb.build()); ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder(); ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb); - ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb); - writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc); + writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifcb.build()); } - private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { - String configName = "verify-ranksetup.cfg"; - String configContent = ""; + private void writeExtraVerifyRanksetupConfig(List<String> config, Collection<? extends DistributableResource> resources) { + for (DistributableResource model : resources) { + String modelPath = getFileRepositoryPath(model.getFilePath().getName(), model.getFileReference()); + int index = config.size() / 2; + config.add(String.format("file[%d].ref \"%s\"", index, model.getFileReference())); + config.add(String.format("file[%d].path \"%s\"", index, modelPath)); + log.log(Level.FINE, index + ": " + model.getPathType() + " -> " + model.getName() + " -> " + modelPath + " -> " + model.getFileReference()); + } + } + private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { List<String> config = new ArrayList<>(); - // Assist verify-ranksetup in finding the actual ONNX model files - for (OnnxModel model : db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()) { - String modelPath = getFileRepositoryPath(model.getFilePath(), model.getFileReference()); - config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, model.getFileReference())); - config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); - } - for (RankExpressionBody expr : db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()) { - String modelPath = getFileRepositoryPath(expr.getFilePath(), expr.getFileReference()); - config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, expr.getFileReference())); - config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); - } + // Assist verify-ranksetup in finding the actual ONNX model files + writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()); + writeExtraVerifyRanksetupConfig(config, db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()); - if ( ! config.isEmpty() ) { - configContent = StringUtilities.implodeMultiline(config); - } - IOUtils.writeFile(dir + configName, configContent, false); + String configContent = config.isEmpty() ? "" : StringUtilities.implodeMultiline(config); + IOUtils.writeFile(dir + "verify-ranksetup.cfg", configContent, false); } - public static String getFileRepositoryPath(Path path, String fileReference) { + public static String getFileRepositoryPath(String name, String fileReference) { ConfigserverConfig cfg = new ConfigserverConfig(new ConfigserverConfig.Builder()); // assume defaults String fileRefDir = Defaults.getDefaults().underVespaHome(cfg.fileReferencesDir()); - return Paths.get(fileRefDir, fileReference, path.getName()).toString(); + return Paths.get(fileRefDir, fileReference, name).toString(); } private static void writeConfig(String dir, String configName, ConfigInstance config) throws IOException { @@ -203,14 +191,15 @@ public class RankSetupValidator extends Validator { } private void validateWarn(Exception e, DeployLogger deployLogger) { - String msg = "Unable to execute '"+ binaryName + "', validation of rank expressions will only take place when you start Vespa: " + - Exceptions.toMessageString(e); + String msg = "Unable to execute '" + binaryName + + "', validation of rank expressions will only take place when you start Vespa: " + + Exceptions.toMessageString(e); deployLogger.logApplicationPackage(Level.WARNING, msg); } private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) { - StringBuilder errMsg = new StringBuilder("For search cluster '").append(sc.getClusterName()).append("', ") - .append("search definition '").append(sdName).append("': error in rank setup. Details:\n"); + StringBuilder errMsg = new StringBuilder("Error in rank setup in schema '").append(sdName) + .append("' for content cluster '").append(sc.getClusterName()).append("'.").append(" Details:\n"); for (String line : output.split("\n")) { // Remove debug lines from start script if (line.startsWith("debug\t")) continue; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index c2c8b8e86cb..e59c944233d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.application.api.DeployLogger; @@ -19,11 +19,11 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets; import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder; -import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import org.w3c.dom.Element; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * A base class for admin model builders, to support common functionality across versions. @@ -33,20 +33,14 @@ import java.util.List; */ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProducerBuilder<Admin> { - private static final int DEFAULT_INTERVAL = 1; // in minutes - private static final String DEFAULT_CLUSTER_NAME = "vespa"; - private final ApplicationType applicationType; protected final List<ConfigServerSpec> configServerSpecs; - private final String fileSourceHost; protected final boolean multitenant; DomAdminBuilderBase(ApplicationType applicationType, - String fileSourceHost, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { this.applicationType = applicationType; - this.fileSourceHost = fileSourceHost; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; } @@ -71,44 +65,37 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu Monitoring monitoring = getMonitoring(XML.getChild(adminElement,"monitoring"), deployState.isHosted()); Metrics metrics = new MetricsBuilder(applicationType, PredefinedMetricSets.get()) .buildMetrics(XML.getChild(adminElement, "metrics")); - FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent); - - Admin admin = new Admin(parent, monitoring, metrics, multitenant, fileDistributionConfigProducer, deployState.isHosted()); - admin.setApplicationType(applicationType); + Admin admin = new Admin(parent, monitoring, metrics, multitenant, deployState.isHosted(), applicationType); doBuildAdmin(deployState, admin, adminElement); new ModelConfigProvider(admin); return admin; } - private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer<?> parent) { - return new FileDistributionConfigProducer(parent, fileSourceHost); - } - protected abstract void doBuildAdmin(DeployState deployState, Admin admin, Element adminE); private Monitoring getMonitoring(Element monitoringElement, boolean isHosted) { - if (monitoringElement == null) return new DefaultMonitoring(DEFAULT_CLUSTER_NAME, DEFAULT_INTERVAL); + if (monitoringElement == null) return new DefaultMonitoring(); + if (isHosted && applicationType.equals(ApplicationType.DEFAULT)) throw new IllegalArgumentException("The 'monitoring' element cannot be used on hosted Vespa."); - Integer minutes = getMonitoringInterval(monitoringElement); - if (minutes == null) - minutes = DEFAULT_INTERVAL; + Optional<Integer> minutes = getMonitoringInterval(monitoringElement); return new DefaultMonitoring(monitoringElement.getAttribute("systemname"), minutes); } - private Integer getMonitoringInterval(Element monitoringE) { - Integer minutes = null; + private Optional<Integer> getMonitoringInterval(Element monitoringE) { String seconds = monitoringE.getAttribute("interval").trim(); if ( ! seconds.isEmpty()) { - minutes = Integer.parseInt(seconds) / 60; - if (!(minutes == 1 || minutes == 5)) { - throw new IllegalArgumentException("The only allowed values for 'interval' attribute in '" + monitoringE.getTagName() + - "' element is 60 or 300."); + int m = Integer.parseInt(seconds) / 60; + if ( ! List.of(1, 5).contains(m)) { + throw new IllegalArgumentException("The only allowed values for 'interval' attribute in '" + + monitoringE.getTagName() + "' element is 60 or 300."); } + return Optional.of(m); } - return minutes; + + return Optional.empty(); } void addLogForwarders(ModelElement logForwardingElement, Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index ad24a324bbb..04c7db8e701 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -32,10 +32,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private static final String ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK = "standalone-zookeeper"; public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, - String fileSourceHost, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { - super(applicationType, fileSourceHost, multitenant, configServerSpecs); + super(applicationType, multitenant, configServerSpecs); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index ae54e9fecc4..c836c5557ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -36,8 +36,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) { - super(context.getApplicationType(), context.getDeployState().getFileRegistry().fileSourceHost(), multitenant, - configServerSpecs); + super(context.getApplicationType(), multitenant, configServerSpecs); this.containerModels = containerModels; this.context = context; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 02a0f606603..792fa3f1884 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.model.container.component.SimpleComponent; /** * A container that is typically used by container clusters set up from the user application. @@ -30,6 +31,10 @@ public final class ApplicationContainer extends Container implements public ApplicationContainer(AbstractConfigProducer<?> parent, String name, boolean retired, int index, DeployState deployState) { super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); + + addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); + addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); + addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.SessionCache")); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index d0bcfde3c0c..1203497f59f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -71,8 +71,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH); private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH + "/*"); - public static final int heapSizePercentageOfTotalNodeMemory = 60; - public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 17; + public static final int heapSizePercentageOfTotalNodeMemory = 70; + public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 18; private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); @@ -107,6 +107,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.SystemInfoProvider"); addSimpleComponent(com.yahoo.container.core.documentapi.DocumentAccessProvider.class.getName()); + addMetricsHandlers(); addTestrunnerComponentsIfTester(deployState); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index d130b08d083..4bd615cebcf 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -174,7 +174,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); - addComponent(new DefaultThreadpoolProvider(this)); + addComponent(new DefaultThreadpoolProvider(this, deployState.featureFlags().metricsproxyNumThreads())); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent(SecurityFilterInvoker.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java index dc01fcc65b1..4d665b0bb58 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -16,19 +16,21 @@ import com.yahoo.vespa.model.container.component.SimpleComponent; class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolConfig.Producer { private final ContainerCluster<?> cluster; + private final int metricsproxyNumThreads; - DefaultThreadpoolProvider(ContainerCluster<?> cluster) { + DefaultThreadpoolProvider(ContainerCluster<?> cluster, int metricsproxyNumThreads) { super(new ComponentModel( BundleInstantiationSpecification.getFromStrings( "default-threadpool", ThreadPoolProvider.class.getName(), null))); this.cluster = cluster; + this.metricsproxyNumThreads = metricsproxyNumThreads; } private int defaultThreadsByClusterType() { if (cluster instanceof MetricsProxyContainerCluster) { - return 2; + return metricsproxyNumThreads; } return 10; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java index c06f5f4c441..b98cabb6f33 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java @@ -19,6 +19,7 @@ import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.VespaModel; import org.xml.sax.SAXException; @@ -69,10 +70,11 @@ public class ModelsEvaluatorTester { RankProfilesConfig rankProfilesConfig = getRankProfilesConfig(rankProfileList); RankingConstantsConfig rankingConstantsConfig = getRankingConstantConfig(rankProfileList); + RankingExpressionsConfig rankingExpressionsConfig = getRankingExpressionsConfig(rankProfileList); OnnxModelsConfig onnxModelsConfig = getOnnxModelsConfig(rankProfileList); FileAcquirer files = createFileAcquirer(rankingConstantsConfig, onnxModelsConfig, temporaryApplicationDir); - return new ModelsEvaluator(rankProfilesConfig, rankingConstantsConfig, onnxModelsConfig, files); + return new ModelsEvaluator(rankProfilesConfig, rankingConstantsConfig, rankingExpressionsConfig, onnxModelsConfig, files); } catch (IOException | SAXException e) { throw new RuntimeException(e); @@ -104,19 +106,25 @@ public class ModelsEvaluatorTester { private static RankProfilesConfig getRankProfilesConfig(RankProfileList rankProfileList) { RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); rankProfileList.getConfig(builder); - return new RankProfilesConfig(builder); + return builder.build(); } private static RankingConstantsConfig getRankingConstantConfig(RankProfileList rankProfileList) { RankingConstantsConfig.Builder builder = new RankingConstantsConfig.Builder(); rankProfileList.getConfig(builder); - return new RankingConstantsConfig(builder); + return builder.build(); + } + + private static RankingExpressionsConfig getRankingExpressionsConfig(RankProfileList rankProfileList) { + RankingExpressionsConfig.Builder builder = new RankingExpressionsConfig.Builder(); + rankProfileList.getConfig(builder); + return builder.build(); } private static OnnxModelsConfig getOnnxModelsConfig(RankProfileList rankProfileList) { OnnxModelsConfig.Builder builder = new OnnxModelsConfig.Builder(); rankProfileList.getConfig(builder); - return new OnnxModelsConfig(builder); + return builder.build(); } private static FileAcquirer createFileAcquirer(RankingConstantsConfig constantsConfig, OnnxModelsConfig onnxModelsConfig, File appDir) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index efb47e97ccb..5b2b4cc6cba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -153,7 +153,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> String clusterName, ContentSearchCluster search) { List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, deployState); + IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0); isc.setRoutingSelector(clusterElem.childAsString("documents.selection")); Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java index ec677911da6..f0a2e52f6bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; +import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; @@ -14,9 +15,10 @@ import org.w3c.dom.Element; /** * Represents specific configuration for a given distributor node. */ -public class Distributor extends ContentNode { +public class Distributor extends ContentNode implements StorDistributormanagerConfig.Producer { PersistenceEngine provider; + private final int numDistributorStripesFlag; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<Distributor> { ModelElement clusterXml; @@ -39,12 +41,27 @@ public class Distributor extends ContentNode { StorageNode.rootFolder + parent.getClusterName() + "/distributor/" + distributionKey, distributionKey); this.provider = provider; + this.numDistributorStripesFlag = properties.featureFlags().numDistributorStripes(); if (distributorBasePort != null) { setBasePort(distributorBasePort); } } + private int tuneNumDistributorStripes() { + if ((numDistributorStripesFlag == -1) && (getHostResource() != null)) { + int cores = (int)getHostResource().realResources().vcpu(); + if (cores <= 16) { + return 1; + } else if (cores <= 64) { + return 2; + } else { + return 4; + } + } + return numDistributorStripesFlag; + } + @Override public void getConfig(StorServerConfig.Builder builder) { super.getConfig(builder); @@ -59,6 +76,11 @@ public class Distributor extends ContentNode { } @Override + public void getConfig(StorDistributormanagerConfig.Builder builder) { + builder.num_distributor_stripes(tuneNumDistributorStripes()); + } + + @Override public String getStartupCommand() { return "exec sbin/vespa-distributord -c $VESPA_CONFIG_ID"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 390c939c4b4..0e3fee5a749 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -43,7 +43,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean hasIndexedDocumentType; private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; - private final int numDistributorStripes; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -106,20 +105,18 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); - int numDistributorStripes = deployState.getProperties().featureFlags().numDistributorStripes(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, numDistributorStripes); + maxInhibitedGroups); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, - int maxActivationInhibitedOutOfSyncGroups, - int numDistributorStripes) + int maxActivationInhibitedOutOfSyncGroups) { super(parent, "distributor"); this.parent = parent; @@ -128,7 +125,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.hasIndexedDocumentType = hasIndexedDocumentType; this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; - this.numDistributorStripes = numDistributorStripes; } @Override @@ -142,7 +138,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.disable_bucket_activation(hasIndexedDocumentType == false); builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); - builder.num_distributor_stripes(numDistributorStripes); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java index a12e183b409..cd9a0c89bf8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java @@ -37,8 +37,7 @@ public class ResourceLimits implements FleetcontrollerConfig.Producer, ProtonCon // storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp builder.cluster_feed_block_limit.put("memory", memoryLimit.orElse(0.8)); builder.cluster_feed_block_limit.put("disk", diskLimit.orElse(0.8)); - builder.cluster_feed_block_limit.put("attribute-enum-store", 0.89); - builder.cluster_feed_block_limit.put("attribute-multi-value", 0.89); + builder.cluster_feed_block_limit.put("attribute-address-space", 0.89); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index c764b5ab449..0997f29729f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -241,7 +241,8 @@ public class StorageGroup { int minNodesPerGroup = (int)Math.ceil((double)nodesSpec.minResources().nodes() / nodesSpec.minResources().groups()); if (minNodesPerGroup < redundancy) { // TODO: Fail on this on Vespa 8, and simplify - context.getDeployLogger().logApplicationPackage(Level.WARNING, + context.getDeployLogger() + .logApplicationPackage(Level.WARNING, "Cluster '" + clusterElement.stringAttribute("id") + "' " + "specifies redundancy " + redundancy + " but cannot be higher than " + "the minimum nodes per group, which is " + minNodesPerGroup); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index fabcbbcc9ec..ad8948153cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -325,13 +325,13 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce } } - public static final NodeResources clusterControllerResources = new NodeResources(0.5, 2, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); + public static final NodeResources clusterControllerResources = new NodeResources(0.25, 1, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private ClusterControllerContainerCluster getDedicatedSharedControllers(ModelElement contentElement, Admin admin, ConfigModelContext context, DeployState deployState) { if (admin.getClusterControllers() == null) { NodesSpecification spec = NodesSpecification.requiredFromSharedParents(deployState.zone().environment().isProduction() ? 3 : 1, - deployState.featureFlags().dedicatedClusterControllerFlavor().orElse(clusterControllerResources), + clusterControllerResources, contentElement, context); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java index d29a2611ac7..2e1902cb1ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.filedistribution; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -15,19 +15,9 @@ import java.util.Map; public class FileDistributionConfigProducer extends AbstractConfigProducer<AbstractConfigProducer<?>> { private final Map<Host, FileDistributionConfigProvider> fileDistributionConfigProviders = new IdentityHashMap<>(); - private final FileDistributor fileDistributor; - public FileDistributionConfigProducer(AbstractConfigProducer<?> ancestor, String fileSourceHost) { - this(ancestor, new FileDistributor(fileSourceHost)); - } - - private FileDistributionConfigProducer(AbstractConfigProducer<?> parent, FileDistributor fileDistributor) { + public FileDistributionConfigProducer(AbstractConfigProducer<?> parent) { super(parent, "filedistribution"); - this.fileDistributor = fileDistributor; - } - - public FileDistributor getFileDistributor() { - return fileDistributor; } public void addFileDistributionConfigProducer(Host host, FileDistributionConfigProvider fileDistributionConfigProvider) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java index 47ee546181a..314c69a6e09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java @@ -1,29 +1,17 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.filedistribution; import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; -import com.yahoo.cloud.config.filedistribution.FilereferencesConfig; -import com.yahoo.config.FileReference; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.ConfigProxy; import com.yahoo.vespa.model.Host; -import java.util.Collection; +public class FileDistributionConfigProvider extends AbstractConfigProducer<AbstractConfigProducer<?>> implements FiledistributorrpcConfig.Producer { -public class FileDistributionConfigProvider extends AbstractConfigProducer - implements FiledistributorrpcConfig.Producer, FilereferencesConfig.Producer { - - private final FileDistributor fileDistributor; - private final boolean sendAllFiles; private final Host host; - public FileDistributionConfigProvider(AbstractConfigProducer parent, - FileDistributor fileDistributor, - boolean sendAllFiles, - Host host) { + public FileDistributionConfigProvider(AbstractConfigProducer<?> parent, Host host) { super(parent, host.getHostname()); - this.fileDistributor = fileDistributor; - this.sendAllFiles = sendAllFiles; this.host = host; } @@ -32,18 +20,4 @@ public class FileDistributionConfigProvider extends AbstractConfigProducer builder.connectionspec("tcp/" + host.getHostname() + ":" + ConfigProxy.BASEPORT); } - @Override - public void getConfig(FilereferencesConfig.Builder builder) { - for (FileReference reference : getFileReferences()) { - builder.filereferences(reference.value()); - } - } - - private Collection<FileReference> getFileReferences() { - if (sendAllFiles) { - return fileDistributor.allFilesToSend(); - } else { - return fileDistributor.filesToSendToHost(host); - } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java deleted file mode 100644 index b2e9cfeec85..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.filedistribution; - -import com.yahoo.config.FileReference; -import com.yahoo.vespa.model.Host; - -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - - -/** - * Sends RPC requests to hosts (tenant hosts and config servers) asking them to start download of files. This is used - * during prepare of an application. Services themselves will also request files, the methods in this class are used - * so that hosts can start downloading files before services gets new config that needs these files. It also tries - * to make sure that all config servers (not just the one where the application was deployed) have the files available. - * - * @author Tony Vaagenes - */ -public class FileDistributor { - - private final String fileSourceHost; - - /** A map from file reference to the hosts to which that file reference should be distributed */ - private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>(); - - public FileDistributor(String fileSourceHost) { - this.fileSourceHost = fileSourceHost; - } - - public void sendFileReference(FileReference reference, Host host) { - filesToHosts.computeIfAbsent(reference, k -> new HashSet<>()).add(host); - } - - /** Returns the files which has been marked for distribution to the given host */ - public Set<FileReference> filesToSendToHost(Host host) { - Set<FileReference> files = new HashSet<>(); - - for (Map.Entry<FileReference,Set<Host>> e : filesToHosts.entrySet()) { - if (e.getValue().contains(host)) { - files.add(e.getKey()); - } - } - return files; - } - - public Set<Host> getTargetHosts() { - Set<Host> hosts = new HashSet<>(); - for (Set<Host> hostSubset: filesToHosts.values()) - hosts.addAll(hostSubset); - return hosts; - } - - /** Returns the host which is the source of the files */ - public String fileSourceHost() { - return fileSourceHost; - } - - public Set<FileReference> allFilesToSend() { - return Set.copyOf(filesToHosts.keySet()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java new file mode 100644 index 00000000000..bdda9cdf582 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepository.java @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.config.FileReference; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Keeps track of what files to send with file distribution + * + * @author Tony Vaagenes + * @author hmusum + */ +public class FileReferencesRepository { + + /** A set of file references that should be distributed */ + private final Set<FileReference> fileReferences = new LinkedHashSet<>(); + + public FileReferencesRepository() { } + + public void add(FileReference reference) { fileReferences.add(reference); } + + public Set<FileReference> allFileReferences() { return Set.copyOf(fileReferences); } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index c99549e82e9..95021dca736 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.content.DispatchSpec; import com.yahoo.vespa.model.content.SearchCoverage; -import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -63,7 +62,7 @@ public class IndexedSearchCluster extends SearchCluster return routingSelector; } - public IndexedSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, int index, DeployState deployState) { + public IndexedSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, int index) { super(parent, clusterName, index); unionCfg = new UnionConfiguration(this, documentDbs); rootDispatch = new DispatchGroup(this); @@ -204,7 +203,8 @@ public class IndexedSearchCluster extends SearchCluster deployState.getProperties(), deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels())); + deployState.getImportedModels(), + deployState.getExecutor())); // TODO: remove explicit adding of user configs when the complete content model is built using builders. db.mergeUserConfigs(spec.getUserConfigs()); documentDbs.add(db); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 890e42eb60c..330c7768321 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -12,8 +12,6 @@ import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; -import java.io.File; -import java.io.IOException; import java.util.List; /** @@ -99,7 +97,8 @@ public class StreamingSearchCluster extends SearchCluster implements deployState.getProperties(), deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels()); + deployState.getImportedModels(), + deployState.getExecutor()); } @Override public DerivedConfiguration getSdConfig() { diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 171112f6bd7..77e3ce8f573 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -51,7 +51,8 @@ ParallelInstances = element parallel { } Upgrade = element upgrade { - attribute policy { xsd:string } + attribute policy { xsd:string }? & + attribute rollout { xsd:string }? } BlockChange = element block-change { diff --git a/config-model/src/test/cfg/application/onnx_name_collision/models/barfoo.onnx b/config-model/src/test/cfg/application/onnx_name_collision/models/barfoo.onnx new file mode 100644 index 00000000000..613af39a958 --- /dev/null +++ b/config-model/src/test/cfg/application/onnx_name_collision/models/barfoo.onnx @@ -0,0 +1,16 @@ + barfoo.py:T + +bar +bazfoo"MulmulZ +bar + + +Z +baz + + +b +foo + + +B
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/onnx_name_collision/models/barfoo.py b/config-model/src/test/cfg/application/onnx_name_collision/models/barfoo.py new file mode 100755 index 00000000000..c9bd7851980 --- /dev/null +++ b/config-model/src/test/cfg/application/onnx_name_collision/models/barfoo.py @@ -0,0 +1,26 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import onnx +from onnx import helper, TensorProto + +INPUT_1 = helper.make_tensor_value_info('bar', TensorProto.FLOAT, [1]) +INPUT_2 = helper.make_tensor_value_info('baz', TensorProto.FLOAT, [1]) +OUTPUT = helper.make_tensor_value_info('foo', TensorProto.FLOAT, [1]) + +nodes = [ + helper.make_node( + 'Mul', + ['bar', 'baz'], + ['foo'], + ), +] +graph_def = helper.make_graph( + nodes, + 'mul', + [ + INPUT_1, + INPUT_2 + ], + [OUTPUT], +) +model_def = helper.make_model(graph_def, producer_name='barfoo.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +onnx.save(model_def, 'barfoo.onnx') diff --git a/config-model/src/test/cfg/application/onnx_name_collision/models/foobar.onnx b/config-model/src/test/cfg/application/onnx_name_collision/models/foobar.onnx new file mode 100644 index 00000000000..a393e33094a --- /dev/null +++ b/config-model/src/test/cfg/application/onnx_name_collision/models/foobar.onnx @@ -0,0 +1,16 @@ + foobar.py:T + +foo +bazbat"MulmulZ +foo + + +Z +baz + + +b +bar + + +B
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/onnx_name_collision/models/foobar.py b/config-model/src/test/cfg/application/onnx_name_collision/models/foobar.py new file mode 100755 index 00000000000..c3e0667aeaa --- /dev/null +++ b/config-model/src/test/cfg/application/onnx_name_collision/models/foobar.py @@ -0,0 +1,26 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import onnx +from onnx import helper, TensorProto + +INPUT_1 = helper.make_tensor_value_info('foo', TensorProto.FLOAT, [1]) +INPUT_2 = helper.make_tensor_value_info('baz', TensorProto.FLOAT, [1]) +OUTPUT = helper.make_tensor_value_info('bar', TensorProto.FLOAT, [1]) + +nodes = [ + helper.make_node( + 'Mul', + ['foo', 'baz'], + ['bat'], + ), +] +graph_def = helper.make_graph( + nodes, + 'mul', + [ + INPUT_1, + INPUT_2 + ], + [OUTPUT], +) +model_def = helper.make_model(graph_def, producer_name='foobar.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +onnx.save(model_def, 'foobar.onnx') diff --git a/config-model/src/test/cfg/application/onnx_name_collision/services.xml b/config-model/src/test/cfg/application/onnx_name_collision/services.xml new file mode 100644 index 00000000000..51fdf976f7f --- /dev/null +++ b/config-model/src/test/cfg/application/onnx_name_collision/services.xml @@ -0,0 +1,12 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <container version="1.0"> + <model-evaluation/> + <nodes> + <node hostalias="node1" /> + </nodes> + </container> + +</services> diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg index c6263d658c5..bb7c7fdd349 100644 --- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg @@ -80,8 +80,8 @@ rankprofile[].fef.property[].value "attribute(bar4).out" rankprofile[].name "overflow" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" -rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" -rankprofile[].fef.property[].value "feature1(argument1,argument2,argument3,argument4).output + feature2(argument1,argument2,argument3,argument4).output + feature3(argument1,argument2,argument3,argument4).output + feature4(argument1,argument2,argument3,argument4).output + feature5(argument1,argument2,argument3,argument4).output + feature6(argument1,argument2,argument3,argument4).output + feature7(argument1,argument2,argument3,argument4).output + feature8(argument1,argument2,argument3,argument4).output + feature9(argument1,argument2,argument3,argument4).output + feature10(argument1,argument2,argument3,argument4).output + feature11(argument1,argument2,argument3,argument4).output + feature12(argument1,argument2,argument3,argument4).output + feature13(argument1,argument2,argument3,argument4).output + feature14(argument1,argument2,argument3,argument4).output + feature15(argument1,argument2,argument3,argument4).output + feature16(argument1,argument2,argument3,argument4).output + feature17(argument1,argument2,argument3,argument4).output + feature18(argument1,argument2,argument3,argument4).output + feature19(argument1,argument2,argument3,argument4).output + feature20(argument1,argument2,argument3,argument4).output + feature21(argument1,argument2,argument3,argument4).output + feature22(argument1,argument2,argument3,argument4).output + feature23(argument1,argument2,argument3,argument4).output + feature24(argument1,argument2,argument3,argument4).output + feature25(argument1,argument2,argument3,argument4).output + feature26(argument1,argument2,argument3,argument4).output + feature27(argument1,argument2,argument3,argument4).output + feature28(argument1,argument2,argument3,argument4).output + feature29(argument1,argument2,argument3,argument4).output + feature30(argument1,argument2,argument3,argument4).output + feature31(argument1,argument2,argument3,argument4).output + feature32(argument1,argument2,argument3,argument4).output + feature33(argument1,argument2,argument3,argument4).output + feature34(argument1,argument2,argument3,argument4).output + feature35(argument1,argument2,argument3,argument4).output + feature36(argument1,argument2,argument3,argument4).output + feature37(argument1,argument2,argument3,argument4).output + feature38(argument1,argument2,argument3,argument4).output + feature39(argument1,argument2,argument3,argument4).output + feature40(argument1,argument2,argument3,argument4).output + feature41(argument1,argument2,argument3,argument4).output + feature42(argument1,argument2,argument3,argument4).output + feature43(argument1,argument2,argument3,argument4).output + feature44(argument1,argument2,argument3,argument4).output + feature45(argument1,argument2,argument3,argument4).output + feature46(argument1,argument2,argument3,argument4).output + feature47(argument1,argument2,argument3,argument4).output + feature48(argument1,argument2,argument3,argument4).output + feature49(argument1,argument2,argument3,argument4).output + feature50(argument1,argument2,argument3,argument4).output + feature51(argument1,argument2,argument3,argument4).output + feature52(argument1,argument2,argument3,argument4).output + feature53(argument1,argument2,argument3,argument4).output + feature54(argument1,argument2,argument3,argument4).output + feature55(argument1,argument2,argument3,argument4).output + feature56(argument1,argument2,argument3,argument4).output + feature57(argument1,argument2,argument3,argument4).output + feature58(argument1,argument2,argument3,argument4).output + feature59(argument1,argument2,argument3,argument4).output + feature60(argument1,argument2,argument3,argument4).output + feature61(argument1,argument2,argument3,argument4).output + feature62(argument1,argument2,argument3,argument4).output + feature63(argument1,argument2,argument3,argument4).output + feature64(argument1,argument2,argument3,argument4).output + feature65(argument1,argument2,argument3,argument4).output + feature66(argument1,argument2,argument3,argument4).output + feature67(argument1,argument2,argument3,argument4).output + feature68(argument1,argument2,argument3,argument4).output + feature69(argument1,argument2,argument3,argument4).output + feature70(argument1,argument2,argument3,argument4).output + feature71(argument1,argument2,argument3,argument4).output + feature72(argument1,argument2,argument3,argument4).output + feature73(argument1,argument2,argument3,argument4).output + feature74(argument1,argument2,argument3,argument4).output + feature75(argument1,argument2,argument3,argument4).output + feature76(argument1,argument2,argument3,argument4).output + feature77(argument1,argument2,argument3,argument4).output + feature78(argument1,argument2,argument3,argument4).output + feature79(argument1,argument2,argument3,argument4).output + feature80(argument1,argument2,argument3,argument4).output + feature81(argument1,argument2,argument3,argument4).output + feature82(argument1,argument2,argument3,argument4).output + feature83(argument1,argument2,argument3,argument4).output + feature84(argument1,argument2,argument3,argument4).output + feature85(argument1,argument2,argument3,argument4).output + feature86(argument1,argument2,argument3,argument4).output + feature87(argument1,argument2,argument3,argument4).output + feature88(argument1,argument2,argument3,argument4).output + feature89(argument1,argument2,argument3,argument4).output + feature90(argument1,argument2,argument3,argument4).output + feature91(argument1,argument2,argument3,argument4).output + feature92(argument1,argument2,argument3,argument4).output + feature93(argument1,argument2,argument3,argument4).output + feature94(argument1,argument2,argument3,argument4).output + feature95(argument1,argument2,argument3,argument4).output + feature96(argument1,argument2,argument3,argument4).output + feature97(argument1,argument2,argument3,argument4).output + feature98(argument1,argument2,argument3,argument4).output + feature99(argument1,argument2,argument3,argument4).output + feature100(argument1,argument2,argument3,argument4).output + feature101(argument1,argument2,argument3,argument4).output + feature102(argument1,argument2,argument3,argument4).output + feature103(argument1,argument2,argument3,argument4).output + feature104(argument1,argument2,argument3,argument4).output + feature105(argument1,argument2,argument3,argument4).output + feature106(argument1,argument2,argument3,argument4).output + feature107(argument1,argument2,argument3,argument4).output + feature108(argument1,argument2,argument3,argument4).output + feature109(argument1,argument2,argument3,argument4).output + feature110(argument1,argument2,argument3,argument4).output + feature111(argument1,argument2,argument3,argument4).output + feature112(argument1,argument2,argument3,argument4).output + feature113(argument1,argument2,argument3,argument4).output + feature114(argument1,argument2,argument3,argument4).output + feature115(argument1,argument2,argument3,argument4).output + feature116(argument1,argument2,argument3,argument4).output + feature117(argument1,argument2,argument3,argument4).output + feature118(argument1,argument2,argument3,argument4).output + feature119(argument1,argument2,argument3,argument4).output + feature120(argument1,argument2,argument3,argument4).output + feature121(argument1,argument2,argument3,argument4).output + feature122(argument1,argument2,argument3,argument4).output + feature123(argument1,argument2,argument3,argument4).output + feature124(argument1,argument2,argument3,argument4).output + feature125(argument1,argument2,argument3,argument4).output + feature126(argument1,argument2,argument3,argument4).output + feature127(argument1,argument2,argument3,argument4).output + feature128(argument1,argument2,argument3,argument4).output + feature129(argument1,argument2,argument3,argument4).output + feature130(argument1,argument2,argument3,argument4).output + feature131(argument1,argument2,argument3,argument4).output + feature132(argument1,argument2,argument3,argument4).output + feature133(argument1,argument2,argument3,argument4).output + feature134(argument1,argument2,argument3,argument4).output + feature135(argument1,argument2,argument3,argument4).output + feature136(argument1,argument2,argument3,argument4).output + feature137(argument1,argument2,argument3,argument4).output + feature138(argument1,argument2,argument3,argument4).output + feature139(argument1,argument2,argument3,argument4).output + feature140(argument1,argument2,argument3,argument4).output + feature141(argument1,argument2,argument3,argument4).output + feature142(argument1,argument2,argument3,argument4).output + feature143(argument1,argument2,argument3,argument4).output + feature144(argument1,argument2,argument3,argument4).output + feature145(argument1,argument2,argument3,argument4).output + feature146(argument1,argument2,argument3,argument4).output + feature147(argument1,argument2,argument3,argument4).output + feature148(argument1,argument2,argument3,argument4).output + feature149(argument1,argument2,argument3,argument4).output + feature150(argument1,argument2,argument3,argument4).output + feature151(argument1,argument2,argument3,argument4).output + feature152(argument1,argument2,argument3,argument4).output + feature153(argument1,argument2,argument3,argument4).output + feature154(argument1,argument2,argument3,argument4).output + feature155(argument1,argument2,argument3,argument4).output + feature156(argument1,argument2,argument3,argument4).output + feature157(argument1,argument2,argument3,argument4).output + feature158(argument1,argument2,argument3,argument4).output + feature159(argument1,argument2,argument3,argument4).output + feature160(argument1,argument2,argument3,argument4).output + feature161(argument1,argument2,argument3,argument4).output + feature162(argument1,argument2,argument3,argument4).output + feature163(argument1,argument2,argument3,argument4).output + feature164(argument1,argument2,argument3,argument4).output + feature165(argument1,argument2,argument3,argument4).output + feature166(argument1,argument2,argument3,argument4).output + feature167(argument1,argument2,argument3,argument4).output + feature168(argument1,argument2,argument3,argument4).output + feature169(argument1,argument2,argument3,argument4).output + feature170(argument1,argument2,argument3,argument4).output + feature171(argument1,argument2,argument3,argument4).output + feature172(argument1,argument2,argument3,argument4).output + feature173(argument1,argument2,argument3,argument4).output + feature174(argument1,argument2,argument3,argument4).output + feature175(argument1,argument2,argument3,argument4).output + feature176(argument1,argument2,argument3,argument4).output + feature177(argument1,argument2,argument3,argument4).output + feature178(argument1,argument2,argument3,argument4).output + feature179(argument1,argument2,argument3,argument4).output + feature180(argument1,argument2,argument3,argument4).output + feature181(argument1,argument2,argument3,argument4).output + feature182(argument1,argument2,argument3,argument4).output + feature183(argument1,argument2,argument3,argument4).output + feature184(argument1,argument2,argument3,argument4).output + feature185(argument1,argument2,argument3,argument4).output + feature186(argument1,argument2,argument3,argument4).output + feature187(argument1,argument2,argument3,argument4).output + feature188(argument1,argument2,argument3,argument4).output + feature189(argument1,argument2,argument3,argument4).output + feature190(argument1,argument2,argument3,argument4).output + feature191(argument1,argument2,argument3,argument4).output + feature192(argument1,argument2,argument3,argument4).output + feature193(argument1,argument2,argument3,argument4).output + feature194(argument1,argument2,argument3,argument4).output + feature195(argument1,argument2,argument3,argument4).output + feature196(argument1,argument2,argument3,argument4).output + feature197(argument1,argument2,argument3,argument4).output + feature198(argument1,argument2,argument3,argument4).output + feature199(argument1,argument2,argument3,argument4).output + feature200(argument1,argument2,argument3,argument4).output + feature201(argument1,argument2,argument3,argument4).output + feature202(argument1,argument2,argument3,argument4).output + feature203(argument1,argument2,argument3,argument4).output + feature204(argument1,argument2,argument3,argument4).output + feature205(argument1,argument2,argument3,argument4).output + feature206(argument1,argument2,argument3,argument4).output + feature207(argument1,argument2,argument3,argument4).output + feature208(argument1,argument2,argument3,argument4).output + feature209(argument1,argument2,argument3,argument4).output + feature210(argument1,argument2,argument3,argument4).output + feature211(argument1,argument2,argument3,argument4).output + feature212(argument1,argument2,argument3,argument4).output + feature213(argument1,argument2,argument3,argument4).output + feature214(argument1,argument2,argument3,argument4).output + feature215(argument1,argument2,argument3,argument4).output + feature216(argument1,argument2,argument3,argument4).output + feature217(argument1,argument2,argument3,argument4).output + feature218(argument1,argument2,argument3,argument4).output + feature219(argument1,argument2,argument3,argument4).output + feature220(argument1,argument2,argument3,argument4).output + feature221(argument1,argument2,argument3,argument4).output + feature222(argument1,argument2,argument3,argument4).output + feature223(argument1,argument2,argument3,argument4).output + feature224(argument1,argument2,argument3,argument4).output + feature225(argument1,argument2,argument3,argument4).output + feature226(argument1,argument2,argument3,argument4).output + feature227(argument1,argument2,argument3,argument4).output + feature228(argument1,argument2,argument3,argument4).output + feature229(argument1,argument2,argument3,argument4).output + feature230(argument1,argument2,argument3,argument4).output + feature231(argument1,argument2,argument3,argument4).output + feature232(argument1,argument2,argument3,argument4).output + feature233(argument1,argument2,argument3,argument4).output + feature234(argument1,argument2,argument3,argument4).output + feature235(argument1,argument2,argument3,argument4).output + feature236(argument1,argument2,argument3,argument4).output + feature237(argument1,argument2,argument3,argument4).output + feature238(argument1,argument2,argument3,argument4).output + feature239(argument1,argument2,argument3,argument4).output + feature240(argument1,argument2,argument3,argument4).output + feature241(argument1,argument2,argument3,argument4).output + feature242(argument1,argument2,argument3,argument4).output + feature243(argument1,argument2,argument3,argument4).output + feature244(argument1,argument2,argument3,argument4).output + feature245(argument1,argument2,argument3,argument4).output + feature246(argument1,argument2,argument3,argument4).output + feature247(argument1,argument2,argument3,argument4).output + feature248(argument1,argument2,argument3,argument4).output + feature249(argument1,argument2,argument3,argument4).output + feature250(argument1,argument2,argument3,argument4).output + feature251(argument1,argument2,argument3,argument4).output + feature252(argument1,argument2,argument3,argument4).output + feature253(argument1,argument2,argument3,argument4).output + feature254(argument1,argument2,argument3,argument4).output + feature255(argument1,argument2,argument3,argument4).output + feature256(argument1,argument2,argument3,argument4).output + feature257(argument1,argument2,argument3,argument4).output + feature258(argument1,argument2,argument3,argument4).output + feature259(argument1,argument2,argument3,argument4).output + feature260(argument1,argument2,argument3,argument4).output + feature261(argument1,argument2,argument3,argument4).output + feature262(argument1,argument2,argument3,argument4).output + feature263(argument1,argument2,argument3,argument4).output + feature264(argument1,argument2,argument3,argument4).output + feature265(argument1,argument2,argument3,argument4).output + feature266(argument1,argument2,argument3,argument4).output + feature267(argument1,argument2,argument3,argument4).output + feature268(argument1,argument2,argument3,argument4).output + feature269(argument1,argument2,argument3,argument4).output + feature270(argument1,argument2,argument3,argument4).output + feature271(argument1,argument2,argument3,argument4).output + feature272(argument1,argument2,argument3,argument4).output + feature273(argument1,argument2,argument3,argument4).output + feature274(argument1,argument2,argument3,argument4).output + feature275(argument1,argument2,argument3,argument4).output + feature276(argument1,argument2,argument3,argument4).output + feature277(argument1,argument2,argument3,argument4).output + feature278(argument1,argument2,argument3,argument4).output + feature279(argument1,argument2,argument3,argument4).output + feature280(argument1,argument2,argument3,argument4).output + feature281(argument1,argument2,argument3,argument4).output + feature282(argument1,argument2,argument3,argument4).output + feature283(argument1,argument2,argument3,argument4).output + feature284(argument1,argument2,argument3,argument4).output + feature285(argument1,argument2,argument3,argument4).output + feature286(argument1,argument2,argument3,argument4).output + feature287(argument1,argument2,argument3,argument4).output + feature288(argument1,argument2,argument3,argument4).output + feature289(argument1,argument2,argument3,argument4).output + feature290(argument1,argument2,argument3,argument4).output + feature291(argument1,argument2,argument3,argument4).output + feature292(argument1,argument2,argument3,argument4).output + feature293(argument1,argument2,argument3,argument4).output + feature294(argument1,argument2,argument3,argument4).output + feature295(argument1,argument2,argument3,argument4).output + feature296(argument1,argument2,argument3,argument4).output + feature297(argument1,argument2,argument3,argument4).output + feature298(argument1,argument2,argument3,argument4).output + feature299(argument1,argument2,argument3,argument4).output + feature300(argument1,argument2,argument3,argument4).output" +rankprofile[].fef.property[].name "rankingExpression(firstphase).expressionName" +rankprofile[].fef.property[].value "overflow.firstphase" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" diff --git a/config-model/src/test/examples/largerankingexpressions/rankexpression.sd b/config-model/src/test/examples/largerankingexpressions/rankexpression.sd new file mode 100644 index 00000000000..55466b7a5d3 --- /dev/null +++ b/config-model/src/test/examples/largerankingexpressions/rankexpression.sd @@ -0,0 +1,64 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search rankexpression { + + document rankexpression { + + field artist type string { + indexing: summary | index + } + + field title type string { + indexing: summary | index + } + + field surl type string { + indexing: summary + } + + field year type int { + indexing: summary | attribute + } + + } + + rank-profile base { + + function large_f() { + expression: 78+closeness(distance)*(attribute(year)+attribute(year))/attribute(year) + } + macro large_m() { + expression: 78+closeness(distance)*(attribute(year)+attribute(year))/attribute(year) + } + } + + rank-profile child_a inherits base { + + function large_local_f() { + expression: large_f() + large_m() + } + + function large_local_m() { + expression: large_f() + large_m() + } + + first-phase { + expression: 0.8+0.2*large_local_f() + large_local_m() + } + } + + rank-profile child_b inherits base { + + function large_local_f() { + expression: large_f() + large_m() + } + + function large_local_m() { + expression: large_f() + large_m() + } + + first-phase { + expression: 0.8+0.2*large_local_f() + large_local_m() + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index b0ddadf11bd..9b9789e141c 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -215,7 +215,7 @@ public class ModelProvisioningTest { assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); assertEquals("Nodes in container1", 1, model.getContainerClusters().get("container1").getContainers().size()); assertEquals("Nodes in cluster without ID", 2, model.getContentClusters().get("content").getRootGroup().getNodes().size()); - assertEquals("Heap size for container", 60, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Heap size for container", 70, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.content, model); assertProvisioned(1, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content"), ClusterSpec.Type.content, model); @@ -269,9 +269,9 @@ public class ModelProvisioningTest { assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); assertEquals("Heap size is lowered with combined clusters", - 17, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + 18, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); assertEquals("Memory for proton is lowered to account for the jvm heap", - (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.17)), protonMemorySize(model.getContentClusters().get("content1"))); + (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.18)), protonMemorySize(model.getContentClusters().get("content1"))); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); } @@ -305,7 +305,7 @@ public class ModelProvisioningTest { assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); assertEquals("Heap size is normal", - 60, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + 70, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); assertEquals("Memory for proton is normal", (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1"))); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java index 056fc27f067..c2bea606bef 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java @@ -1,10 +1,15 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import org.junit.Ignore; import org.junit.Test; +import java.util.function.Function; +import java.util.regex.Pattern; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Tests rank feature names. @@ -43,4 +48,47 @@ public class FeatureNamesTestCase { FeatureNames.asQueryFeature("foo.bar").toString()); } + @Test + public void testLegalFeatureNames() { + assertTrue(FeatureNames.notNeedQuotes("_")); + assertFalse(FeatureNames.notNeedQuotes("-")); + assertTrue(FeatureNames.notNeedQuotes("_-")); + assertTrue(FeatureNames.notNeedQuotes("0_-azAZxy98-_")); + assertFalse(FeatureNames.notNeedQuotes("0_-azAZxy98-_+")); + } + + @Test + @Ignore + /* + * Unignore to verify performance + * 2021/09/05 performance was a factor of 5.25 + * 'Identifier handcoded validity check took 4301ms + * Identifier regexp validity check took 22609ms' + */ + public void benchMarkPatternMatching() { + Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); + String[] strings = new String[1000]; + for (int i = 0; i < strings.length; i++) { + strings[i] = i + "-legal_string" + i; + } + + countValid(strings, 1000, "handcoded warmup", FeatureNames::notNeedQuotes); + countValid(strings, 1000, "regexp warmup", (s) -> identifierRegexp.matcher(s).matches()); + + countValid(strings, 100000, "handcoded", FeatureNames::notNeedQuotes); + countValid(strings, 100000, "regexp", (s) -> identifierRegexp.matcher(s).matches()); + } + + private void countValid(String [] strings, int numReps, String text, Function<String, Boolean> func) { + long start = System.nanoTime(); + int validCount = 0; + for (int i = 0; i < numReps; i++) { + for (String s : strings) { + if (func.apply(s)) validCount++; + } + } + long end = System.nanoTime(); + assertEquals(strings.length * numReps, validCount); + System.out.println("Identifier " + text + " validity check took " + (end - start)/1000000 + "ms"); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 4453f327bb4..2490359ba1e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -1,12 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.yolean.Exceptions; import org.junit.Test; @@ -27,7 +24,7 @@ public class IncorrectRankingExpressionFileRefTestCase extends SchemaTestCase { Search search = SearchBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", registry, new QueryProfileRegistry()); - new DerivedConfiguration(search, new BaseDeployLogger(), new TestProperties(), registry, new QueryProfileRegistry(), new ImportedMlModels()); // cause rank profile parsing + new DerivedConfiguration(search, registry); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { String message = Exceptions.toMessageString(e); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index c1fe5e42dfa..8c0c594c0cb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -1,12 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.yolean.Exceptions; import org.junit.Test; @@ -27,7 +23,7 @@ public class RankingExpressionValidationTestCase extends SchemaTestCase { try { RankProfileRegistry registry = new RankProfileRegistry(); Search search = importWithExpression(expression, registry); - new DerivedConfiguration(search, new BaseDeployLogger(), new TestProperties(), registry, new QueryProfileRegistry(), new ImportedMlModels()); // cause rank profile parsing + new DerivedConfiguration(search, registry); // cause rank profile parsing fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 95137e2a080..737598a08d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; @@ -49,15 +50,14 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase { properties, builder.getRankProfileRegistry(), builder.getQueryProfileRegistry(), - new ImportedMlModels()); + new ImportedMlModels(), new InThreadExecutorService()); return export(dirName, builder, config); } DerivedConfiguration derive(String dirName, SearchBuilder builder, Search search) throws IOException { DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry(), - builder.getQueryProfileRegistry(), - new ImportedMlModels()); + builder.getQueryProfileRegistry()); return export(dirName, builder, config); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 139dae65c63..ffcdc076d77 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -9,7 +9,6 @@ import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; /** @@ -32,7 +31,7 @@ public class EmptyRankProfileTestCase extends SchemaTestCase { doc.addField(new SDField("c", DataType.STRING)); search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); - new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); + new DerivedConfiguration(search, rankProfileRegistry); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 370315ad08b..138fb333621 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -105,7 +105,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testRankExpression() throws IOException, ParseException { assertCorrectDeriving("rankexpression", null, - new TestProperties().useExternalRankExpression(true), new TestableDeployLogger()); + new TestProperties().useExternalRankExpression(true).largeRankExpressionLimit(1024), new TestableDeployLogger()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index 2e36319bd8c..e9e618cb333 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -11,7 +11,6 @@ import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processing; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -42,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); - DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); + DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry); // Check attribute fields derived.getAttributeFields(); // TODO: assert content @@ -73,7 +72,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); - DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(),new ImportedMlModels()); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); // Check il script addition assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", @@ -100,7 +99,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { field2.setLiteralBoost(20); search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); - new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); + new DerivedConfiguration(search, rankProfileRegistry); assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", "clear_state | guard { input body | tokenize normalize stem:\"BEST\" | summary body | index body; }", "clear_state | guard { input title | tokenize | index title_literal; }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java index d0712aaeb4d..861ae10cdf6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -14,6 +15,6 @@ import java.io.IOException; public class RankProfilesTestCase extends AbstractExportingTestCase { @Test public void testRankProfiles() throws IOException, ParseException { - assertCorrectDeriving("rankprofiles"); + assertCorrectDeriving("rankprofiles", null, new TestProperties(), new TestableDeployLogger()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java index 83e11c365f8..51961a6083d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -1,11 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; import java.io.File; @@ -33,10 +31,7 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase { toDir.mkdirs(); deleteContent(toDir); - DerivedConfiguration config = new DerivedConfiguration(search, - builder.getRankProfileRegistry(), - new QueryProfileRegistry(), - new ImportedMlModels()); + DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry()); config.export(toDirName); checkDir(toDirName, expectedResultsDirName); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 364eb9dc014..4f7a6118296 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -3,14 +3,12 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processing; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -34,7 +32,7 @@ public class TypeConversionTestCase extends SchemaTestCase { document.addField(a); new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); - DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java index 809ccdb3a3a..259d0d67106 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java @@ -1,16 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; import java.io.IOException; @@ -90,7 +86,7 @@ public class ImplicitSearchFieldsTestCase extends SchemaTestCase { sb.importFile("src/test/examples/nextgen/simple.sd"); sb.build(); assertNotNull(sb.getSearch()); - new DerivedConfiguration(sb.getSearch(), new BaseDeployLogger(), new TestProperties(), sb.getRankProfileRegistry(), new QueryProfileRegistry(), new ImportedMlModels()); + new DerivedConfiguration(sb.getSearch(), sb.getRankProfileRegistry()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 9dd569a2f64..00ac5ac5405 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.collections.Pair; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; @@ -22,10 +21,12 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class RankingExpressionsTestCase extends SchemaTestCase { @@ -75,7 +76,53 @@ public class RankingExpressionsTestCase extends SchemaTestCase { public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { RankProfileRegistry registry = new RankProfileRegistry(); Search search = createSearch("src/test/examples/rankingexpressioninfile", new TestProperties(), registry); - new DerivedConfiguration(search, new BaseDeployLogger(), new TestProperties(), registry, new QueryProfileRegistry(), new ImportedMlModels()); // rank profile parsing happens during deriving + new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving } + private void verifyProfile(RankProfile profile, List<String> expectedFunctions, List<Pair<String, String>> rankProperties, + LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels models, + AttributeFields attributes, ModelContext.Properties properties) { + var functions = profile.getFunctions(); + assertEquals(expectedFunctions.size(), functions.size()); + for (String func : expectedFunctions) { + assertTrue(functions.containsKey(func)); + } + + RawRankProfile raw = new RawRankProfile(profile, largeExpressions, queryProfiles, models, attributes, properties); + assertEquals(rankProperties.size(), raw.configProperties().size()); + for (int i = 0; i < rankProperties.size(); i++) { + assertEquals(rankProperties.get(i).getFirst(), raw.configProperties().get(i).getFirst()); + assertEquals(rankProperties.get(i).getSecond(), raw.configProperties().get(i).getSecond()); + } + } + + private void verifySearch(Search search, RankProfileRegistry rankProfileRegistry, LargeRankExpressions largeExpressions, + QueryProfileRegistry queryProfiles, ImportedMlModels models, ModelContext.Properties properties) + { + AttributeFields attributes = new AttributeFields(search); + + verifyProfile(rankProfileRegistry.get(search, "base"), Arrays.asList("large_f", "large_m"), + Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", "base.large_f"), new Pair<>("rankingExpression(large_m).expressionName", "base.large_m")), + largeExpressions, queryProfiles, models, attributes, properties); + for (String child : Arrays.asList("child_a", "child_b")) { + verifyProfile(rankProfileRegistry.get(search, child), Arrays.asList("large_f", "large_m", "large_local_f", "large_local_m"), + Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", child + ".large_f"), new Pair<>("rankingExpression(large_m).expressionName", child + ".large_m"), + new Pair<>("rankingExpression(large_local_f).expressionName", child + ".large_local_f"), new Pair<>("rankingExpression(large_local_m).expressionName", child + ".large_local_m"), + new Pair<>("vespa.rank.firstphase", "rankingExpression(firstphase)"), new Pair<>("rankingExpression(firstphase).expressionName", child + ".firstphase")), + largeExpressions, queryProfiles, models, attributes, properties); + } + } + + @Test + public void testLargeInheritedFunctions() throws IOException, ParseException { + ModelContext.Properties properties = new TestProperties().useExternalRankExpression(true).largeRankExpressionLimit(50); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry()); + QueryProfileRegistry queryProfiles = new QueryProfileRegistry(); + ImportedMlModels models = new ImportedMlModels(); + Search search = createSearch("src/test/examples/largerankingexpressions", properties, rankProfileRegistry); + verifySearch(search, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); + // Need to verify that second derivation works as that will happen if same sd is used in multiple content clusters + verifySearch(search, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index e99a92b530a..a975a02e149 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -47,8 +47,7 @@ public class QuotaValidatorTest { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!", - ValidationTester.censorNumbers(e.getMessage())); + assertEquals("Deployment would make your tenant exceed its quota and has been blocked! Please contact support to update your plan.", e.getMessage()); } } @@ -59,8 +58,7 @@ public class QuotaValidatorTest { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("publiccd: Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!", - ValidationTester.censorNumbers(e.getMessage())); + assertEquals("publiccd: Deployment would make your tenant exceed its quota and has been blocked! Please contact support to update your plan.", e.getMessage()); } } @@ -71,8 +69,7 @@ public class QuotaValidatorTest { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("publiccd: Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!", - ValidationTester.censorNumbers(e.getMessage())); + assertEquals("publiccd: Deployment would make your tenant exceed its quota and has been blocked! Please contact support to update your plan.", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index e720238901b..3e87bdad1dc 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -210,9 +210,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { DeployState deployState = DeployState.createTestState(); final DomAdminV2Builder domAdminBuilder = - new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, - deployState.getFileRegistry().fileSourceHost(), multitenant, - configServerSpecs); + new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, multitenant, configServerSpecs); Admin admin = domAdminBuilder.build(deployState, root, xml); admin.addPerHostServices(root.hostSystem().getHosts(), deployState); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index ee9c9ccb681..dd4f4c3b7d5 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -123,8 +123,8 @@ public class ContainerClusterTest { public void requireThatHeapSizeAsPercentageOfPhysicalMemoryForHostedAndNot() { boolean hosted = true; boolean combined = true; // a cluster running on content nodes (only relevant with hosted) - verifyHeapSizeAsPercentageOfPhysicalMemory( hosted, ! combined, null, 60); - verifyHeapSizeAsPercentageOfPhysicalMemory( hosted, combined, null, 17); + verifyHeapSizeAsPercentageOfPhysicalMemory( hosted, ! combined, null, 70); + verifyHeapSizeAsPercentageOfPhysicalMemory( hosted, combined, null, 18); verifyHeapSizeAsPercentageOfPhysicalMemory(! hosted, ! combined, null, 0); // Explicit value overrides all defaults diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 13d02fc1fb8..c79238ed1d6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -4,12 +4,15 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.container.ComponentsConfig; import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.metrics.MetricsmanagerConfig; @@ -798,13 +801,25 @@ public class ContentClusterTest extends ContentBaseTest { assertPrepareRestartCommand(createClusterWithFlushOnShutdownOverride(true, true)); } - private static ContentCluster createOneNodeCluster(boolean isHostedVespa) throws Exception { - return createOneNodeCluster("<content version=\"1.0\" id=\"mockcluster\">" + + private static String oneNodeClusterXml() { + return "<content version=\"1.0\" id=\"mockcluster\">" + " <documents/>" + " <group>" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + " </group>" + - "</content>", isHostedVespa); + "</content>"; + } + + private static ContentCluster createOneNodeCluster(boolean isHostedVespa) throws Exception { + return createOneNodeCluster(oneNodeClusterXml(), new TestProperties().setHostedVespa(isHostedVespa)); + } + + private static ContentCluster createOneNodeCluster(TestProperties props) throws Exception { + return createOneNodeCluster(oneNodeClusterXml(), props); + } + + private static ContentCluster createOneNodeCluster(TestProperties props, Optional<Flavor> flavor) throws Exception { + return createOneNodeCluster(oneNodeClusterXml(), props, flavor); } private static ContentCluster createClusterWithFlushOnShutdownOverride(boolean flushOnShutdown, boolean isHostedVespa) throws Exception { @@ -818,13 +833,20 @@ public class ContentClusterTest extends ContentBaseTest { " <group>" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + " </group>" + - "</content>", isHostedVespa); + "</content>", new TestProperties().setHostedVespa(isHostedVespa)); } - private static ContentCluster createOneNodeCluster(String clusterXml, boolean isHostedVespa) throws Exception { + private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props) throws Exception { + return createOneNodeCluster(clusterXml, props, Optional.empty()); + } + + private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, Optional<Flavor> flavor) throws Exception { DeployState.Builder deployStateBuilder = new DeployState.Builder() - .properties(new TestProperties().setHostedVespa(isHostedVespa)); - MockRoot root = ContentClusterUtils.createMockRoot(Collections.emptyList(), deployStateBuilder); + .properties(props); + MockRoot root = flavor.isPresent() ? + ContentClusterUtils.createMockRoot(new SingleNodeProvisioner(flavor.get()), + Collections.emptyList(), deployStateBuilder) : + ContentClusterUtils.createMockRoot(Collections.emptyList(), deployStateBuilder); ContentCluster cluster = ContentClusterUtils.createCluster(clusterXml, root); root.freezeModelTopology(); cluster.validate(); @@ -1015,41 +1037,58 @@ public class ContentClusterTest extends ContentBaseTest { } } - private StorDistributormanagerConfig resolveStorDistributormanagerConfig(TestProperties props) { - VespaModel model = createEnd2EndOneNode(props); + private StorDistributormanagerConfig resolveStorDistributormanagerConfig(TestProperties props) throws Exception { + var cc = createOneNodeCluster(props); - ContentCluster cc = model.getContentClusters().get("storage"); var builder = new StorDistributormanagerConfig.Builder(); cc.getDistributorNodes().getConfig(builder); return (new StorDistributormanagerConfig(builder)); } - private int resolveMaxInhibitedGroupsConfigWithFeatureFlag(int maxGroups) { + private int resolveMaxInhibitedGroupsConfigWithFeatureFlag(int maxGroups) throws Exception { var cfg = resolveStorDistributormanagerConfig(new TestProperties().maxActivationInhibitedOutOfSyncGroups(maxGroups)); return cfg.max_activation_inhibited_out_of_sync_groups(); } @Test - public void default_distributor_max_inhibited_group_activation_config_controlled_by_properties() { + public void default_distributor_max_inhibited_group_activation_config_controlled_by_properties() throws Exception { assertEquals(0, resolveMaxInhibitedGroupsConfigWithFeatureFlag(0)); assertEquals(2, resolveMaxInhibitedGroupsConfigWithFeatureFlag(2)); } - private int resolveNumDistributorStripesConfigWithFeatureFlag(TestProperties props) { - var cfg = resolveStorDistributormanagerConfig(props); - return cfg.num_distributor_stripes(); + private int resolveNumDistributorStripesConfigWithFeatureFlag(TestProperties props, Optional<Flavor> flavor) throws Exception { + var cc = createOneNodeCluster(props, flavor); + var builder = new StorDistributormanagerConfig.Builder(); + cc.getDistributorNodes().getChildren().get("0").getConfig(builder); + return (new StorDistributormanagerConfig(builder)).num_distributor_stripes(); } - private int resolveNumDistributorStripesConfigWithFeatureFlag(int numStripes) { - return resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties().setNumDistributorStripes(numStripes)); + private int resolveNumDistributorStripesConfigWithFeatureFlag(int numStripes) throws Exception { + return resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties().setNumDistributorStripes(numStripes), Optional.empty()); + } + + private int resolveTunedNumDistributorStripesConfig(int numCpuCores) throws Exception { + var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minCpuCores(numCpuCores))); + return resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties().setNumDistributorStripes(-1), + Optional.of(flavor)); } @Test - public void num_distributor_stripes_config_controlled_by_properties() { - assertEquals(0, resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties())); + public void num_distributor_stripes_config_controlled_by_properties() throws Exception { + assertEquals(0, resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties(), Optional.empty())); assertEquals(0, resolveNumDistributorStripesConfigWithFeatureFlag(0)); assertEquals(1, resolveNumDistributorStripesConfigWithFeatureFlag(1)); + assertEquals(4, resolveNumDistributorStripesConfigWithFeatureFlag(4)); + } + + @Test + public void num_distributor_stripes_config_tuned_by_flavor() throws Exception { + assertEquals(1, resolveTunedNumDistributorStripesConfig(1)); + assertEquals(1, resolveTunedNumDistributorStripesConfig(16)); + assertEquals(2, resolveTunedNumDistributorStripesConfig(17)); + assertEquals(2, resolveTunedNumDistributorStripesConfig(64)); + assertEquals(4, resolveTunedNumDistributorStripesConfig(65)); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 46bd005deb6..b9a98dab432 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -99,7 +99,7 @@ public class ContentSearchClusterTest { private static void assertClusterControllerResourceLimits(double expDiskLimit, double expMemoryLimit, ContentCluster cluster) { var limits = getFleetcontrollerConfig(cluster).cluster_feed_block_limit(); - assertEquals(4, limits.size()); + assertEquals(3, limits.size()); assertEquals(expDiskLimit, limits.get("disk"), EPSILON); assertEquals(expMemoryLimit, limits.get("memory"), EPSILON); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index d17f2d36b9c..cc1b96dc588 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -124,11 +124,10 @@ public class FleetControllerClusterTest { private void assertLimits(double expDisk, double expMemory, FleetcontrollerConfig config) { var limits = config.cluster_feed_block_limit(); - assertEquals(4, limits.size()); + assertEquals(3, limits.size()); assertEquals(expDisk, limits.get("disk"), DELTA); assertEquals(expMemory, limits.get("memory"), DELTA); - assertEquals(0.89, limits.get("attribute-enum-store"), DELTA); - assertEquals(0.89, limits.get("attribute-multi-value"), DELTA); + assertEquals(0.89, limits.get("attribute-address-space"), DELTA); } private FleetcontrollerConfig getConfigForResourceLimitsTuning(Double diskLimit, Double memoryLimit) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 9a74c328dd8..ac4a989c89b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.utils; import com.yahoo.config.application.api.ApplicationPackage; @@ -9,13 +9,11 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; -import com.yahoo.net.HostName; import com.yahoo.text.XML; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.DefaultMonitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import org.w3c.dom.Document; import java.util.Collections; @@ -37,7 +35,7 @@ public class ContentClusterUtils { return createMockRoot(provisioner, schemas, new DeployState.Builder()); } - private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas, DeployState.Builder deployStateBuilder) { + public static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas, DeployState.Builder deployStateBuilder) { ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSchemas(schemas).build(); DeployState deployState = deployStateBuilder.applicationPackage(applicationPackage) .modelHostProvisioner(provisioner) @@ -58,11 +56,15 @@ public class ContentClusterUtils { } public static ContentCluster createCluster(String clusterXml, MockRoot root) { + ConfigModelContext.ApplicationType applicationType = ConfigModelContext.ApplicationType.DEFAULT; + Admin admin = new Admin(root, + new DefaultMonitoring(), + new Metrics(), + false, + root.getDeployState().isHosted(), + applicationType); Document doc = XML.getDocument(clusterXml); - Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), false, - new FileDistributionConfigProducer(root, HostName.getLocalhost()), - root.getDeployState().isHosted()); - ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), + ConfigModelContext context = ConfigModelContext.create(applicationType, root.getDeployState(), null,null, root, null); return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java deleted file mode 100644 index bd158aedea9..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.filedistribution; - -import com.yahoo.config.FileReference; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.test.MockHosts; -import org.junit.Test; - -import java.util.Arrays; -import java.util.HashSet; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -/** - * @author bratseth - */ -public class FileDistributorTestCase { - - @Test - public void fileDistributor() { - MockHosts hosts = new MockHosts(); - FileRegistry fileRegistry = new MockFileRegistry(); - FileDistributor fileDistributor = new FileDistributor("mockhost"); - - String file1 = "component/path1"; - String file2 = "component/path2"; - FileReference ref1 = fileRegistry.addFile(file1); - FileReference ref2 = fileRegistry.addFile(file2); - fileDistributor.sendFileReference(ref1, hosts.host1); - fileDistributor.sendFileReference(ref2, hosts.host1); - fileDistributor.sendFileReference(ref1, hosts.host2); // same file reference as above - fileDistributor.sendFileReference(ref2, hosts.host3); - - assertEquals(new HashSet<>(Arrays.asList(hosts.host1, hosts.host2, hosts.host3)), - fileDistributor.getTargetHosts()); - - assertNotNull(ref1); - assertNotNull(ref2); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepositoryTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepositoryTestCase.java new file mode 100644 index 00000000000..cffd1cb1e2e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileReferencesRepositoryTestCase.java @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; + +/** + * @author bratseth + */ +public class FileReferencesRepositoryTestCase { + + @Test + public void fileDistributor() { + FileRegistry fileRegistry = new MockFileRegistry(); + FileReferencesRepository fileReferencesRepository = new FileReferencesRepository(); + + String file1 = "component/path1"; + String file2 = "component/path2"; + FileReference ref1 = fileRegistry.addFile(file1); + FileReference ref2 = fileRegistry.addFile(file2); + fileReferencesRepository.add(ref1); + fileReferencesRepository.add(ref2); + fileReferencesRepository.add(ref1); // same file reference as above + fileReferencesRepository.add(ref2); + + assertNotNull(ref1); + assertNotNull(ref2); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index bdae01d5e09..2dec6a99af3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -124,7 +124,7 @@ public class ModelEvaluationTest { fileMap.put(onnxModel.fileref().value(), appDir.append(onnxModel.fileref().value()).toFile()); } FileAcquirer fileAcquirer = MockFileAcquirer.returnFiles(fileMap); - ModelsEvaluator evaluator = new ModelsEvaluator(config, constantsConfig, onnxModelsConfig, fileAcquirer); + ModelsEvaluator evaluator = new ModelsEvaluator(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer); assertEquals(5, evaluator.models().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index 5c64dd9f8cd..b369560be74 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -14,6 +14,7 @@ import com.yahoo.tensor.Tensor; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.junit.Test; @@ -38,6 +39,26 @@ import static org.junit.Assert.assertTrue; public class StatelessOnnxEvaluationTest { @Test + public void testStatelessOnnxModelNameCollision() throws IOException { + Path appDir = Path.fromString("src/test/cfg/application/onnx_name_collision"); + try { + ImportedModelTester tester = new ImportedModelTester("onnx", appDir); + VespaModel model = tester.createVespaModel(); + ApplicationContainerCluster cluster = model.getContainerClusters().get("container"); + RankProfilesConfig.Builder b = new RankProfilesConfig.Builder(); + cluster.getConfig(b); + RankProfilesConfig config = new RankProfilesConfig(b); + assertEquals(2, config.rankprofile().size()); + + Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet()); + assertTrue(modelNames.contains("foobar")); + assertTrue(modelNames.contains("barfoo")); + } finally { + IOUtils.recursiveDeleteDir(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + } + + @Test public void testStatelessOnnxModelEvaluation() throws IOException { Path appDir = Path.fromString("src/test/cfg/application/onnx"); Path storedAppDir = appDir.append("copy"); @@ -73,6 +94,10 @@ public class StatelessOnnxEvaluationTest { cluster.getConfig(cb); RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb); + RankingExpressionsConfig.Builder ce = new RankingExpressionsConfig.Builder(); + cluster.getConfig(ce); + RankingExpressionsConfig expressionsConfig = ce.build(); + OnnxModelsConfig.Builder ob = new OnnxModelsConfig.Builder(); cluster.getConfig(ob); OnnxModelsConfig onnxModelsConfig = new OnnxModelsConfig(ob); @@ -87,7 +112,7 @@ public class StatelessOnnxEvaluationTest { fileMap.put(onnxModel.fileref().value(), appDir.append(onnxModel.fileref().value()).toFile()); } FileAcquirer fileAcquirer = MockFileAcquirer.returnFiles(fileMap); - ModelsEvaluator modelsEvaluator = new ModelsEvaluator(config, constantsConfig, onnxModelsConfig, fileAcquirer); + ModelsEvaluator modelsEvaluator = new ModelsEvaluator(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer); assertEquals(1, modelsEvaluator.models().size()); Model mul = modelsEvaluator.models().get("mul"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 00b4ce02411..0973aa66550 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -21,7 +21,7 @@ import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; public class NodeResourcesTuningTest { private static final double delta = 0.00001; - private static final double combinedFactor = 1 - 17.0/100; + private static final double combinedFactor = 1 - 18.0/100; @Test public void require_that_hwinfo_disk_size_is_set() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java index 773929434cd..34828202592 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.FileNode; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.model.test.MockRoot; @@ -29,7 +28,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class FileSenderTest { @@ -53,12 +51,7 @@ public class FileSenderTest { } @Override - public FileReference addBlob(ByteBuffer blob) { - return null; - } - - @Override - public String fileSourceHost() { + public FileReference addBlob(String name, ByteBuffer blob) { return null; } @@ -80,7 +73,7 @@ public class FileSenderTest { serviceList = new ArrayList<>(); serviceList.add(service); ConfigDefinitionKey key = new ConfigDefinitionKey("myname", "mynamespace"); - def = new ConfigDefinition("myname", "1", "mynamespace"); + def = new ConfigDefinition("myname", "mynamespace"); builder = new ConfigPayloadBuilder(def); Map<ConfigDefinitionKey, ConfigPayloadBuilder> builderMap = new HashMap<>(); builderMap.put(key, builder); |