aboutsummaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorLester Solbakken <lesters@users.noreply.github.com>2021-05-27 10:41:31 +0200
committerGitHub <noreply@github.com>2021-05-27 10:41:31 +0200
commite3d4dbac364216f8d93493d4a5f34835a268fbcf (patch)
tree90bc2cf28e08123a55854c2db1217f556d349a2e /config-model
parent92efe91ec3d7be1902e7ca9c0e290c7859d535af (diff)
parent6b6e59869ab5259a8cd2e382cd2b5164a963a293 (diff)
Merge branch 'master' into lesters/wire-in-stateless-onnx-rt
Diffstat (limited to 'config-model')
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java9
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java6
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java16
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java72
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java6
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java1
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java58
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java35
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java35
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java93
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java65
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java1
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/Search.java48
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java114
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java29
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java29
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java17
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java12
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostResource.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java22
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java14
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java88
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/Container.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java16
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java123
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java13
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java7
-rw-r--r--config-model/src/main/javacc/SDParser.jj14
-rw-r--r--config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java57
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java14
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java15
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java6
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java5
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java3
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java3
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java18
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java3
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java8
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java3
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java30
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java12
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java4
75 files changed, 774 insertions, 526 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
index a77dd65c608..3fd2d009412 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
@@ -432,7 +432,7 @@ public class DeployState implements ConfigDefinitionStore {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage, logger);
SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage);
- SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles, validationParameters);
+ SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, queryProfiles, validationParameters);
return new DeployState(applicationPackage,
searchDocumentModel,
rankProfileRegistry,
@@ -458,16 +458,15 @@ public class DeployState implements ConfigDefinitionStore {
}
private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry,
- DeployLogger logger,
QueryProfiles queryProfiles,
ValidationParameters validationParameters) {
Collection<NamedReader> readers = applicationPackage.getSearchDefinitions();
Map<String, String> names = new LinkedHashMap<>();
- SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry, queryProfiles.getRegistry());
+ SearchBuilder builder = new SearchBuilder(applicationPackage, logger, rankProfileRegistry, queryProfiles.getRegistry());
for (NamedReader reader : readers) {
try {
String readerName = reader.getName();
- String topLevelName = builder.importReader(reader, readerName, logger);
+ String topLevelName = builder.importReader(reader, readerName);
String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX);
names.put(topLevelName, sdName);
if ( ! sdName.equals(topLevelName)) {
@@ -483,7 +482,7 @@ public class DeployState implements ConfigDefinitionStore {
closeIgnoreException(reader.getReader());
}
}
- builder.build(! validationParameters.ignoreValidationErrors(), logger);
+ builder.build(! validationParameters.ignoreValidationErrors());
return SearchDocumentModel.fromBuilderAndNames(builder, names);
}
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index e6bf3a835c6..66e728957f3 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -52,6 +52,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
private double feedConcurrency = 0.5;
private boolean useBucketExecutorForPruneRemoved;
private boolean enableFeedBlockInDistributor = true;
+ private boolean useExternalRankExpression = false;
private int clusterControllerMaxHeapSizeInMb = 128;
private int metricsProxyMaxHeapSizeInMb = 256;
private int maxActivationInhibitedOutOfSyncGroups = 0;
@@ -98,7 +99,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
@Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; }
@Override public int numDistributorStripes() { return numDistributorStripes; }
@Override public boolean allowDisableMtls() { return allowDisableMtls; }
+ @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; }
+ public TestProperties useExternalRankExpression(boolean value) {
+ useExternalRankExpression = value;
+ return this;
+ }
public TestProperties setFeedConcurrency(double feedConcurrency) {
this.feedConcurrency = feedConcurrency;
return this;
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
index 602e0c80d4b..2784c111019 100644
--- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
@@ -43,7 +43,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce
private final String subId;
private String configId = null;
- private List<Service> descendantServices = new ArrayList<>();
+ private final List<Service> descendantServices = new ArrayList<>();
private AbstractConfigProducer parent = null;
@@ -59,8 +59,8 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce
* Creates a new AbstractConfigProducer with the given parent and subId.
* This constructor will add the resulting producer to the children of parent.
*
- * @param parent The parent of this ConfigProducer
- * @param subId The fragment of the config id for the producer
+ * @param parent the parent of this ConfigProducer
+ * @param subId the fragment of the config id for the producer
*/
public AbstractConfigProducer(AbstractConfigProducer parent, String subId) {
this(subId);
@@ -69,7 +69,13 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce
}
}
- protected final void setParent(AbstractConfigProducer parent) {
+ /** Removes this from the config model */
+ protected void remove() {
+ if (parent != null)
+ parent.removeChild(this);
+ }
+
+ protected final void setParent(AbstractConfigProducer<?> parent) {
this.parent = parent;
computeConfigId();
}
@@ -92,7 +98,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce
/**
* Adds a child to this config producer.
*
- * @param child The child config producer to add.
+ * @param child the child config producer to add
*/
protected void addChild(CHILD child) {
if (child == null) {
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
index 2594c64b951..7200c3211ba 100644
--- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
@@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ComponentInfo;
import com.yahoo.config.application.api.UnparsedConfigDefinition;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.component.Version;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ApplicationName;
import com.yahoo.config.provision.InstanceName;
@@ -116,6 +117,7 @@ public class MockApplicationPackage implements ApplicationPackage {
public List<NamedReader> getSearchDefinitions() {
ArrayList<NamedReader> readers = new ArrayList<>();
SearchBuilder searchBuilder = new SearchBuilder(this,
+ new BaseDeployLogger(),
new RankProfileRegistry(),
queryProfileRegistry);
for (String sd : schemas) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
new file mode 100644
index 00000000000..77ce2dd41b5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
@@ -0,0 +1,72 @@
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.utils.FileSender;
+
+import java.util.Collection;
+import java.util.Objects;
+
+public class DistributableResource {
+ public enum PathType { FILE, URI };
+
+ /** The search definition-unique name of this constant */
+ private final String name;
+ private String path = null;
+ private String fileReference = "";
+ private PathType pathType = PathType.FILE;
+
+ public PathType getPathType() {
+ return pathType;
+ }
+
+ public DistributableResource(String name) {
+ this(name, null);
+ }
+ public DistributableResource(String name, String path) {
+ this.name = name;
+ this.path = path;
+ }
+
+ public void setFileName(String fileName) {
+ Objects.requireNonNull(fileName, "Filename cannot be null");
+ this.path = fileName;
+ this.pathType = PathType.FILE;
+ }
+
+ public void setUri(String uri) {
+ Objects.requireNonNull(uri, "uri cannot be null");
+ this.path = uri;
+ this.pathType = PathType.URI;
+ }
+
+ protected void setFileReference(String fileReference) { this.fileReference = fileReference; }
+ /** Initiate sending of this constant to some services over file distribution */
+ public void sendTo(Collection<? extends AbstractService> services) {
+ FileReference reference = (pathType == PathType.FILE)
+ ? FileSender.sendFileToServices(path, services)
+ : FileSender.sendUriToServices(path, services);
+ this.fileReference = reference.value();
+ }
+
+ public String getName() { return name; }
+ public String getFileName() { return path; }
+ public Path getFilePath() { return Path.fromString(path); }
+ public String getUri() { return path; }
+ public String getFileReference() { return fileReference; }
+
+ public void validate() {
+ if (path == null || path.isEmpty())
+ throw new IllegalArgumentException("Distributable resource must have a file or uri.");
+ }
+
+ public String toString() {
+ StringBuilder b = new StringBuilder();
+ b.append("resource '").append(name)
+ .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path)
+ .append("' with ref '").append(fileReference)
+ .append("'");
+ return b.toString();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java
index 5940f908be8..6e17bd0f0bb 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java
@@ -1,6 +1,8 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.searchdefinition.document.SDDocumentType;
/**
@@ -11,8 +13,8 @@ import com.yahoo.searchdefinition.document.SDDocumentType;
*/
public class DocumentOnlySearch extends Search {
- public DocumentOnlySearch() {
- // empty
+ public DocumentOnlySearch(ApplicationPackage applicationPackage, DeployLogger deployLogger) {
+ super(applicationPackage, deployLogger);
}
@Override
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java
index 6ac73ad45a9..efd9ed5e88f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java
@@ -28,6 +28,7 @@ public interface ImmutableSearch {
Reader getRankingExpression(String fileName);
ApplicationPackage applicationPackage();
RankingConstants rankingConstants();
+ RankExpressionFiles rankExpressionFiles();
OnnxModels onnxModels();
Stream<ImmutableSDField> allImportedFields();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java
index 3e5726d6d94..3c42987512b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java
@@ -1,14 +1,9 @@
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition;
-import com.yahoo.config.FileReference;
-import com.yahoo.path.Path;
import com.yahoo.tensor.TensorType;
-import com.yahoo.vespa.model.AbstractService;
import com.yahoo.vespa.model.ml.OnnxModelInfo;
-import com.yahoo.vespa.model.utils.FileSender;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -19,42 +14,26 @@ import java.util.Objects;
*
* @author lesters
*/
-public class OnnxModel {
+public class OnnxModel extends DistributableResource {
- public enum PathType {FILE, URI};
-
- private final String name;
- private PathType pathType = PathType.FILE;
- private String path = null;
- private String fileReference = "";
private OnnxModelInfo modelInfo = null;
private Map<String, String> inputMap = new HashMap<>();
private Map<String, String> outputMap = new HashMap<>();
public OnnxModel(String name) {
- this.name = name;
+ super(name);
}
public OnnxModel(String name, String fileName) {
- this(name);
- this.path = fileName;
+ super(name, fileName);
validate();
}
- public void setFileName(String fileName) {
- Objects.requireNonNull(fileName, "Filename cannot be null");
- this.path = fileName;
- this.pathType = PathType.FILE;
- }
-
+ @Override
public void setUri(String uri) {
throw new IllegalArgumentException("URI for ONNX models are not currently supported");
}
- public PathType getPathType() {
- return pathType;
- }
-
public void addInputNameMapping(String onnxName, String vespaName) {
addInputNameMapping(onnxName, vespaName, true);
}
@@ -90,20 +69,6 @@ public class OnnxModel {
this.modelInfo = modelInfo;
}
- /** Initiate sending of this constant to some services over file distribution */
- public void sendTo(Collection<? extends AbstractService> services) {
- FileReference reference = (pathType == OnnxModel.PathType.FILE)
- ? FileSender.sendFileToServices(path, services)
- : FileSender.sendUriToServices(path, services);
- this.fileReference = reference.value();
- }
-
- public String getName() { return name; }
- public String getFileName() { return path; }
- public Path getFilePath() { return Path.fromString(path); }
- public String getUri() { return path; }
- public String getFileReference() { return fileReference; }
-
public Map<String, String> getInputMap() { return Collections.unmodifiableMap(inputMap); }
public Map<String, String> getOutputMap() { return Collections.unmodifiableMap(outputMap); }
@@ -114,19 +79,4 @@ public class OnnxModel {
TensorType getTensorType(String onnxName, Map<String, TensorType> inputTypes) {
return modelInfo != null ? modelInfo.getTensorType(onnxName, inputTypes) : TensorType.empty;
}
-
- public void validate() {
- if (path == null || path.isEmpty())
- throw new IllegalArgumentException("ONNX models must have a file or uri.");
- }
-
- public String toString() {
- StringBuilder b = new StringBuilder();
- b.append("onnx-model '").append(name)
- .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path)
- .append("' with ref '").append(fileReference)
- .append("'");
- return b.toString();
- }
-
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java
new file mode 100644
index 00000000000..56385efeb0b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java
@@ -0,0 +1,35 @@
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.utils.FileSender;
+
+import java.util.Collection;
+
+public class RankExpressionFile extends DistributableResource {
+
+ public RankExpressionFile(String name, String path) {
+ super(name, path);
+ validate();
+ }
+
+ @Override
+ public void sendTo(Collection<? extends AbstractService> services) {
+ /*
+ * TODO This is a very dirty hack due to using both SEARCH_DEFINITIONS_DIR and SCHEMA_DIR
+ * and doing so inconsistently, combined with using both fields from application package on disk and in zookeeper.
+ * The mess is spread out nicely, but ZookeeperClient, and writeSearchDefinitions and ZkApplicationPackage and FilesApplicationPackage
+ * should be consolidated
+ */
+ try {
+ setFileReference(FileSender.sendFileToServices(ApplicationPackage.SCHEMAS_DIR + "/" + getFileName(), services).value());
+ } catch (IllegalArgumentException e1) {
+ try {
+ setFileReference(FileSender.sendFileToServices(ApplicationPackage.SEARCH_DEFINITIONS_DIR + "/" + getFileName(), services).value());
+ } catch (IllegalArgumentException e2) {
+ throw new IllegalArgumentException("Failed to find expression file '" + getFileName() + "' in '"
+ + ApplicationPackage.SEARCH_DEFINITIONS_DIR + "' or '" + ApplicationPackage.SCHEMAS_DIR + "'.", e2);
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java
new file mode 100644
index 00000000000..ebc91e0693f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java
@@ -0,0 +1,35 @@
+package com.yahoo.searchdefinition;
+
+import com.yahoo.vespa.model.AbstractService;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RankExpressionFiles {
+ private final Map<String, RankExpressionFile> expressions = new HashMap<>();
+
+ public void add(RankExpressionFile expression) {
+ expression.validate();
+ String name = expression.getName();
+ if (expressions.containsKey(name))
+ throw new IllegalArgumentException("Rank expression file '" + name + "' defined twice");
+ expressions.put(name, expression);
+ }
+
+ /** Returns the ranking constant with the given name, or null if not present */
+ public RankExpressionFile get(String name) {
+ return expressions.get(name);
+ }
+
+ /** Returns a read-only map of the ranking constants in this indexed by name */
+ public Map<String, RankExpressionFile> asMap() {
+ return Collections.unmodifiableMap(expressions);
+ }
+
+ /** Initiate sending of these constants to some services over file distribution */
+ public void sendTo(Collection<? extends AbstractService> services) {
+ expressions.values().forEach(constant -> constant.sendTo(services));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index 95b291cf744..b0a7d2aaca2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -52,6 +52,8 @@ import java.util.stream.Stream;
*/
public class RankProfile implements Cloneable {
+ public final static String FIRST_PHASE = "firstphase";
+ public final static String SECOND_PHASE = "secondphase";
/** The search definition-unique name of this rank profile */
private final String name;
@@ -76,6 +78,8 @@ public class RankProfile implements Cloneable {
/** The ranking expression to be used for second phase */
private RankingExpression secondPhaseRanking = null;
+ private Set<String> externalFileExpressions = new HashSet<>();
+
/** Number of hits to be reranked in second phase, -1 means use default */
private int rerankCount = -1;
@@ -165,6 +169,10 @@ public class RankProfile implements Cloneable {
return search != null ? search.rankingConstants() : model.rankingConstants();
}
+ public RankExpressionFiles rankExpressionFiles() {
+ return search != null ? search.rankExpressionFiles() : model.rankExpressionFiles();
+ }
+
public Map<String, OnnxModel> onnxModels() {
return search != null ? search.onnxModels().asMap() : onnxModels.asMap();
}
@@ -316,7 +324,7 @@ public class RankProfile implements Cloneable {
public void addConstant(String name, Value value) {
if (value instanceof TensorValue) {
- TensorType type = ((TensorValue)value).type();
+ TensorType type = value.type();
if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty()))
throw new IllegalArgumentException("Illegal type of constant " + name + " type " + type +
": Dense tensor dimensions must have a size");
@@ -370,9 +378,44 @@ public class RankProfile implements Cloneable {
this.firstPhaseRanking = rankingExpression;
}
+ public String getUniqueExpressionName(String name) {
+ return getName() + "_" + name;
+ }
+ public String getFirstPhaseFile() {
+ String name = FIRST_PHASE;
+ if (externalFileExpressions.contains(name)) {
+ return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName();
+ }
+ if ((firstPhaseRanking == null) && (getInherited() != null)) {
+ return getInherited().getFirstPhaseFile();
+ }
+ return null;
+ }
+
+ public String getSecondPhaseFile() {
+ String name = SECOND_PHASE;
+ if (externalFileExpressions.contains(name)) {
+ return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName();
+ }
+ if ((secondPhaseRanking == null) && (getInherited() != null)) {
+ return getInherited().getSecondPhaseFile();
+ }
+ return null;
+ }
+
+ public String getExpressionFile(String name) {
+ if (externalFileExpressions.contains(name)) {
+ return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName();
+ }
+ if (getInherited() != null) {
+ return getInherited().getExpressionFile(name);
+ }
+ return null;
+ }
+
public void setFirstPhaseRanking(String expression) {
try {
- this.firstPhaseRanking = parseRankingExpression("firstphase", expression);
+ this.firstPhaseRanking = parseRankingExpression(FIRST_PHASE, expression);
}
catch (ParseException e) {
throw new IllegalArgumentException("Illegal first phase ranking function", e);
@@ -389,13 +432,9 @@ public class RankProfile implements Cloneable {
return null;
}
- public void setSecondPhaseRanking(RankingExpression rankingExpression) {
- this.secondPhaseRanking = rankingExpression;
- }
-
public void setSecondPhaseRanking(String expression) {
try {
- this.secondPhaseRanking = parseRankingExpression("secondphase", expression);
+ this.secondPhaseRanking = parseRankingExpression(SECOND_PHASE, expression);
}
catch (ParseException e) {
throw new IllegalArgumentException("Illegal second phase ranking function", e);
@@ -436,7 +475,6 @@ public class RankProfile implements Cloneable {
* the final (with inheritance included) summary features of the given parent.
* The profile must be the profile which is directly inherited by this.
*
- * @param parentProfile
*/
public void setInheritedSummaryFeatures(String parentProfile) {
if ( ! parentProfile.equals(inheritedName))
@@ -496,12 +534,7 @@ public class RankProfile implements Cloneable {
private void addRankProperty(RankProperty rankProperty) {
// Just the usual multimap semantics here
- List<RankProperty> properties = rankProperties.get(rankProperty.getName());
- if (properties == null) {
- properties = new ArrayList<>(1);
- rankProperties.put(rankProperty.getName(), properties);
- }
- properties.add(rankProperty);
+ rankProperties.computeIfAbsent(rankProperty.getName(), (String key) -> new ArrayList<>(1)).add(rankProperty);
}
@Override
@@ -675,18 +708,27 @@ public class RankProfile implements Cloneable {
}
}
- private Reader openRankingExpressionReader(String expName, String expression) {
- if ( ! expression.startsWith("file:")) return new StringReader(expression);
-
+ private static String extractFileName(String expression) {
String fileName = expression.substring("file:".length()).trim();
if ( ! fileName.endsWith(ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX))
fileName = fileName + ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX;
+ return fileName;
+ }
+
+ private Reader openRankingExpressionReader(String expName, String expression) {
+ if ( ! expression.startsWith("file:")) return new StringReader(expression);
+
+ String fileName = extractFileName(expression);
File file = new File(fileName);
- if ( ! (file.isAbsolute()) && file.getPath().contains("/")) // See ticket 4102122
+ if ( ! file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122
throw new IllegalArgumentException("In " + getName() +", " + expName + ", ranking references file '" + file +
"' in subdirectory, which is not supported.");
+ /* TODO balder: Disabled until end-2-end verified
+ rankExpressionFiles().add(new RankExpressionFile(getUniqueExpressionName(expName), fileName));
+ externalFileExpressions.add(expName);
+ */
return search.getRankingExpression(fileName);
}
@@ -918,12 +960,12 @@ public class RankProfile implements Cloneable {
*/
public static class RankSetting implements Serializable {
- private String fieldName;
+ private final String fieldName;
- private Type type;
+ private final Type type;
/** The rank value */
- private Object value;
+ private final Object value;
public enum Type {
@@ -932,10 +974,10 @@ public class RankProfile implements Cloneable {
WEIGHT("weight"),
PREFERBITVECTOR("preferbitvector",true);
- private String name;
+ private final String name;
/** True if this setting really pertains to an index, not a field within an index */
- private boolean isIndexLevel;
+ private final boolean isIndexLevel;
Type(String name) {
this(name,false);
@@ -1008,8 +1050,8 @@ public class RankProfile implements Cloneable {
/** A rank property. Rank properties are Value Objects */
public static class RankProperty implements Serializable {
- private String name;
- private String value;
+ private final String name;
+ private final String value;
public RankProperty(String name, String value) {
this.name = name;
@@ -1084,7 +1126,6 @@ public class RankProfile implements Cloneable {
public void setMinGroups(int value) { minGroups = value; }
public void setCutoffFactor(double value) { cutoffFactor = value; }
public void setCutoffStrategy(Diversity.CutoffStrategy strategy) { cutoffStrategy = strategy; }
- public void setCutoffStrategy(String strategy) { cutoffStrategy = Diversity.CutoffStrategy.valueOf(strategy); }
public String getAttribute() { return attribute; }
public int getMinGroups() { return minGroups; }
public double getCutoffFactor() { return cutoffFactor; }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
index b41cf582204..8e376f13615 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
@@ -1,13 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition;
-import com.yahoo.config.FileReference;
import com.yahoo.tensor.TensorType;
-import com.yahoo.vespa.model.AbstractService;
-import com.yahoo.vespa.model.utils.FileSender;
-
-import java.util.Collection;
-import java.util.Objects;
/**
* A global ranking constant distributed using file distribution.
@@ -17,81 +11,38 @@ import java.util.Objects;
* @author arnej
* @author bratseth
*/
-public class RankingConstant {
-
- public enum PathType {FILE, URI};
-
- /** The search definition-unique name of this constant */
- private final String name;
+public class RankingConstant extends DistributableResource {
private TensorType tensorType = null;
- private String path = null;
- private String fileReference = "";
-
- public PathType getPathType() {
- return pathType;
- }
-
- private PathType pathType = PathType.FILE;
public RankingConstant(String name) {
- this.name = name;
+ super(name);
}
public RankingConstant(String name, TensorType type, String fileName) {
- this(name);
+ super(name, fileName);
this.tensorType = type;
- this.path = fileName;
validate();
}
- public void setFileName(String fileName) {
- Objects.requireNonNull(fileName, "Filename cannot be null");
- this.path = fileName;
- this.pathType = PathType.FILE;
- }
-
- public void setUri(String uri) {
- Objects.requireNonNull(uri, "uri cannot be null");
- this.path = uri;
- this.pathType = PathType.URI;
- }
-
public void setType(TensorType type) {
this.tensorType = type;
}
- /** Initiate sending of this constant to some services over file distribution */
- public void sendTo(Collection<? extends AbstractService> services) {
- FileReference reference = (pathType == RankingConstant.PathType.FILE)
- ? FileSender.sendFileToServices(path, services)
- : FileSender.sendUriToServices(path, services);
- this.fileReference = reference.value();
- }
-
- public String getName() { return name; }
- public String getFileName() { return path; }
- public String getUri() { return path; }
- public String getFileReference() { return fileReference; }
public TensorType getTensorType() { return tensorType; }
public String getType() { return tensorType.toString(); }
public void validate() {
- if (path == null || path.isEmpty())
- throw new IllegalArgumentException("Ranking constants must have a file or uri.");
+ super.validate();
if (tensorType == null)
- throw new IllegalArgumentException("Ranking constant '" + name + "' must have a type.");
+ throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type.");
if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty()))
- throw new IllegalArgumentException("Illegal type in field " + name + " type " + tensorType +
+ throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType +
": Dense tensor dimensions must have a size");
}
public String toString() {
- StringBuilder b = new StringBuilder();
- b.append("constant '").append(name)
- .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path)
- .append("' with ref '").append(fileReference)
- .append("' of type '").append(tensorType)
- .append("'");
+ StringBuilder b = new StringBuilder(super.toString())
+ .append("' of type '").append(tensorType).append("'");
return b.toString();
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
index adefa5566ab..82381aa63fc 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
@@ -40,5 +40,4 @@ public class RankingConstants {
public void sendTo(Collection<? extends AbstractService> services) {
constants.values().forEach(constant -> constant.sendTo(services));
}
-
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
index 9b7434dccab..b07740620f8 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -2,6 +2,8 @@
package com.yahoo.searchdefinition;
import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.document.Field;
import com.yahoo.searchdefinition.derived.SummaryClass;
import com.yahoo.searchdefinition.document.Attribute;
@@ -15,6 +17,7 @@ import com.yahoo.searchdefinition.document.TemporaryImportedFields;
import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
import com.yahoo.vespa.documentmodel.DocumentSummary;
import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.model.AbstractService;
import java.io.Reader;
import java.util.ArrayList;
@@ -27,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
+import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;
@@ -42,7 +46,6 @@ import java.util.stream.Stream;
// Ensure that after the processing step, all implicit instances of the above types are explicitly represented
public class Search implements ImmutableSearch {
- private static final Logger log = Logger.getLogger(Search.class.getName());
private static final String SD_DOC_FIELD_NAME = "sddocname";
private static final List<String> RESERVED_NAMES = Arrays.asList(
"index", "index_url", "summary", "attribute", "select_input", "host", SummaryClass.DOCUMENT_ID_FIELD,
@@ -59,7 +62,7 @@ public class Search implements ImmutableSearch {
private String name;
/** True if this doesn't define a search, just a document type */
- private boolean documentsOnly = false;
+ private final boolean documentsOnly;
private boolean rawAsBase64 = false;
@@ -78,6 +81,9 @@ public class Search implements ImmutableSearch {
/** The explicitly defined summaries of this search definition. _Must_ preserve order. */
private final Map<String, DocumentSummary> summaries = new LinkedHashMap<>();
+ /** External rank expression files of this */
+ private final RankExpressionFiles rankExpressionFiles = new RankExpressionFiles();
+
/** Ranking constants of this */
private final RankingConstants rankingConstants = new RankingConstants();
@@ -88,26 +94,33 @@ public class Search implements ImmutableSearch {
private Optional<ImportedFields> importedFields = Optional.empty();
private final ApplicationPackage applicationPackage;
+ private final DeployLogger deployLogger;
- /**
- * Creates a search definition which just holds a set of documents which should not (here, directly) be searchable
- */
- protected Search() {
- applicationPackage = null;
- documentsOnly = true;
+ /** Testin only */
+ public Search(String name) {
+ this(name, null, new BaseDeployLogger());
}
-
/**
* Creates a proper search definition
*
* @param name of the the searchdefinition
* @param applicationPackage the application containing this
*/
- public Search(String name, ApplicationPackage applicationPackage) {
- this.applicationPackage = applicationPackage;
+ public Search(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger) {
+ this(applicationPackage, deployLogger, false);
this.name = name;
}
+ protected Search(ApplicationPackage applicationPackage, DeployLogger deployLogger) {
+ this(applicationPackage, deployLogger, true);
+ }
+
+ private Search(ApplicationPackage applicationPackage, DeployLogger deployLogger, boolean documentsOnly) {
+ this.applicationPackage = applicationPackage;
+ this.deployLogger = deployLogger;
+ this.documentsOnly = documentsOnly;
+ }
+
protected void setName(String name) {
this.name = name;
}
@@ -172,11 +185,20 @@ public class Search implements ImmutableSearch {
}
@Override
+ public RankExpressionFiles rankExpressionFiles() { return rankExpressionFiles; }
+
+ @Override
public RankingConstants rankingConstants() { return rankingConstants; }
@Override
public OnnxModels onnxModels() { return onnxModels; }
+ public void sendTo(Collection<? extends AbstractService> services) {
+ rankingConstants.sendTo(services);
+ rankExpressionFiles.sendTo(services);
+ onnxModels.sendTo(services);
+ }
+
public Optional<TemporaryImportedFields> temporaryImportedFields() {
return temporaryImportedFields;
}
@@ -310,7 +332,7 @@ public class Search implements ImmutableSearch {
*/
public void addExtraField(SDField field) {
if (fields.containsKey(field.getName())) {
- log.warning("Duplicate field " + field.getName() + " in search definition " + getName());
+ deployLogger.logApplicationPackage(Level.WARNING, "Duplicate field " + field.getName() + " in search definition " + getName());
} else {
field.setIsExtraField(true);
fields.put(field.getName(), field);
@@ -422,7 +444,7 @@ public class Search implements ImmutableSearch {
if (current.getRankType() != null &&
!consolidated.getRankType().equals(current.getRankType()))
{
- log.warning("Conflicting rank type settings for " +
+ deployLogger.logApplicationPackage(Level.WARNING, "Conflicting rank type settings for " +
first.getName() + " in " + this + ", using " +
consolidated.getRankType());
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 852fdc17726..fc110504f4d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -41,53 +41,72 @@ import java.util.List;
public class SearchBuilder {
private final DocumentTypeManager docTypeMgr = new DocumentTypeManager();
- private List<Search> searchList = new LinkedList<>();
- private ApplicationPackage app;
- private boolean isBuilt = false;
- private DocumentModel model = new DocumentModel();
+ private final DocumentModel model = new DocumentModel();
+ private final ApplicationPackage app;
private final RankProfileRegistry rankProfileRegistry;
private final QueryProfileRegistry queryProfileRegistry;
+ private final DeployLogger deployLogger;
+ private List<Search> searchList = new LinkedList<>();
+ private boolean isBuilt = false;
/** True to build the document aspect only, skipping instantiation of rank profiles */
private final boolean documentsOnly;
/** For testing only */
public SearchBuilder() {
- this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry());
+ this(new RankProfileRegistry(), new QueryProfileRegistry());
+ }
+
+ /** For testing only */
+ public SearchBuilder(DeployLogger deployLogger) {
+ this(MockApplicationPackage.createEmpty(), deployLogger);
+ }
+
+ /** For testing only */
+ public SearchBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) {
+ this(MockApplicationPackage.createEmpty(), deployLogger, rankProfileRegistry);
}
/** Used for generating documents for typed access to document fields in Java */
public SearchBuilder(boolean documentsOnly) {
- this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly);
+ this(MockApplicationPackage.createEmpty(), new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly);
}
/** For testing only */
- public SearchBuilder(ApplicationPackage app) {
- this(app, new RankProfileRegistry(), new QueryProfileRegistry());
+ public SearchBuilder(ApplicationPackage app, DeployLogger deployLogger) {
+ this(app, deployLogger, new RankProfileRegistry(), new QueryProfileRegistry());
+ }
+
+ /** For testing only */
+ public SearchBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) {
+ this(app, deployLogger, rankProfileRegistry, new QueryProfileRegistry());
}
/** For testing only */
public SearchBuilder(RankProfileRegistry rankProfileRegistry) {
- this(MockApplicationPackage.createEmpty(), rankProfileRegistry, new QueryProfileRegistry());
+ this(rankProfileRegistry, new QueryProfileRegistry());
}
/** For testing only */
public SearchBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) {
- this(MockApplicationPackage.createEmpty(), rankProfileRegistry, queryProfileRegistry);
+ this(MockApplicationPackage.createEmpty(), new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry);
}
public SearchBuilder(ApplicationPackage app,
+ DeployLogger deployLogger,
RankProfileRegistry rankProfileRegistry,
QueryProfileRegistry queryProfileRegistry) {
- this(app, rankProfileRegistry, queryProfileRegistry, false);
+ this(app, deployLogger, rankProfileRegistry, queryProfileRegistry, false);
}
- public SearchBuilder(ApplicationPackage app,
- RankProfileRegistry rankProfileRegistry,
- QueryProfileRegistry queryProfileRegistry,
- boolean documentsOnly) {
+ private SearchBuilder(ApplicationPackage app,
+ DeployLogger deployLogger,
+ RankProfileRegistry rankProfileRegistry,
+ QueryProfileRegistry queryProfileRegistry,
+ boolean documentsOnly) {
this.app = app;
this.rankProfileRegistry = rankProfileRegistry;
this.queryProfileRegistry = queryProfileRegistry;
+ this.deployLogger = deployLogger;
this.documentsOnly = documentsOnly;
}
@@ -95,29 +114,17 @@ public class SearchBuilder {
* Import search definition.
*
* @param fileName The name of the file to import.
- * @param deployLogger Logger for deploy messages.
* @return The name of the imported object.
* @throws IOException Thrown if the file can not be read for some reason.
* @throws ParseException Thrown if the file does not contain a valid search definition. ```
*/
- public String importFile(String fileName, DeployLogger deployLogger) throws IOException, ParseException {
+ public String importFile(String fileName) throws IOException, ParseException {
File file = new File(fileName);
- return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent(), deployLogger);
+ return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent());
}
- /**
- * Import search definition.
- *
- * @param fileName The name of the file to import.
- * @return The name of the imported object.
- * @throws IOException Thrown if the file can not be read for some reason.
- * @throws ParseException Thrown if the file does not contain a valid search definition.
- */
- public String importFile(String fileName) throws IOException, ParseException {
- return importFile(fileName, new BaseDeployLogger());
- }
- public String importFile(Path file) throws IOException, ParseException {
- return importFile(file.toString(), new BaseDeployLogger());
+ private String importFile(Path file) throws IOException, ParseException {
+ return importFile(file.toString());
}
/**
@@ -129,17 +136,8 @@ public class SearchBuilder {
* @return The name of the imported object.
* @throws ParseException Thrown if the file does not contain a valid search definition.
*/
- public String importReader(NamedReader reader, String searchDefDir, DeployLogger deployLogger) throws IOException, ParseException {
- return importString(IOUtils.readAll(reader), searchDefDir, deployLogger);
- }
-
- /**
- * See #{@link #importReader}
- *
- * Convenience, should only be used for testing as logs will be swallowed.
- */
public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException {
- return importString(IOUtils.readAll(reader), searchDefDir, new BaseDeployLogger());
+ return importString(IOUtils.readAll(reader), searchDefDir);
}
/**
@@ -150,21 +148,10 @@ public class SearchBuilder {
* @throws ParseException thrown if the file does not contain a valid search definition.
*/
public String importString(String str) throws ParseException {
- return importString(str, null, new BaseDeployLogger());
- }
-
- /**
- * Import search definition.
- *
- * @param str the string to parse.
- * @return the name of the imported object.
- * @throws ParseException thrown if the file does not contain a valid search definition.
- */
- public String importString(String str, DeployLogger logger) throws ParseException {
- return importString(str, null, logger);
+ return importString(str, null);
}
- private String importString(String str, String searchDefDir, DeployLogger deployLogger) throws ParseException {
+ private String importString(String str, String searchDefDir) throws ParseException {
SimpleCharStream stream = new SimpleCharStream(str);
try {
return importRawSearch(new SDParser(stream, deployLogger, app, rankProfileRegistry, documentsOnly)
@@ -208,7 +195,7 @@ public class SearchBuilder {
* @throws IllegalStateException Thrown if this method has already been called.
*/
public void build() {
- build(true, new BaseDeployLogger());
+ build(true);
}
/**
@@ -216,9 +203,8 @@ public class SearchBuilder {
* #getSearch(String)} method.
*
* @throws IllegalStateException Thrown if this method has already been called.
- * @param deployLogger The logger to use during build
*/
- public void build(boolean validate, DeployLogger deployLogger) {
+ public void build(boolean validate) {
if (isBuilt) throw new IllegalStateException("Model already built");
List<Search> built = new ArrayList<>();
@@ -325,9 +311,9 @@ public class SearchBuilder {
}
public static SearchBuilder createFromString(String sd, DeployLogger logger) throws ParseException {
- SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty());
- builder.importString(sd, logger);
- builder.build(true, logger);
+ SearchBuilder builder = new SearchBuilder(logger);
+ builder.importString(sd);
+ builder.build(true);
return builder;
}
@@ -386,12 +372,13 @@ public class SearchBuilder {
QueryProfileRegistry queryprofileRegistry)
throws IOException, ParseException {
SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(),
+ deployLogger,
rankProfileRegistry,
queryprofileRegistry);
for (String fileName : fileNames) {
builder.importFile(fileName);
}
- builder.build(true, deployLogger);
+ builder.build(true);
return builder;
}
@@ -413,15 +400,16 @@ public class SearchBuilder {
public static SearchBuilder createFromDirectory(String dir,
RankProfileRegistry rankProfileRegistry,
QueryProfileRegistry queryProfileRegistry,
- DeployLogger logger,
+ DeployLogger deployLogger,
ApplicationPackage applicationPackage) throws IOException, ParseException {
SearchBuilder builder = new SearchBuilder(applicationPackage,
+ deployLogger,
rankProfileRegistry,
queryProfileRegistry);
for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) {
builder.importFile(i.next());
}
- builder.build(true, logger);
+ builder.build(true);
return builder;
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
index 4d6ce783947..52d99c523ea 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -85,7 +85,8 @@ public class DerivedConfiguration {
summaries = new Summaries(search, deployLogger);
summaryMap = new SummaryMap(search);
juniperrc = new Juniperrc(search);
- rankProfileList = new RankProfileList(search, search.rankingConstants(), attributeFields, rankProfileRegistry, queryProfiles, importedModels, deployProperties);
+ rankProfileList = new RankProfileList(search, search.rankingConstants(), search.rankExpressionFiles(), attributeFields,
+ rankProfileRegistry, queryProfiles, importedModels, deployProperties);
indexingScript = new IndexingScript(search);
indexInfo = new IndexInfo(search);
indexSchema = new IndexSchema(search);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
index 42fa1df802b..d414b9ed79f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
@@ -6,6 +6,8 @@ import com.yahoo.config.model.api.ModelContext;
import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.OnnxModel;
import com.yahoo.searchdefinition.OnnxModels;
+import com.yahoo.searchdefinition.RankExpressionFile;
+import com.yahoo.searchdefinition.RankExpressionFiles;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.RankingConstant;
import com.yahoo.searchdefinition.RankingConstants;
@@ -14,6 +16,7 @@ import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.Search;
import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
import com.yahoo.vespa.model.AbstractService;
import java.util.Collection;
@@ -25,21 +28,21 @@ import java.util.logging.Logger;
*
* @author bratseth
*/
-public class RankProfileList extends Derived implements RankProfilesConfig.Producer,
- RankingConstantsConfig.Producer,
- OnnxModelsConfig.Producer {
+public class RankProfileList extends Derived implements RankProfilesConfig.Producer {
private static final Logger log = Logger.getLogger(RankProfileList.class.getName());
private final Map<String, RawRankProfile> rankProfiles = new java.util.LinkedHashMap<>();
private final RankingConstants rankingConstants;
+ private final RankExpressionFiles rankExpressionFiles;
private final OnnxModels onnxModels;
public static RankProfileList empty = new RankProfileList();
private RankProfileList() {
- this.rankingConstants = new RankingConstants();
- this.onnxModels = new OnnxModels();
+ rankingConstants = new RankingConstants();
+ rankExpressionFiles = new RankExpressionFiles();
+ onnxModels = new OnnxModels();
}
/**
@@ -50,6 +53,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
*/
public RankProfileList(Search search,
RankingConstants rankingConstants,
+ RankExpressionFiles rankExpressionFiles,
AttributeFields attributeFields,
RankProfileRegistry rankProfileRegistry,
QueryProfileRegistry queryProfiles,
@@ -57,7 +61,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
ModelContext.Properties deployProperties) {
setName(search == null ? "default" : search.getName());
this.rankingConstants = rankingConstants;
- this.onnxModels = search == null ? new OnnxModels() : search.onnxModels(); // as ONNX models come from parsing rank expressions
+ this.rankExpressionFiles = rankExpressionFiles;
+ onnxModels = search == null ? new OnnxModels() : search.onnxModels(); // as ONNX models come from parsing rank expressions
deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties);
}
@@ -93,11 +98,9 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
return rankProfiles.get(name);
}
- public void sendConstantsTo(Collection<? extends AbstractService> services) {
+ public void sendTo(Collection<? extends AbstractService> services) {
rankingConstants.sendTo(services);
- }
-
- public void sendOnnxModelsTo(Collection<? extends AbstractService> services) {
+ rankExpressionFiles.sendTo(services);
onnxModels.sendTo(services);
}
@@ -111,7 +114,10 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
}
}
- @Override
+ public void getConfig(RankingExpressionsConfig.Builder builder) {
+ rankExpressionFiles.asMap().values().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference())));
+ }
+
public void getConfig(RankingConstantsConfig.Builder builder) {
for (RankingConstant constant : rankingConstants.asMap().values()) {
if ("".equals(constant.getFileReference()))
@@ -124,7 +130,6 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
}
}
- @Override
public void getConfig(OnnxModelsConfig.Builder builder) {
for (OnnxModel model : onnxModels.asMap().values()) {
if ("".equals(model.getFileReference()))
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index c656a426b61..41ac1e17d93 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -125,8 +125,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
*/
private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>();
+ private final RankProfile rankProfile;
private RankingExpression firstPhaseRanking = null;
-
private RankingExpression secondPhaseRanking = null;
private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>();
@@ -159,6 +159,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
private final Map<String, String> attributeTypes;
private final Map<String, String> queryFeatureTypes;
+ private final boolean useExternalExpressionFiles;
private Set<String> filterFields = new java.util.LinkedHashSet<>();
@@ -166,11 +167,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
* Creates a raw rank profile from the given rank profile
*/
Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels,
- AttributeFields attributeFields, ModelContext.Properties deployProperties)
+ AttributeFields attributeFields, ModelContext.Properties deployProperties)
{
+ this.rankProfile = rankProfile;
RankProfile compiled = rankProfile.compile(queryProfiles, importedModels);
attributeTypes = compiled.getAttributeTypes();
queryFeatureTypes = compiled.getQueryFeatureTypes();
+ useExternalExpressionFiles = deployProperties.featureFlags().useExternalRankExpressions();
deriveRankingFeatures(compiled, deployProperties);
deriveRankTypeSetting(compiled, attributeFields);
deriveFilterFields(compiled);
@@ -230,13 +233,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
Map<String, String> functionProperties) {
SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties);
for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) {
+ if (useExternalExpressionFiles && rankProfile.getExpressionFile(e.getKey()) != null) continue;
String propertyName = RankingExpression.propertyName(e.getKey());
- if (context.serializedFunctions().containsKey(propertyName)) {
- continue;
- }
+ if (context.serializedFunctions().containsKey(propertyName)) continue;
+
String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString();
- context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString);
+ context.addFunctionSerialization(propertyName, expressionString);
for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet())
context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue());
if (e.getValue().function().returnType().isPresent())
@@ -333,7 +336,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
public List<Pair<String, String>> derive() {
List<Pair<String, String>> properties = new ArrayList<>();
for (RankProfile.RankProperty property : rankProperties) {
- if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) {
+ if (("rankingExpression(" + RankProfile.FIRST_PHASE + ").rankingScript").equals(property.getName())) {
// Could have been set by function expansion. Set expressions, then skip this property.
try {
firstPhaseRanking = new RankingExpression(property.getValue());
@@ -341,7 +344,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
throw new IllegalArgumentException("Could not parse first phase expression", e);
}
}
- else if ("rankingExpression(secondphase).rankingScript".equals(property.getName())) {
+ else if (("rankingExpression(" + RankProfile.SECOND_PHASE + ").rankingScript").equals(property.getName())) {
try {
secondPhaseRanking = new RankingExpression(property.getValue());
} catch (ParseException e) {
@@ -352,8 +355,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
properties.add(new Pair<>(property.getName(), property.getValue()));
}
}
- properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase"));
- properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase"));
+ properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, rankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE));
+ properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, rankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE));
for (FieldRankSettings settings : fieldRankSettings.values()) {
properties.addAll(settings.deriveRankProperties());
}
@@ -420,7 +423,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
return properties;
}
- private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) {
+ private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String fileName, String phase) {
List<Pair<String, String>> properties = new ArrayList<>();
if (expression == null) return properties;
@@ -428,7 +431,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
if ("".equals(name))
name = phase;
- if (expression.getRoot() instanceof ReferenceNode) {
+ if (useExternalExpressionFiles && (fileName != null)) {
+ properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + rankProfile.getUniqueExpressionName(name) + ")"));
+ } else if (expression.getRoot() instanceof ReferenceNode) {
properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString()));
} else {
properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
index 6e16b1a120e..8dea1b65079 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -92,21 +92,30 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
* Preferred constructor when building from XML. Use this if you are building
* in doBuild() in an AbstractConfigProducerBuilder.
* build() will call initService() in that case, after setting hostalias and baseport.
- * @param parent Parent config producer in the model tree.
- * @param name Name of this service.
+ *
+ * @param parent the parent config producer in the model tree
+ * @param name the name of this service
*/
- public AbstractService(AbstractConfigProducer parent, String name) {
+ public AbstractService(AbstractConfigProducer<?> parent, String name) {
super(parent, name);
}
/**
* Only used for testing. Stay away.
- * @param name Name of this service.
+ *
+ * @param name the name of this service.
*/
public AbstractService(String name) {
super(name);
}
+ @Override
+ public void remove() {
+ super.remove();
+ if (hostResource != null)
+ hostResource.deallocateService(this);
+ }
+
/**
* Distribute affinity on a collection of services. Services that are located on the same host
* will be assigned a specific cpu socket on that host.
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java
index a80982fe75b..b6a9d2bf97a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java
@@ -6,9 +6,12 @@ import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.provision.NetworkPorts;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
import java.util.Optional;
import java.util.logging.Level;
@@ -36,6 +39,7 @@ public class HostPorts {
private PortFinder portFinder = new PortFinder(Collections.emptyList());
+ private boolean flushed = false;
private Optional<NetworkPorts> networkPortsList = Optional.empty();
public HostPorts(String hostname) {
@@ -143,8 +147,16 @@ public class HostPorts {
return allocator.result();
}
+ void deallocatePorts(NetworkPortRequestor service) {
+ if (flushed)
+ throw new IllegalStateException("Cannot deallocate ports after calling flushPortReservations()");
+ portDB.entrySet().removeIf(entry -> entry.getValue().getServiceName().equals(service.getServiceName()));
+ allocatedPorts--;
+ }
+
public void flushPortReservations() {
this.networkPortsList = Optional.of(new NetworkPorts(portFinder.allocations()));
+ this.flushed = true;
}
/**
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
index 3bc07db9507..ef041d06978 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
@@ -79,6 +79,11 @@ public class HostResource implements Comparable<HostResource> {
return ports;
}
+ void deallocateService(AbstractService service) {
+ hostPorts.deallocatePorts(service);
+ services.remove(service.getServiceName());
+ }
+
/**
* Returns the service with the given "sentinel name" on this Host,
* or null if the name does not match any service.
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index 4dc38c09ab1..ab00e9d295f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -33,6 +33,7 @@ import com.yahoo.container.QrConfig;
import com.yahoo.path.Path;
import com.yahoo.searchdefinition.OnnxModel;
import com.yahoo.searchdefinition.OnnxModels;
+import com.yahoo.searchdefinition.RankExpressionFiles;
import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.RankingConstants;
@@ -129,6 +130,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
/** The global ranking constants of this model */
private final RankingConstants rankingConstants = new RankingConstants();
+ /** External rank expression files of this */
+ private final RankExpressionFiles rankExpressionFiles = new RankExpressionFiles();
+
/** The validation overrides of this. This is never null. */
private final ValidationOverrides validationOverrides;
@@ -181,15 +185,17 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
createGlobalRankProfiles(deployState.getDeployLogger(), deployState.getImportedModels(),
deployState.rankProfileRegistry(), deployState.getQueryProfiles());
- this.rankProfileList = new RankProfileList(null, // null search -> global
- rankingConstants, AttributeFields.empty,
- deployState.rankProfileRegistry(),
- deployState.getQueryProfiles().getRegistry(),
- deployState.getImportedModels(),
- deployState.getProperties());
+ rankProfileList = new RankProfileList(null, // null search -> global
+ rankingConstants,
+ rankExpressionFiles,
+ AttributeFields.empty,
+ deployState.rankProfileRegistry(),
+ deployState.getQueryProfiles().getRegistry(),
+ deployState.getImportedModels(),
+ deployState.getProperties());
HostSystem hostSystem = root.hostSystem();
- if (complete) { // create a a completed, frozen model
+ if (complete) { // create a completed, frozen model
configModelRepo.readConfigModels(deployState, this, builder, root, configModelRegistry);
addServiceClusters(deployState, builder);
setupRouting(deployState);
@@ -260,6 +266,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
/** Returns the global ranking constants of this */
public RankingConstants rankingConstants() { return rankingConstants; }
+ public RankExpressionFiles rankExpressionFiles() { return rankExpressionFiles; }
+
/** Creates a mutable model with no services instantiated */
public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException {
return new VespaModel(new NullConfigModelRegistry(), deployState, false,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index d1b9bbb4e58..e080ce43730 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -148,15 +148,23 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable
public void setClusterControllers(ClusterControllerContainerCluster clusterControllers, DeployLogger deployLogger) {
this.clusterControllers = clusterControllers;
- if (isHostedVespa)
+ if (isHostedVespa) {
+ // Prefer to put Slobroks on the admin cluster running cluster controllers to avoid unnecessary
+ // movement of the slobroks when there are changes to the content cluster nodes
+ removeSlobroks();
addSlobroks(createSlobroksOn(clusterControllers, deployLogger));
+ }
+ }
+
+ private void removeSlobroks() {
+ slobroks.forEach(Slobrok::remove);
+ slobroks.clear();
}
private List<Slobrok> createSlobroksOn(ClusterControllerContainerCluster clusterControllers, DeployLogger deployLogger) {
List<Slobrok> slobroks = new ArrayList<>();
- int index = this.slobroks.size();
for (ClusterControllerContainer clusterController : clusterControllers.getContainers()) {
- Slobrok slobrok = new Slobrok(this, index++);
+ Slobrok slobrok = new Slobrok(this, clusterController.index());
slobrok.setHostResource(clusterController.getHostResource());
slobroks.add(slobrok);
slobrok.initService(deployLogger);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java
index eed886b707f..00b6d872ddd 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java
@@ -9,9 +9,10 @@ import com.yahoo.vespa.model.PortAllocBridge;
/**
* Represents a Slobrok service.
*
- * @author gjoranv
+ * @author gjoranv
*/
public class Slobrok extends AbstractService implements StateserverConfig.Producer {
+
private static final long serialVersionUID = 1L;
public final static int BASEPORT = 19099;
@@ -25,7 +26,7 @@ public class Slobrok extends AbstractService implements StateserverConfig.Produc
* @param parent the parent ConfigProducer.
* @param index unique index for all slobroks
*/
- public Slobrok(AbstractConfigProducer parent, int index) {
+ public Slobrok(AbstractConfigProducer<?> parent, int index) {
super(parent, "slobrok." + index);
portsMeta.on(0).tag("rpc").tag("admin").tag("status");
portsMeta.on(1).tag("http").tag("state");
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
index 7049213ddab..d87c6596fa4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
@@ -8,6 +8,8 @@ import com.yahoo.log.InvalidLogFormatException;
import com.yahoo.log.LogMessage;
import com.yahoo.path.Path;
import com.yahoo.searchdefinition.OnnxModel;
+import com.yahoo.searchdefinition.RankExpressionFile;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.yolean.Exceptions;
import com.yahoo.system.ProcessExecuter;
@@ -38,7 +40,6 @@ import java.util.logging.Logger;
import java.util.logging.Level;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
/**
* Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done
@@ -116,50 +117,61 @@ public class RankSetupValidator extends Validator {
}
private void writeConfigs(String dir, AbstractConfigProducer<?> producer) throws IOException {
- RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder();
- ((RankProfilesConfig.Producer) producer).getConfig(rpcb);
- RankProfilesConfig rpc = new RankProfilesConfig(rpcb);
- writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc);
-
- IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder();
- ((IndexschemaConfig.Producer) producer).getConfig(iscb);
- IndexschemaConfig isc = new IndexschemaConfig(iscb);
- writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc);
-
- AttributesConfig.Builder acb = new AttributesConfig.Builder();
- ((AttributesConfig.Producer) producer).getConfig(acb);
- AttributesConfig ac = new AttributesConfig(acb);
- writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac);
-
- RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder();
- ((RankingConstantsConfig.Producer) producer).getConfig(rccb);
- RankingConstantsConfig rcc = new RankingConstantsConfig(rccb);
- writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc);
-
- OnnxModelsConfig.Builder omcb = new OnnxModelsConfig.Builder();
- ((OnnxModelsConfig.Producer) producer).getConfig(omcb);
- OnnxModelsConfig omc = new OnnxModelsConfig(omcb);
- writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omc);
-
- ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder();
- ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb);
- ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb);
- writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc);
+ RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder();
+ ((RankProfilesConfig.Producer) producer).getConfig(rpcb);
+ RankProfilesConfig rpc = new RankProfilesConfig(rpcb);
+ writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc);
+
+ IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder();
+ ((IndexschemaConfig.Producer) producer).getConfig(iscb);
+ IndexschemaConfig isc = new IndexschemaConfig(iscb);
+ writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc);
+
+ AttributesConfig.Builder acb = new AttributesConfig.Builder();
+ ((AttributesConfig.Producer) producer).getConfig(acb);
+ AttributesConfig ac = new AttributesConfig(acb);
+ writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac);
+
+ RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder();
+ ((RankingConstantsConfig.Producer) producer).getConfig(rccb);
+ RankingConstantsConfig rcc = new RankingConstantsConfig(rccb);
+ writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc);
+
+ RankingExpressionsConfig.Builder recb = new RankingExpressionsConfig.Builder();
+ ((RankingExpressionsConfig.Producer) producer).getConfig(recb);
+ RankingExpressionsConfig rec = new RankingExpressionsConfig(recb);
+ writeConfig(dir, RankingExpressionsConfig.getDefName() + ".cfg", rec);
+
+ OnnxModelsConfig.Builder omcb = new OnnxModelsConfig.Builder();
+ ((OnnxModelsConfig.Producer) producer).getConfig(omcb);
+ OnnxModelsConfig omc = new OnnxModelsConfig(omcb);
+ writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omc);
+
+ ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder();
+ ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb);
+ ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb);
+ writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc);
}
private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException {
String configName = "verify-ranksetup.cfg";
String configContent = "";
+ List<String> config = new ArrayList<>();
// Assist verify-ranksetup in finding the actual ONNX model files
- Map<String, OnnxModel> models = db.getDerivedConfiguration().getSearch().onnxModels().asMap();
- if (models.values().size() > 0) {
- List<String> config = new ArrayList<>(models.values().size() * 2);
- for (OnnxModel model : models.values()) {
- String modelPath = getFileRepositoryPath(model.getFilePath(), model.getFileReference());
- config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, model.getFileReference()));
- config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath));
- }
+ for (OnnxModel model : db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()) {
+ String modelPath = getFileRepositoryPath(model.getFilePath(), model.getFileReference());
+ config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, model.getFileReference()));
+ config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath));
+ }
+
+ for (RankExpressionFile expr : db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()) {
+ String modelPath = getFileRepositoryPath(expr.getFilePath(), expr.getFileReference());
+ config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, expr.getFileReference()));
+ config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath));
+ }
+
+ if ( ! config.isEmpty() ) {
configContent = StringUtilities.implodeMultiline(config);
}
IOUtils.writeFile(dir + configName, configContent, false);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
index a0673824907..bbec64d45fb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
@@ -67,7 +67,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
admin,
allocateHosts(admin.hostSystem(), "slobroks", nodesSpecification));
}
- else { // TODO: Remove
+ else { // These will be removed later, if an admin cluster (for cluster controllers) is assigned
createSlobroks(deployLogger,
admin,
pickContainerHostsForSlobrok(nodesSpecification.minResources().nodes(), 2));
@@ -153,8 +153,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
List<HostResource> picked = sortedContainerHostsFrom(model, count, !retired);
// if we can return multiple hosts, include retired nodes which would have been picked before
- // (probably - assuming all previous nodes were retired, which is always true for a single cluster
- // at the moment (Sept 2015)) to ensure a smoother transition between the old and new topology
+ // (probably - assuming all previous nodes were retired, which is always true for a single cluster,
+ // to ensure a smoother transition between the old and new topology
// by including both new and old nodes during the retirement period
picked.addAll(sortedContainerHostsFrom(model, count, retired));
@@ -166,6 +166,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
List<HostResource> hosts = model.getCluster().getContainers().stream()
.filter(container -> retired == container.isRetired())
.map(Container::getHostResource)
+ .sorted(HostResource::comparePrimarilyByIndexTo)
.collect(Collectors.toList());
return hosts.subList(0, Math.min(count, hosts.size()));
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java
index 3a4e8a70613..36bf792ee82 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java
@@ -43,7 +43,6 @@ public class DomFederationSearcherBuilder extends VespaDomBuilder.DomConfigProdu
return XML.getChild(searcherSpec, "source-set") != null;
}
-
private List<FederationSearcherModel.TargetSpec> readSources(Element searcherSpec) {
List<FederationSearcherModel.TargetSpec> sources = new ArrayList<>();
for (Element source : XML.getChildren(searcherSpec, "source")) {
@@ -76,14 +75,14 @@ public class DomFederationSearcherBuilder extends VespaDomBuilder.DomConfigProdu
}
@Override
- protected FederationSearcher doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element searcherElement) {
+ protected FederationSearcher doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element searcherElement) {
FederationSearcherModel model = new FederationSearcherModelBuilder(searcherElement).build();
Optional<Component> targetSelector = buildTargetSelector(deployState, ancestor, searcherElement, model.getComponentId());
return new FederationSearcher(model, targetSelector);
}
- private Optional<Component> buildTargetSelector(DeployState deployState, AbstractConfigProducer ancestor, Element searcherElement, ComponentId namespace) {
+ private Optional<Component> buildTargetSelector(DeployState deployState, AbstractConfigProducer<?> ancestor, Element searcherElement, ComponentId namespace) {
Element targetSelectorElement = XML.getChild(searcherElement, "target-selector");
if (targetSelectorElement == null)
return Optional.empty();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java
index 0106123666d..9fb19efbf75 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java
@@ -16,9 +16,10 @@ import java.util.List;
import java.util.Map;
/**
+ * Creates top level search chains(searchchain, provider) from xml.
+ *
* @author Tony Vaagenes
* @author gjoranv
- * Creates top level search chains(searchchain, provider) from xml.
*/
public class SearchChainsBuilder extends ChainsBuilder<Searcher<?>, SearchChain> {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index 4e78f44d0fe..317ed0f66c7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -25,6 +25,7 @@ import com.yahoo.search.config.QrStartConfig;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer;
import com.yahoo.vespa.model.container.component.BindingPattern;
import com.yahoo.vespa.model.container.component.Component;
@@ -58,6 +59,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
RankProfilesConfig.Producer,
RankingConstantsConfig.Producer,
OnnxModelsConfig.Producer,
+ RankingExpressionsConfig.Producer,
ServletPathsConfig.Producer,
ContainerMbusConfig.Producer,
MetricsProxyApiConfig.Producer,
@@ -233,6 +235,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
if (modelEvaluation != null) modelEvaluation.getConfig(builder);
}
+ public void getConfig(RankingExpressionsConfig.Builder builder) {
+ if (modelEvaluation != null) modelEvaluation.getConfig(builder);
+ }
+
@Override
public void getConfig(ContainerMbusConfig.Builder builder) {
if (mbusParams != null) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index 6ef29269bc1..cd596038137 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -79,7 +79,6 @@ public abstract class Container extends AbstractService implements
private final ComponentGroup<Component<?, ?>> components = new ComponentGroup<>(this, "components");
private final JettyHttpServer defaultHttpServer;
- private final boolean enableJdiscHttp2;
protected Container(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) {
this(parent, name, false, index, deployState);
@@ -100,8 +99,6 @@ public abstract class Container extends AbstractService implements
addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext"));
appendJvmOptions(jvmOmitStackTraceInFastThrowOption(deployState.featureFlags()));
-
- this.enableJdiscHttp2 = deployState.featureFlags().enableJdiscHttp2();
}
protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) {
@@ -183,7 +180,7 @@ public abstract class Container extends AbstractService implements
}
private void initDefaultJettyConnector() {
- defaultHttpServer.addConnector(new ConnectorFactory.Builder("SearchServer", getSearchPort()).enableHttp2(enableJdiscHttp2).build());
+ defaultHttpServer.addConnector(new ConnectorFactory.Builder("SearchServer", getSearchPort()).build());
}
private ContainerServiceType myServiceType = null;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
index 510d2fe3d99..37bfb8821c3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
@@ -7,6 +7,7 @@ import com.yahoo.searchdefinition.derived.RankProfileList;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
import com.yahoo.vespa.model.container.component.Handler;
import com.yahoo.vespa.model.container.component.SystemBindingPattern;
@@ -18,10 +19,12 @@ import java.util.Objects;
*
* @author bratseth
*/
-public class ContainerModelEvaluation implements RankProfilesConfig.Producer,
- RankingConstantsConfig.Producer,
- OnnxModelsConfig.Producer
-{
+
+public class ContainerModelEvaluation implements
+ RankProfilesConfig.Producer,
+ RankingConstantsConfig.Producer,
+ OnnxModelsConfig.Producer,
+ RankingExpressionsConfig.Producer {
private final static String BUNDLE_NAME = "model-evaluation";
private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName();
@@ -38,8 +41,7 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer,
}
public void prepare(List<ApplicationContainer> containers) {
- rankProfileList.sendConstantsTo(containers);
- rankProfileList.sendOnnxModelsTo(containers);
+ rankProfileList.sendTo(containers);
}
@Override
@@ -57,6 +59,10 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer,
rankProfileList.getConfig(builder);
}
+ public void getConfig(RankingExpressionsConfig.Builder builder) {
+ rankProfileList.getConfig(builder);
+ }
+
public static Handler<?> getHandler() {
Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME));
handler.addServerBindings(
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java
index 9b0075c79c3..0d024398de7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java
@@ -20,7 +20,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig
private final String name;
private final int listenPort;
private final SslProvider sslProviderComponent;
- private final boolean enableHttp2;
private volatile ComponentId defaultRequestFilterChain;
private volatile ComponentId defaultResponseFilterChain;
@@ -33,7 +32,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig
this.sslProviderComponent = builder.sslProvider != null ? builder.sslProvider : new DefaultSslProvider(name);
this.defaultRequestFilterChain = builder.defaultRequestFilterChain;
this.defaultResponseFilterChain = builder.defaultResponseFilterChain;
- this.enableHttp2 = builder.enableHttp2 != null ? builder.enableHttp2 : false;
addChild(sslProviderComponent);
inject(sslProviderComponent);
}
@@ -42,7 +40,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig
public void getConfig(ConnectorConfig.Builder connectorBuilder) {
connectorBuilder.listenPort(listenPort);
connectorBuilder.name(name);
- connectorBuilder.http2Enabled(enableHttp2);
sslProviderComponent.amendConnectorConfig(connectorBuilder);
}
@@ -69,7 +66,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig
private SslProvider sslProvider;
private ComponentId defaultRequestFilterChain;
private ComponentId defaultResponseFilterChain;
- private Boolean enableHttp2;
public Builder(String name, int listenPort) {
this.name = name;
@@ -88,8 +84,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig
this.defaultResponseFilterChain = filterChain; return this;
}
- public Builder enableHttp2(boolean enabled) { this.enableHttp2 = enabled; return this; }
-
public ConnectorFactory build() {
return new ConnectorFactory(this);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
index 9d715073f77..766aa46fc01 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
@@ -29,32 +29,32 @@ public class HostedSslConnectorFactory extends ConnectorFactory {
* Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore.
*/
public static HostedSslConnectorFactory withProvidedCertificate(
- String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth, boolean enableHttp2) {
+ String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth) {
ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider(
serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth);
- return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, enableHttp2);
+ return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth);
}
/**
* Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application.
*/
public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(
- String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates, boolean enableHttp2) {
+ String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) {
ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider(
serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false);
- return new HostedSslConnectorFactory(sslProvider, true, false, enableHttp2);
+ return new HostedSslConnectorFactory(sslProvider, true, false);
}
/**
* Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration).
*/
- public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, boolean enableHttp2) {
- return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, enableHttp2);
+ public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName) {
+ return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false);
}
private HostedSslConnectorFactory(SslProvider sslProvider, boolean enforceClientAuth,
- boolean enforceHandshakeClientAuth, boolean enableHttp2) {
- super(new Builder("tls4443", 4443).sslProvider(sslProvider).enableHttp2(enableHttp2));
+ boolean enforceHandshakeClientAuth) {
+ super(new Builder("tls4443", 4443).sslProvider(sslProvider));
this.enforceClientAuth = enforceClientAuth;
this.enforceHandshakeClientAuth = enforceHandshakeClientAuth;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
index 3b616c34a03..22fa5332d5e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
@@ -39,7 +39,7 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil
.map(ComponentId::new)
.ifPresent(builder::defaultResponseFilterChain);
SslProvider sslProviderComponent = getSslConfigComponents(name, serverSpec);
- return builder.sslProvider(sslProviderComponent).enableHttp2(deployState.featureFlags().enableJdiscHttp2()).build();
+ return builder.sslProvider(sslProviderComponent).build();
}
SslProvider getSslConfigComponents(String serverName, Element serverSpec) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java
index ceb48732116..6b4cb003cda 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java
@@ -11,7 +11,12 @@ import com.yahoo.search.federation.FederationConfig;
import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec;
import com.yahoo.vespa.model.container.component.Component;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
/**
* Config producer for the FederationSearcher.
@@ -26,8 +31,8 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem
* Generates config for a single search chain contained in a target.
*/
private static final class SearchChainConfig {
+
private final SearchChain searchChain;
- //Zero if not applicable
final ComponentId providerId;
final FederationOptions targetOptions;
final List<String> documentTypes;
@@ -61,6 +66,7 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem
* which can be federated to as a single entity.
*/
private static abstract class Target {
+
final ComponentId id;
final FederationOptions targetOptions;
@@ -79,41 +85,36 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem
}
protected abstract void getSearchChainsConfig(FederationConfig.Target.Builder tb);
+
}
private static class SearchChainTarget extends Target {
+
private final SearchChainConfig searchChainConfig;
- public SearchChainTarget(SearchChain searchChain,
- FederationOptions targetOptions) {
+ public SearchChainTarget(SearchChain searchChain, FederationOptions targetOptions) {
super(searchChain.getComponentId(), targetOptions);
- searchChainConfig = new SearchChainConfig(
- searchChain,
- null,
- targetOptions,
- searchChain.getDocumentTypes());
+ searchChainConfig = new SearchChainConfig(searchChain, null, targetOptions, searchChain.getDocumentTypes());
}
@Override
protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) {
tB.searchChain(searchChainConfig.getSearchChainConfig());
}
+
}
private static class SourceGroupTarget extends Target {
+
private final SearchChainConfig leaderConfig;
- private final List<SearchChainConfig> participantsConfig =
- new ArrayList<>();
+ private final List<SearchChainConfig> participantsConfig = new ArrayList<>();
- public SourceGroupTarget(SourceGroup group,
- FederationOptions targetOptions) {
+ public SourceGroupTarget(SourceGroup group, FederationOptions targetOptions) {
super(group.getComponentId(), applyDefaultSourceGroupOptions(targetOptions));
leaderConfig = createConfig(group.leader(), targetOptions);
- for (Source participant : group.participants()) {
- participantsConfig.add(
- createConfig(participant, targetOptions));
- }
+ for (Source participant : group.participants())
+ participantsConfig.add(createConfig(participant, targetOptions));
}
private static FederationOptions applyDefaultSourceGroupOptions(FederationOptions targetOptions) {
@@ -121,64 +122,49 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem
return targetOptions.inherit(defaultSourceGroupOption);
}
- private SearchChainConfig createConfig(Source source,
- FederationOptions targetOptions) {
- return new SearchChainConfig(
- source,
- source.getParentProvider().getComponentId(),
- targetOptions,
- source.getDocumentTypes());
+ private SearchChainConfig createConfig(Source source, FederationOptions targetOptions) {
+ return new SearchChainConfig(source,
+ source.getParentProvider().getComponentId(),
+ targetOptions,
+ source.getDocumentTypes());
}
@Override
protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) {
tB.searchChain(leaderConfig.getSearchChainConfig());
- for (SearchChainConfig participant : participantsConfig) {
+ for (SearchChainConfig participant : participantsConfig)
tB.searchChain(participant.getSearchChainConfig());
- }
}
}
private static class TargetResolver {
+
final ComponentRegistry<SearchChain> searchChainRegistry;
final SourceGroupRegistry sourceGroupRegistry;
- /**
- * @return true if searchChain.id newer than sourceGroup.id
- */
- private boolean newerVersion(SearchChain searchChain,
- SourceGroup sourceGroup) {
- if (searchChain == null || sourceGroup == null) {
- return false;
- } else {
- return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId());
- }
+ /** Returns true if searchChain.id newer than sourceGroup.id */
+ private boolean newerVersion(SearchChain searchChain, SourceGroup sourceGroup) {
+ if (searchChain == null || sourceGroup == null) return false;
+ return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId());
}
- /**
- * @return true if a newer than b
- */
+ /** Returns true if a newer than b */
private boolean newerVersion(ComponentId a, ComponentId b) {
return a.compareTo(b) > 0;
}
-
- TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry,
- SourceGroupRegistry sourceGroupRegistry) {
+ TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, SourceGroupRegistry sourceGroupRegistry) {
this.searchChainRegistry = searchChainRegistry;
this.sourceGroupRegistry = sourceGroupRegistry;
}
Target resolve(FederationSearcherModel.TargetSpec specification) {
- SearchChain searchChain = searchChainRegistry.getComponent(
- specification.sourceSpec);
- SourceGroup sourceGroup = sourceGroupRegistry.getComponent(
- specification.sourceSpec);
+ SearchChain searchChain = searchChainRegistry.getComponent(specification.sourceSpec);
+ SourceGroup sourceGroup = sourceGroupRegistry.getComponent(specification.sourceSpec);
if (searchChain == null && sourceGroup == null) {
return null;
- } else if (sourceGroup == null ||
- newerVersion(searchChain, sourceGroup)) {
+ } else if (sourceGroup == null || newerVersion(searchChain, sourceGroup)) {
return new SearchChainTarget(searchChain, specification.federationOptions);
} else {
return new SourceGroupTarget(sourceGroup, specification.federationOptions);
@@ -186,26 +172,21 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem
}
}
- private final Map<ComponentId, Target> resolvedTargets =
- new LinkedHashMap<>();
+ private final Map<ComponentId, Target> resolvedTargets = new LinkedHashMap<>();
public FederationSearcher(FederationSearcherModel searcherModel, Optional<Component> targetSelector) {
super(searcherModel);
this.targetSelector = targetSelector;
- if (targetSelector.isPresent())
- addChild(targetSelector.get());
+ targetSelector.ifPresent(selector -> addChild(selector));
}
@Override
public void getConfig(FederationConfig.Builder builder) {
- for (Target target : resolvedTargets.values()) {
+ for (Target target : resolvedTargets.values())
builder.target(target.getTargetConfig());
- }
- if (targetSelector.isPresent()) {
- builder.targetSelector(targetSelector.get().getGlobalComponentId().stringValue());
- }
+ targetSelector.ifPresent(selector -> builder.targetSelector(selector.getGlobalComponentId().stringValue()));
}
@Override
@@ -213,10 +194,8 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem
initialize(getSearchChains().allChains(), getSearchChains().allSourceGroups());
}
- void initialize(ComponentRegistry<SearchChain> searchChainRegistry,
- SourceGroupRegistry sourceGroupRegistry) {
- TargetResolver targetResolver = new TargetResolver(
- searchChainRegistry, sourceGroupRegistry);
+ void initialize(ComponentRegistry<SearchChain> searchChainRegistry, SourceGroupRegistry sourceGroupRegistry) {
+ TargetResolver targetResolver = new TargetResolver(searchChainRegistry, sourceGroupRegistry);
addSourceTargets(targetResolver, model.targets);
@@ -229,16 +208,14 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem
Target target = targetResolver.resolve(targetSpec);
if (target == null) {
- throw new RuntimeException("Can't find source " +
- targetSpec.sourceSpec +
- " used as a source for federation '" +
- getComponentId() + "'");
+ throw new RuntimeException("Can't find source " + targetSpec.sourceSpec +
+ " used as a source for federation '" + getComponentId() + "'");
}
Target duplicate = resolvedTargets.put(target.id, target);
if (duplicate != null && !duplicate.targetOptions.equals(target.targetOptions)) {
- throw new RuntimeException("Search chain " + target.id + " added twice with different federation options"
- + " to the federation searcher " + getComponentId());
+ throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" +
+ " to the federation searcher " + getComponentId());
}
}
}
@@ -248,23 +225,21 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem
for (GenericTarget genericTarget : defaultTargets(searchChainRegistry.allComponents())) {
ComponentSpecification specification = genericTarget.getComponentId().toSpecification();
- //Can't use genericTarget directly, as it might be part of a source group.
+ // Can't use genericTarget directly, as it might be part of a source group.
Target federationTarget = targetResolver.resolve(new TargetSpec(specification, new FederationOptions()));
- //Do not replace manually added sources, as they might have manually configured federation options
+ // Do not replace manually added sources, as they might have manually configured federation options
if (!resolvedTargets.containsKey(federationTarget.id))
resolvedTargets.put(federationTarget.id, federationTarget);
}
}
-
private static List<GenericTarget> defaultTargets(Collection<SearchChain> allSearchChains) {
- Collection<Provider> providers =
- CollectionUtil.filter(allSearchChains, Provider.class);
+ Collection<Provider> providers = CollectionUtil.filter(allSearchChains, Provider.class);
List<GenericTarget> defaultTargets = new ArrayList<>();
- for (Provider provider : providers) {
+ for (Provider provider : providers)
defaultTargets.addAll(provider.defaultFederationTargets());
- }
return defaultTargets;
}
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java
index ee4edf3fd8c..10e0f3e55da 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java
@@ -7,6 +7,7 @@ import com.yahoo.vespa.model.container.component.ConfigProducerGroup;
import java.util.Arrays;
import java.util.Collection;
+import java.util.List;
/**
* Base config producer for search chains that communicate with backends.
@@ -15,7 +16,7 @@ import java.util.Collection;
*/
public class Provider extends GenericTarget {
- private ConfigProducerGroup<Source> sources;
+ private final ConfigProducerGroup<Source> sources;
public Provider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) {
super(specWithoutInnerSearchers, federationOptions);
@@ -37,9 +38,10 @@ public class Provider extends GenericTarget {
public Collection<? extends GenericTarget> defaultFederationTargets() {
if (sources.getComponents().isEmpty()) {
- return Arrays.asList(this);
+ return List.of(this);
} else {
return sources.getComponents();
}
}
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 87406b8bc9a..b477587bcac 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -431,7 +431,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
// If the deployment contains certificate/private key reference, setup TLS port
HostedSslConnectorFactory connectorFactory;
- boolean enableHttp2 = deployState.featureFlags().enableJdiscHttp2();
if (deployState.endpointCertificateSecrets().isPresent()) {
boolean authorizeClient = deployState.zone().system().isPublic();
if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) {
@@ -445,10 +444,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
.orElse(false);
connectorFactory = authorizeClient
- ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get(), enableHttp2)
- : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, enableHttp2);
+ ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get())
+ : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets, enforceHandshakeClientAuth);
} else {
- connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, enableHttp2);
+ connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName);
}
cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory));
server.addConnector(connectorFactory);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java
index c7d56c9b4b5..70f2acd3c7b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java
@@ -1,6 +1,7 @@
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content;
+import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder;
@@ -35,28 +36,31 @@ public class ClusterResourceLimits {
public static class Builder {
private final boolean enableFeedBlockInDistributor;
+ private final boolean hostedVespa;
+ private final DeployLogger deployLogger;
private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder();
private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder();
- public Builder(boolean enableFeedBlockInDistributor) {
+ public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, DeployLogger deployLogger) {
this.enableFeedBlockInDistributor = enableFeedBlockInDistributor;
+ this.hostedVespa = hostedVespa;
+ this.deployLogger = deployLogger;
}
public ClusterResourceLimits build(ModelElement clusterElem) {
- ModelElement tuningElem = clusterElem.childByPath("tuning");
- if (tuningElem != null) {
- ctrlBuilder = DomResourceLimitsBuilder.createBuilder(tuningElem);
- }
-
- ModelElement protonElem = clusterElem.childByPath("engine.proton");
- if (protonElem != null) {
- nodeBuilder = DomResourceLimitsBuilder.createBuilder(protonElem);
- }
+ ctrlBuilder = createBuilder(clusterElem.childByPath("tuning"));
+ nodeBuilder = createBuilder(clusterElem.childByPath("engine.proton"));
deriveLimits();
return new ClusterResourceLimits(this);
}
+ private ResourceLimits.Builder createBuilder(ModelElement element) {
+ return element == null
+ ? new ResourceLimits.Builder()
+ : DomResourceLimitsBuilder.createBuilder(element, hostedVespa, deployLogger);
+ }
+
public void setClusterControllerBuilder(ResourceLimits.Builder builder) {
ctrlBuilder = builder;
}
@@ -88,7 +92,7 @@ public class ClusterResourceLimits {
Optional<Double> contentNodeLimit,
Consumer<Double> setter) {
// TODO: remove this when feed block in distributor is default enabled.
- if (!clusterControllerLimit.isPresent() && !contentNodeLimit.isPresent()) {
+ if (clusterControllerLimit.isEmpty() && contentNodeLimit.isEmpty()) {
setter.accept(0.8);
}
}
@@ -96,7 +100,7 @@ public class ClusterResourceLimits {
private void deriveClusterControllerLimit(Optional<Double> clusterControllerLimit,
Optional<Double> contentNodeLimit,
Consumer<Double> setter) {
- if (!clusterControllerLimit.isPresent()) {
+ if (clusterControllerLimit.isEmpty()) {
contentNodeLimit.ifPresent(limit ->
// TODO: emit warning when feed block in distributor is default enabled.
setter.accept(Double.max(0.0, (limit - 0.01))));
@@ -106,7 +110,7 @@ public class ClusterResourceLimits {
private void deriveContentNodeLimit(Optional<Double> contentNodeLimit,
Optional<Double> clusterControllerLimit,
Consumer<Double> setter) {
- if (!contentNodeLimit.isPresent()) {
+ if (contentNodeLimit.isEmpty()) {
clusterControllerLimit.ifPresent(limit ->
setter.accept(calcContentNodeLimit(limit)));
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index f6a45842bd9..b5cae857ce0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -25,7 +25,6 @@ import com.yahoo.vespa.config.content.core.BucketspacesConfig;
import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
import com.yahoo.vespa.model.HostResource;
import com.yahoo.vespa.model.admin.Admin;
-import com.yahoo.vespa.model.admin.Configserver;
import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster;
import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerComponent;
import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerConfigurer;
@@ -134,7 +133,10 @@ public class ContentCluster extends AbstractConfigProducer implements
globallyDistributedDocuments, routingSelection,
deployState.zone(), deployState.isHosted());
boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor();
- var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor).build(contentElement);
+ var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor,
+ stateIsHosted(deployState),
+ deployState.getDeployLogger())
+ .build(contentElement);
c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement),
contentElement,
resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml());
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java
index 210f062f9b2..9f4852629d0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java
@@ -1,9 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content.cluster;
+import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import com.yahoo.vespa.model.content.ResourceLimits;
+import java.util.logging.Level;
+
/**
* Builder for feed block resource limits.
*
@@ -11,10 +14,16 @@ import com.yahoo.vespa.model.content.ResourceLimits;
*/
public class DomResourceLimitsBuilder {
- public static ResourceLimits.Builder createBuilder(ModelElement contentXml) {
+ public static ResourceLimits.Builder createBuilder(ModelElement contentXml, boolean hostedVespa, DeployLogger deployLogger) {
ResourceLimits.Builder builder = new ResourceLimits.Builder();
ModelElement resourceLimits = contentXml.child("resource-limits");
- if (resourceLimits == null) {
+ if (resourceLimits == null) { return builder; }
+
+ if (hostedVespa) {
+ deployLogger.logApplicationPackage(Level.WARNING, "Element " + resourceLimits +
+ " is not allowed, default limits will be used");
+ // TODO: Throw exception when we are sure nobody is using this
+ //throw new IllegalArgumentException("Element " + element + " is not allowed to be set, default limits will be used");
return builder;
}
if (resourceLimits.child("disk") != null) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
index fb9402614f6..5bb57f4ff6c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -8,7 +8,12 @@ import com.yahoo.config.application.api.FileRegistry;
import com.yahoo.vespa.model.ConfigProxy;
import com.yahoo.vespa.model.Host;
-import java.util.*;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
/**
* Sends RPC requests to hosts (tenant hosts and config servers) asking them to start download of files. This is used
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java
index 15e6c5993b3..fbba377c950 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java
@@ -111,6 +111,8 @@ public final class DocumentProtocol implements Protocol,
addSelector(cluster.getConfigId(), cluster.getRoutingSelector(), clusterBuilder);
if (cluster.getSearch().hasIndexedCluster())
addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder);
+ else
+ clusterBuilder.defaultRoute(cluster.getConfigId());
builder.cluster(cluster.getConfigId(), clusterBuilder);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
index 705c48d1c9b..3ca5b9e3a0a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
@@ -39,8 +39,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer<Abstr
public void prepareToDistributeFiles(List<SearchNode> backends) {
for (SchemaSpec sds : localSDS) {
- sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends);
- sds.getSearchDefinition().getSearch().onnxModels().sendTo(backends);
+ sds.getSearchDefinition().getSearch().sendTo(backends);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java
index 8d4c1675bbf..9f0ff7db9f8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.config.search.SummaryConfig;
import com.yahoo.vespa.config.search.SummarymapConfig;
import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
import com.yahoo.vespa.configdefinition.IlscriptsConfig;
@@ -26,6 +27,7 @@ public class DocumentDatabase extends AbstractConfigProducer<DocumentDatabase> i
AttributesConfig.Producer,
RankProfilesConfig.Producer,
RankingConstantsConfig.Producer,
+ RankingExpressionsConfig.Producer,
OnnxModelsConfig.Producer,
IndexschemaConfig.Producer,
JuniperrcConfig.Producer,
@@ -75,6 +77,11 @@ public class DocumentDatabase extends AbstractConfigProducer<DocumentDatabase> i
}
@Override
+ public void getConfig(RankingExpressionsConfig.Builder builder) {
+ derivedCfg.getRankProfileList().getConfig(builder);
+ }
+
+ @Override
public void getConfig(RankingConstantsConfig.Builder builder) {
derivedCfg.getRankProfileList().getConfig(builder);
}
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 8f99c388fb1..0cc1db5a2d7 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -79,16 +79,6 @@ public class SDParser {
private RankProfileRegistry rankProfileRegistry;
private boolean documentsOnly;
- /** For testing only */
- public SDParser(String input, DeployLogger deployLogger) {
- this(new SimpleCharStream(input), deployLogger);
- }
-
- /** For testing only */
- public SDParser(SimpleCharStream stream, DeployLogger deployLogger) {
- this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry(), false);
- }
-
/**
* Creates a parser
*
@@ -435,7 +425,7 @@ Search rootSchema(String dir) :
Search search;
}
{
- ( ( <SCHEMA> | <SEARCH> ) name = identifier() { search = new Search(name, app);
+ ( ( <SCHEMA> | <SEARCH> ) name = identifier() { search = new Search(name, app, deployLogger);
rankProfileRegistry.add(new DefaultRankProfile(search, rankProfileRegistry));
rankProfileRegistry.add(new UnrankedRankProfile(search, rankProfileRegistry));}
lbrace() (rootSchemaItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>)
@@ -476,7 +466,7 @@ Object rootSchemaItem(Search search) : { }
*/
Search rootDocument(String dir) :
{
- Search search = new DocumentOnlySearch();
+ Search search = new DocumentOnlySearch(app, deployLogger);
}
{
( (rootDocumentItem(search) (<NL>)*)*<EOF> )
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 326bf577acc..2b72420614d 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -494,9 +494,8 @@ public class ModelProvisioningTest {
Set<HostResource> clusterControllerHosts = admin.getClusterControllers().getContainers()
.stream().map(cc -> cc.getHostResource()).collect(Collectors.toSet());
Set<HostResource> slobrokHosts = admin.getSlobroks().stream().map(Slobrok::getHost).collect(Collectors.toSet());
- assertEquals(6, slobrokHosts.size());
- assertTrue("Slobroks are assigned from container and cluster controller nodes",
- union(containerHosts, clusterControllerHosts).containsAll(slobrokHosts));
+ assertEquals(3, slobrokHosts.size());
+ assertTrue("Slobroks are assigned on cluster controller nodes", clusterControllerHosts.containsAll(slobrokHosts));
assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost()));
assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size());
assertEquals("Dedicated admin cluster controllers when hosted", 3, admin.getClusterControllers().getContainers().size());
@@ -560,6 +559,40 @@ public class ModelProvisioningTest {
}
@Test
+ public void testSlobroksOnContainersIfNoContentClusters() {
+ String services =
+ "<?xml version='1.0' encoding='utf-8' ?>\n" +
+ "<services>" +
+ " <admin version='4.0'/>" +
+ " <container version='1.0' id='foo'>" +
+ " <nodes count='10'/>" +
+ " </container>" +
+ "</services>";
+
+ int numberOfHosts = 10;
+ VespaModelTester tester = new VespaModelTester();
+ tester.addHosts(numberOfHosts);
+ VespaModel model = tester.createModel(services, true);
+ assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size());
+
+ // Check container cluster
+ assertEquals(1, model.getContainerClusters().size());
+ Set<HostResource> containerHosts = model.getContainerClusters().get("foo").getContainers().stream()
+ .map(Container::getHost)
+ .collect(Collectors.toSet());
+ assertEquals(10, containerHosts.size());
+
+ // Check admin clusters
+ Admin admin = model.getAdmin();
+ Set<HostResource> slobrokHosts = admin.getSlobroks().stream().map(Slobrok::getHost).collect(Collectors.toSet());
+ assertEquals(3, slobrokHosts.size());
+ assertTrue("Slobroks are assigned from container nodes",
+ containerHosts.containsAll(slobrokHosts));
+ assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost()));
+ assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size());
+ }
+
+ @Test
public void testUsingNodesAndGroupCountAttributesWithoutDedicatedClusterControllers() {
String services =
"<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -602,9 +635,8 @@ public class ModelProvisioningTest {
Set<HostResource> clusterControllerHosts = admin.getClusterControllers().getContainers()
.stream().map(cc -> cc.getHostResource()).collect(Collectors.toSet());
Set<HostResource> slobrokHosts = admin.getSlobroks().stream().map(Slobrok::getHost).collect(Collectors.toSet());
- assertEquals(6, slobrokHosts.size());
- assertTrue("Slobroks are assigned from container and cluster controller nodes",
- union(containerHosts, clusterControllerHosts).containsAll(slobrokHosts));
+ assertEquals(3, slobrokHosts.size());
+ assertTrue("Slobroks are assigned on cluster controller nodes", clusterControllerHosts.containsAll(slobrokHosts));
assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost()));
assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size());
assertEquals(3, admin.getClusterControllers().getContainers().size());
@@ -1613,7 +1645,7 @@ public class ModelProvisioningTest {
tester.addHosts(6);
VespaModel model = tester.createModel(services, true);
assertEquals(6, model.getRoot().hostSystem().getHosts().size());
- assertEquals(5, model.getAdmin().getSlobroks().size());
+ assertEquals(3, model.getAdmin().getSlobroks().size());
assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes(true));
}
@@ -2056,9 +2088,6 @@ public class ModelProvisioningTest {
" </nodes>",
" <engine>",
" <proton>",
- " <resource-limits>",
- " <memory>0.92</memory>",
- " </resource-limits>",
" <tuning>",
" <searchnode>",
" <flushstrategy>",
@@ -2084,7 +2113,6 @@ public class ModelProvisioningTest {
assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override
assertEquals(1000, cfg.flush().memory().maxmemory()); // from explicit tuning
assertEquals((long) ((128 - reservedMemoryGb) * GB / 8), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning
- assertEquals(0.92, cfg.writefilter().memorylimit(), 0.0001); // from explicit resource-limits
}
private static ProtonConfig getProtonConfig(VespaModel model, String configId) {
@@ -2138,11 +2166,4 @@ public class ModelProvisioningTest {
assertProvisioned(nodeCount, id, null, type, model);
}
- private Set<HostResource> union(Set<HostResource> a, Set<HostResource> b) {
- Set<HostResource> union = new HashSet<>();
- union.addAll(a);
- union.addAll(b);
- return union;
- }
-
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
index c2dd3e45dea..f475fbe6d78 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java
@@ -135,7 +135,7 @@ public class DocumentGraphValidatorTest {
}
private static Search createSearchWithName(String name, Search... parents) {
- Search campaignSearch = new Search(name, null);
+ Search campaignSearch = new Search(name);
SDDocumentType document = new SDDocumentType(name);
campaignSearch.addDocument(document);
document.setDocumentReferences(new DocumentReferences(Collections.emptyMap()));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
index e46208c770d..6b65303fe56 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
@@ -23,14 +23,16 @@ import static org.junit.Assert.assertTrue;
*/
public class DocumentReferenceResolverTest {
+ private static final String BAR = "bar";
+ private static final String FOO = "foo";
@Rule
public final ExpectedException exceptionRule = ExpectedException.none();
@Test
public void reference_from_one_document_to_another_is_resolved() {
// Create bar document with no fields
- Search barSearch = new Search();
- SDDocumentType barDocument = new SDDocumentType("bar", barSearch);
+ Search barSearch = new Search(BAR);
+ SDDocumentType barDocument = new SDDocumentType(BAR, barSearch);
barSearch.addDocument(barDocument);
// Create foo document with document reference to bar and add another field
@@ -38,7 +40,7 @@ public class DocumentReferenceResolverTest {
("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
AttributeUtils.addAttributeAspect(fooRefToBarField);
SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT);
- Search fooSearch = new Search();
+ Search fooSearch = new Search(FOO);
SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
fooDocument.addField(fooRefToBarField);
fooDocument.addField(irrelevantField);
@@ -60,7 +62,7 @@ public class DocumentReferenceResolverTest {
SDField fooRefToBarField = new SDField(
"bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar")));
AttributeUtils.addAttributeAspect(fooRefToBarField);
- Search fooSearch = new Search();
+ Search fooSearch = new Search(FOO);
SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
fooDocument.addField(fooRefToBarField);
fooSearch.addDocument(fooDocument);
@@ -76,14 +78,14 @@ public class DocumentReferenceResolverTest {
@Test
public void throws_exception_if_reference_is_not_an_attribute() {
// Create bar document with no fields
- Search barSearch = new Search();
+ Search barSearch = new Search(BAR);
SDDocumentType barDocument = new SDDocumentType("bar", barSearch);
barSearch.addDocument(barDocument);
// Create foo document with document reference to bar
SDField fooRefToBarField = new SDField
("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
- Search fooSearch = new Search();
+ Search fooSearch = new Search(FOO);
SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
fooDocument.addField(fooRefToBarField);
fooSearch.addDocument(fooDocument);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
index fcbb89b5c42..f12129155b1 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
@@ -18,30 +18,33 @@ public class ImportedFieldsEnumeratorTest {
@Test
public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() {
- Search parentSearch = new Search();
- SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch);
+ String PARENT = "parent";
+ Search parentSearch = new Search(PARENT);
+ SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSearch);
var parentField = new SDField("their_field", DataType.INT);
AttributeUtils.addAttributeAspect(parentField);
parentDocument.addField(parentField);
parentSearch.addDocument(parentDocument);
- Search fooSearch = new Search();
+ String FOO = "foo";
+ Search fooSearch = new Search(FOO);
SDField fooRefToParent = new SDField(
"foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
AttributeUtils.addAttributeAspect(fooRefToParent);
var fooImports = fooSearch.temporaryImportedFields().get();
fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field"));
fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field"));
- SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
+ SDDocumentType fooDocument = new SDDocumentType(FOO, fooSearch);
fooSearch.addDocument(fooDocument);
- Search barSearch = new Search();
+ String BAR = "bar";
+ Search barSearch = new Search(BAR);
SDField barRefToParent = new SDField(
"bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
AttributeUtils.addAttributeAspect(barRefToParent);
var barImports = barSearch.temporaryImportedFields().get();
barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field"));
- SDDocumentType barDocument = new SDDocumentType("bar", barSearch);
+ SDDocumentType barDocument = new SDDocumentType(BAR, barSearch);
barSearch.addDocument(barDocument);
var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java
index 02ec597c3ed..84819439048 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java
@@ -33,7 +33,7 @@ public class RankProfileRegistryTest {
@Test(expected = IllegalArgumentException.class)
public void testRankProfileDuplicateNameIsIllegal() {
- Search search = new Search("foo", null);
+ Search search = new Search("foo");
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
RankProfile barRankProfile = new RankProfile("bar", search, rankProfileRegistry);
rankProfileRegistry.add(barRankProfile);
@@ -42,7 +42,7 @@ public class RankProfileRegistryTest {
@Test
public void testRankProfileDuplicateNameLegalForOverridableRankProfiles() {
- Search search = new Search("foo", null);
+ Search search = new Search("foo");
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
index 0f83a4db98f..91e8640308a 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
@@ -38,7 +38,7 @@ public class RankProfileTestCase extends SchemaTestCase {
@Test
public void testRankProfileInheritance() {
- Search search = new Search("test", null);
+ Search search = new Search("test");
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
SDDocumentType document = new SDDocumentType("test");
SDField a = document.addField("a", DataType.STRING);
@@ -173,7 +173,7 @@ public class RankProfileTestCase extends SchemaTestCase {
" rank-profile p1 {}\n" +
" rank-profile p2 {}\n" +
"}");
- builder.build(true, new BaseDeployLogger());
+ builder.build(true);
Search search = builder.getSearch();
assertEquals(4, registry.all().size());
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
index 47862a2611b..139dae65c63 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
@@ -21,7 +21,7 @@ public class EmptyRankProfileTestCase extends SchemaTestCase {
@Test
public void testDeriving() {
- Search search = new Search("test", null);
+ Search search = new Search("test");
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
SDDocumentType doc = new SDDocumentType("test");
search.addDocument(doc);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index 770bea55af2..370315ad08b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;
+import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.searchdefinition.SearchBuilder;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -103,7 +104,8 @@ public class ExportingTestCase extends AbstractExportingTestCase {
@Test
public void testRankExpression() throws IOException, ParseException {
- assertCorrectDeriving("rankexpression");
+ assertCorrectDeriving("rankexpression", null,
+ new TestProperties().useExternalRankExpression(true), new TestableDeployLogger());
}
@Test
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
index 130450e483d..b2cac8cd4c2 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
@@ -23,7 +23,7 @@ public class IdTestCase extends AbstractExportingTestCase {
@Test
public void testExplicitUpperCaseIdField() {
- Search search = new Search("test", null);
+ Search search = new Search("test");
SDDocumentType document = new SDDocumentType("test");
search.addDocument(document);
SDField uri = new SDField("URI", DataType.URI);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java
index 1b51fd354f3..c0a8a47e4ee 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java
@@ -143,7 +143,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase {
@Test
public void testIndexSettingInheritance() {
SDDocumentType parent = new SDDocumentType("parent");
- Search parentSearch = new Search("parent", null);
+ Search parentSearch = new Search("parent");
parentSearch.addDocument(parent);
SDField prefixed = parent.addField("prefixed", DataType.STRING);
prefixed.parseIndexingScript("{ index }");
@@ -151,7 +151,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase {
SDDocumentType child = new SDDocumentType("child");
child.inherit(parent);
- Search childSearch = new Search("child", null);
+ Search childSearch = new Search("child");
childSearch.addDocument(child);
prefixed = (SDField)child.getField("prefixed");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java
index be720cbe4dd..2e36319bd8c 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java
@@ -30,7 +30,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase {
*/
@Test
public void testLiteralBoost() {
- Search search=new Search("literalboost", null);
+ Search search=new Search("literalboost");
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
SDDocumentType document=new SDDocumentType("literalboost");
search.addDocument(document);
@@ -62,7 +62,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase {
*/
@Test
public void testNonDefaultRankLiteralBoost() {
- Search search=new Search("literalboost", null);
+ Search search=new Search("literalboost");
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
SDDocumentType document=new SDDocumentType("literalboost");
search.addDocument(document);
@@ -88,7 +88,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase {
/** Tests literal boosts in two fields going to the same index */
@Test
public void testTwoLiteralBoostFields() {
- Search search=new Search("msb", null);
+ Search search=new Search("msb");
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
SDDocumentType document=new SDDocumentType("msb");
search.addDocument(document);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
index 0c677456a87..f72f67f3711 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
@@ -62,7 +62,7 @@ public class SearchOrdererTestCase extends SchemaTestCase {
}
private static Search createSchema(String name, Map<String, Search> schemas) {
- Search search = new Search(name, null);
+ Search search = new Search(name);
SDDocumentType document = new SDDocumentType(name);
document.setDocumentReferences(new DocumentReferences(emptyMap()));
search.addDocument(document);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
index 07d7405b1db..24d694cd442 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
@@ -72,7 +72,7 @@ public class SummaryMapTestCase extends SchemaTestCase {
}
@Test
public void testPositionDeriving() {
- Search search = new Search("store", null);
+ Search search = new Search("store");
SDDocumentType document = new SDDocumentType("store");
search.addDocument(document);
String fieldName = "location";
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
index c03e915aa8b..364eb9dc014 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
@@ -25,7 +25,7 @@ public class TypeConversionTestCase extends SchemaTestCase {
/** Tests that exact-string stuff is not spilled over to the default index */
@Test
public void testExactStringToStringTypeConversion() {
- Search search = new Search("test", null);
+ Search search = new Search("test");
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
SDDocumentType document = new SDDocumentType("test");
search.addDocument(document);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
index 37b49955bbf..b813f08d49e 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java
@@ -20,7 +20,7 @@ public class VsmFieldsTestCase {
@Test
public void reference_type_field_is_unsearchable() {
- Search search = new Search("test", MockApplicationPackage.createEmpty());
+ Search search = new Search("test", MockApplicationPackage.createEmpty(), new TestableDeployLogger());
search.addDocument(new SDDocumentType("test"));
SDField refField = new TemporarySDField("ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type")));
refField.parseIndexingScript("{ summary }");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
index defaf565a8b..83949ea6f8d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
@@ -6,6 +6,7 @@ import com.yahoo.document.DataType;
import com.yahoo.document.Field;
import com.yahoo.searchdefinition.DocumentReference;
import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.TestableDeployLogger;
import com.yahoo.searchdefinition.document.ImportedField;
import com.yahoo.searchdefinition.document.ImportedFields;
import com.yahoo.searchdefinition.document.ImportedSimpleField;
@@ -44,14 +45,14 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest {
}
private static Search createSearchWithDocument(String documentName) {
- Search search = new Search(documentName, MockApplicationPackage.createEmpty());
+ Search search = new Search(documentName, MockApplicationPackage.createEmpty(), new TestableDeployLogger());
SDDocumentType document = new SDDocumentType(documentName, search);
search.addDocument(document);
return search;
}
private static ImportedFields createSingleImportedField(String fieldName) {
- Search targetSearch = new Search("target_doc", MockApplicationPackage.createEmpty());
+ Search targetSearch = new Search("target_doc", MockApplicationPackage.createEmpty(), new TestableDeployLogger());
SDField targetField = new SDField("target_field", DataType.INT);
DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSearch);
ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
index 7863c544b60..833dbc14a53 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
@@ -153,7 +153,7 @@ public class IndexingScriptRewriterTestCase extends SchemaTestCase {
private static ScriptExpression processField(SDField unprocessedField) {
SDDocumentType sdoc = new SDDocumentType("test");
sdoc.addField(unprocessedField);
- Search search = new Search("test", null);
+ Search search = new Search("test");
search.addDocument(sdoc);
new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false);
return unprocessedField.getIndexingScript();
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
index 7d6a8507dd3..ecafd7a4222 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java
@@ -10,6 +10,7 @@ import com.yahoo.document.TemporaryStructuredDataType;
import com.yahoo.searchdefinition.DocumentReference;
import com.yahoo.searchdefinition.DocumentReferences;
import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.TestableDeployLogger;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.document.TemporaryImportedField;
@@ -29,7 +30,7 @@ public class ParentChildSearchModel {
}
protected Search createSearch(String name) {
- Search result = new Search(name, app);
+ Search result = new Search(name, app, new TestableDeployLogger());
result.addDocument(new SDDocumentType(name));
return result;
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
index e6616ce0dd1..683242c1333 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
@@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.processing;
import com.google.common.collect.ImmutableList;
import com.yahoo.config.application.api.ApplicationPackage;
import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.path.Path;
import com.yahoo.search.query.profile.QueryProfileRegistry;
@@ -62,7 +63,7 @@ class RankProfileSearchFixture {
String rankProfiles, String constant, String field)
throws ParseException {
this.queryProfileRegistry = queryProfileRegistry;
- SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, queryProfileRegistry);
+ SearchBuilder builder = new SearchBuilder(applicationpackage, new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry);
String sdContent = "search test {\n" +
" " + (constant != null ? constant : "") + "\n" +
" document test {\n" +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
index 12fe7e151c0..e3d81be6743 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
@@ -384,7 +384,7 @@ public class RankingExpressionTypeResolverTestCase {
@Test
public void undeclaredQueryFeaturesAreAccepted() throws Exception {
InspectableDeployLogger logger = new InspectableDeployLogger();
- SearchBuilder builder = new SearchBuilder();
+ SearchBuilder builder = new SearchBuilder(logger);
builder.importString(joinLines(
"search test {",
" document test { ",
@@ -401,8 +401,8 @@ public class RankingExpressionTypeResolverTestCase {
" }",
" }",
"}"
- ), logger);
- builder.build(true, logger);
+ ));
+ builder.build(true);
String message = logger.findMessage("The following query features");
assertNull(message);
}
@@ -410,7 +410,7 @@ public class RankingExpressionTypeResolverTestCase {
@Test
public void undeclaredQueryFeaturesAreAcceptedWithWarningWhenUsingTensors() throws Exception {
InspectableDeployLogger logger = new InspectableDeployLogger();
- SearchBuilder builder = new SearchBuilder();
+ SearchBuilder builder = new SearchBuilder(logger);
builder.importString(joinLines(
"search test {",
" document test { ",
@@ -427,8 +427,8 @@ public class RankingExpressionTypeResolverTestCase {
" }",
" }",
"}"
- ), logger);
- builder.build(true, logger);
+ ));
+ builder.build(true);
String message = logger.findMessage("The following query features");
assertNotNull(message);
assertEquals("WARNING: The following query features used in 'my_rank_profile' are not declared in query profile types and " +
@@ -439,7 +439,7 @@ public class RankingExpressionTypeResolverTestCase {
@Test
public void noWarningWhenUsingTensorsWhenQueryFeaturesAreDeclared() throws Exception {
InspectableDeployLogger logger = new InspectableDeployLogger();
- SearchBuilder builder = new SearchBuilder();
+ SearchBuilder builder = new SearchBuilder(logger);
QueryProfileType myType = new QueryProfileType("mytype");
myType.addField(new FieldDescription("rank.feature.query(foo)",
new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))),
@@ -467,8 +467,8 @@ public class RankingExpressionTypeResolverTestCase {
" }",
" }",
"}"
- ), logger);
- builder.build(true, logger);
+ ));
+ builder.build(true);
String message = logger.findMessage("The following query features");
assertNull(message);
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java
index c64dbcdef03..2ae6ffbf343 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java
@@ -1,6 +1,7 @@
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.processing;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.RankProfile;
@@ -84,7 +85,7 @@ public class RankingExpressionWithTransformerTokensTestCase {
" document test {}\n" +
" rank-profile my_profile inherits default {}\n" +
"}";
- SearchBuilder searchBuilder = new SearchBuilder(application, rankProfileRegistry, queryProfileRegistry);
+ SearchBuilder searchBuilder = new SearchBuilder(application, new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry);
searchBuilder.importString(sdContent);
searchBuilder.build();
Search search = searchBuilder.getSearch();
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
index 8861432d97b..0695ae6a249 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
@@ -2,16 +2,22 @@
package com.yahoo.searchdefinition.processing;
import com.yahoo.collections.Pair;
+import com.yahoo.config.FileReference;
import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.search.query.profile.QueryProfileRegistry;
-import com.yahoo.searchdefinition.*;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.SchemaTestCase;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.SearchBuilder;
import com.yahoo.searchdefinition.derived.DerivedConfiguration;
import com.yahoo.searchdefinition.derived.AttributeFields;
import com.yahoo.searchdefinition.derived.RawRankProfile;
import com.yahoo.searchdefinition.derived.TestableDeployLogger;
import com.yahoo.searchdefinition.parser.ParseException;
import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
import org.junit.Test;
import java.io.IOException;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java
index eecab3c03d7..b0c65b3ea76 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java
@@ -21,7 +21,7 @@ public class ReservedRankingExpressionFunctionNamesTestCase {
public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException {
TestDeployLogger deployLogger = new TestDeployLogger();
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
- SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
+ SearchBuilder builder = new SearchBuilder(deployLogger, rankProfileRegistry);
builder.importString(
"search test {\n" +
" document test { \n" +
@@ -50,7 +50,7 @@ public class ReservedRankingExpressionFunctionNamesTestCase {
" }\n" +
" }\n" +
"}\n");
- builder.build(true, deployLogger);
+ builder.build(true);
assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile"));
assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile_2"));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
index 8308b638497..6ab74ef2eae 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
@@ -195,7 +195,7 @@ public class TensorTransformTestCase extends SchemaTestCase {
" }\n" +
" }\n" +
"}\n");
- builder.build(true, new BaseDeployLogger());
+ builder.build(true);
Search s = builder.getSearch();
RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels());
List<Pair<String, String>> testRankProperties = new RawRankProfile(test,
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
index cec313f98d8..28e70130e75 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
@@ -6,6 +6,7 @@ import com.yahoo.document.DataType;
import com.yahoo.document.Field;
import com.yahoo.searchdefinition.DocumentReference;
import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.TestableDeployLogger;
import com.yahoo.searchdefinition.document.ImportedField;
import com.yahoo.searchdefinition.document.ImportedFields;
import com.yahoo.searchdefinition.document.ImportedSimpleField;
@@ -45,14 +46,14 @@ public class ValidateFieldTypesTest {
}
private static Search createSearchWithDocument(String documentName) {
- Search search = new Search(documentName, MockApplicationPackage.createEmpty());
+ Search search = new Search(documentName, MockApplicationPackage.createEmpty(), new TestableDeployLogger());
SDDocumentType document = new SDDocumentType(documentName, search);
search.addDocument(document);
return search;
}
private static ImportedFields createSingleImportedField(String fieldName, DataType dataType) {
- Search targetSearch = new Search("target_doc", MockApplicationPackage.createEmpty());
+ Search targetSearch = new Search("target_doc", MockApplicationPackage.createEmpty(), new TestableDeployLogger());
SDField targetField = new SDField("target_field", dataType);
DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSearch);
ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java
index 27a32f3e754..f9ccdb1a2aa 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.model.application.validation.change.search;
import com.yahoo.config.application.api.ValidationId;
-import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction;
import org.junit.Test;
@@ -11,9 +10,9 @@ import java.time.Instant;
import java.util.Arrays;
import java.util.List;
+import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction;
import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newReindexAction;
import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction;
-import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction;
public class DocumentDatabaseChangeValidatorTest {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java
index da2d5e2e3a0..469e4649c14 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java
@@ -1,6 +1,11 @@
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.searchdefinition.derived.TestableDeployLogger;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import org.junit.Test;
import java.util.Optional;
@@ -43,7 +48,7 @@ public class ClusterResourceLimitsTest {
return this;
}
public ClusterResourceLimits build() {
- var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor);
+ var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, false, new BaseDeployLogger());
builder.setClusterControllerBuilder(ctrlBuilder);
builder.setContentNodeBuilder(nodeBuilder);
return builder.build();
@@ -114,6 +119,29 @@ public class ClusterResourceLimitsTest {
new Fixture(true));
}
+ @Test
+ // TODO: Change to expect exception being thrown when no one uses this in hosted
+ public void default_resource_limits_when_hosted_and_warning_is_logged() {
+ TestableDeployLogger logger = new TestableDeployLogger();
+ final boolean hosted = true;
+
+ ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, hosted, logger);
+ ClusterResourceLimits limits = builder.build(new ModelElement(XML.getDocument("<cluster id=\"test\">" +
+ " <tuning>\n" +
+ " <resource-limits>\n" +
+ " <memory>0.92</memory>\n" +
+ " </resource-limits>\n" +
+ " </tuning>\n" +
+ "</cluster>")
+ .getDocumentElement()));
+
+ assertLimits(0.8, 0.8, limits.getClusterControllerLimits());
+ assertLimits(0.9, 0.9, limits.getContentNodeLimits());
+
+ assertEquals(1, logger.warnings.size());
+ assertEquals("Element resource-limits is not allowed, default limits will be used", logger.warnings.get(0));
+ }
+
private void assertLimits(Double expCtrlDisk, Double expCtrlMemory, Double expNodeDisk, Double expNodeMemory, Fixture f) {
var limits = f.build();
assertLimits(expCtrlDisk, expCtrlMemory, limits.getClusterControllerLimits());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
index b1bd44d93b4..22e38b30959 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
@@ -1,6 +1,7 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.vespa.config.content.FleetcontrollerConfig;
@@ -22,9 +23,12 @@ public class FleetControllerClusterTest {
MockRoot root = new MockRoot("", deployState);
var clusterElement = new ModelElement(doc.getDocumentElement());
return new ClusterControllerConfig.Builder("storage",
- clusterElement,
- new ClusterResourceLimits.Builder(enableFeedBlockInDistributor).build(clusterElement).getClusterControllerLimits()).
- build(root.getDeployState(), root, clusterElement.getXml());
+ clusterElement,
+ new ClusterResourceLimits.Builder(enableFeedBlockInDistributor,
+ false,
+ new BaseDeployLogger())
+ .build(clusterElement).getClusterControllerLimits())
+ .build(root.getDeployState(), root, clusterElement.getXml());
}
private ClusterControllerConfig parse(String xml) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
index bbf59fc66e4..bdae01d5e09 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
@@ -14,6 +14,7 @@ import com.yahoo.tensor.TensorType;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import org.junit.Test;
@@ -96,6 +97,10 @@ public class ModelEvaluationTest {
cluster.getConfig(cb);
RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb);
+ RankingExpressionsConfig.Builder ce = new RankingExpressionsConfig.Builder();
+ cluster.getConfig(ce);
+ RankingExpressionsConfig expressionsConfig = ce.build();
+
OnnxModelsConfig.Builder ob = new OnnxModelsConfig.Builder();
cluster.getConfig(ob);
OnnxModelsConfig onnxModelsConfig = new OnnxModelsConfig(ob);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
index ea4b3db5ebb..66b60ec3782 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
@@ -40,7 +40,7 @@ public class SearchClusterTest {
public void testSdConfigLogical() {
// sd1
SDDocumentType sdt1 = new SDDocumentType("s1");
- Search search1 = new Search("s1", null);
+ Search search1 = new Search("s1");
SDField f1 = new SDField("f1", DataType.STRING);
f1.addAttribute(new Attribute("f1", DataType.STRING));
f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1"))));
@@ -49,7 +49,7 @@ public class SearchClusterTest {
// sd2
SDDocumentType sdt2 = new SDDocumentType("s2");
- Search search2 = new Search("s2", null);
+ Search search2 = new Search("s2");
SDField f2=new SDField("f2", DataType.STRING);
f2.addAttribute(new Attribute("f2", DataType.STRING));
f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2"))));