summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2018-09-18 13:38:38 -0700
committerGitHub <noreply@github.com>2018-09-18 13:38:38 -0700
commitb38fc011d4a5cdcdbdb5d71ca77252502957fa92 (patch)
treee5a20afb500270aac9e4476ab96d60329af99574
parent04a2bea1982819f1e250dfc40b7fbff3fcfe303b (diff)
parent1c37f6a5646d991de68759f2312164799a89ccaa (diff)
Merge pull request #6992 from vespa-engine/bratseth/rank-type-information
Bratseth/rank type information
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java35
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java5
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java16
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java318
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java32
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java30
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java155
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java (renamed from config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java)10
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java (renamed from config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java)20
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java8
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java10
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java6
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java25
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java (renamed from config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java)21
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java201
-rw-r--r--config-model/src/main/javacc/SDParser.jj48
-rw-r--r--config-model/src/test/derived/gemini2/gemini.sd8
-rw-r--r--config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd4
-rw-r--r--config-model/src/test/examples/simple.sd2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java8
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java13
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java30
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java32
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java22
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java22
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java6
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java11
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java6
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java3
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java14
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java26
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java62
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java6
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java26
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java (renamed from config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java)10
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java18
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java14
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java2
-rw-r--r--config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java3
-rw-r--r--model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java4
-rwxr-xr-xsearchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java17
-rwxr-xr-xsearchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java8
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java18
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java14
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java22
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java6
-rwxr-xr-xsearchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java4
-rwxr-xr-xsearchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java56
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java10
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java8
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java12
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java22
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java2
-rw-r--r--vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java13
120 files changed, 798 insertions, 863 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java
new file mode 100644
index 00000000000..9335c0b4005
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java
@@ -0,0 +1,35 @@
+package com.yahoo.searchdefinition;
+
+import java.util.List;
+
+/**
+ * A rank profile which ignores all calls made to it which may fail in a document only setting.
+ * This is used by the search definition parser when it is requested to parse documents only,
+ * to avoid having to check for this in every method which adds to the rank profile.
+ * (And why do we ever want to parse documents only? Because it is used when generating Java classes
+ * from documents, where the full application package may not be available.)
+ *
+ * @author bratseth
+ */
+public class DocumentsOnlyRankProfile extends RankProfile {
+
+ public DocumentsOnlyRankProfile(String name, Search search, RankProfileRegistry rankProfileRegistry) {
+ super(name, search, rankProfileRegistry);
+ }
+
+ @Override
+ public void setFirstPhaseRanking(String expression) {
+ // Ignore
+ }
+
+ @Override
+ public void setSecondPhaseRanking(String expression) {
+ // Ignore
+ }
+
+ @Override
+ public void addFunction(String name, List<String> arguments, String expression, boolean inline) {
+ // Ignore
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java
index afd33da369f..0d9ea00bf73 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java
@@ -21,7 +21,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Stack;
import java.util.stream.Collectors;
/**
@@ -71,7 +70,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement
currentResolutionCallStack.stream().map(Reference::toString).collect(Collectors.joining(" -> ")) +
" -> " + reference);
- // A reference to a macro argument?
+ // A reference to a function argument?
Optional<String> binding = boundIdentifier(reference);
if (binding.isPresent()) {
try {
@@ -117,7 +116,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement
}
/**
- * Returns the default type for this simple feature, or nullif it does not have a default
+ * Returns the default type for this simple feature, or null if it does not have a default
*/
public TensorType defaultTypeOf(Reference reference) {
if ( ! FeatureNames.isSimpleFeature(reference))
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java
deleted file mode 100644
index 1e8f6cb91b9..00000000000
--- a/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package com.yahoo.searchdefinition;
-
-import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.searchdefinition.processing.MinimalProcessing;
-import com.yahoo.vespa.model.container.search.QueryProfiles;
-
-public class MinimalProcessingSearchBuilder extends SearchBuilder {
- public MinimalProcessingSearchBuilder() {
- super();
- }
-
- @Override
- protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) {
- new MinimalProcessing().process(search, deployLogger, getRankProfileRegistry(), queryProfiles, validate);
- }
-}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index b7e1f9d4538..16e494c2db1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -95,13 +95,9 @@ public class RankProfile implements Serializable, Cloneable {
/** The properties of this - a multimap */
private Map<String, List<RankProperty>> rankProperties = new LinkedHashMap<>();
- private Boolean ignoreDefaultRankFeatures=null;
+ private Boolean ignoreDefaultRankFeatures = null;
- private String secondPhaseRankingString=null;
-
- private String firstPhaseRankingString=null;
-
- private Map<String, Macro> macros= new LinkedHashMap<>();
+ private Map<String, RankingExpressionFunction> functions = new LinkedHashMap<>();
private Set<String> filterFields = new HashSet<>();
@@ -339,13 +335,22 @@ public class RankProfile implements Serializable, Cloneable {
* Returns null if no expression is set.
*/
public RankingExpression getFirstPhaseRanking() {
- if (firstPhaseRanking!=null) return firstPhaseRanking;
- if (getInherited()!=null) return getInherited().getFirstPhaseRanking();
+ if (firstPhaseRanking != null) return firstPhaseRanking;
+ if (getInherited() != null) return getInherited().getFirstPhaseRanking();
return null;
}
public void setFirstPhaseRanking(RankingExpression rankingExpression) {
- this.firstPhaseRanking=rankingExpression;
+ this.firstPhaseRanking = rankingExpression;
+ }
+
+ public void setFirstPhaseRanking(String expression) {
+ try {
+ this.firstPhaseRanking = parseRankingExpression("firstphase", expression);
+ }
+ catch (ParseException e) {
+ throw new IllegalArgumentException("Illegal first phase ranking function", e);
+ }
}
/**
@@ -353,31 +358,22 @@ public class RankProfile implements Serializable, Cloneable {
* Returns null if no expression is set.
*/
public RankingExpression getSecondPhaseRanking() {
- if (secondPhaseRanking!=null) return secondPhaseRanking;
- if (getInherited()!=null) return getInherited().getSecondPhaseRanking();
+ if (secondPhaseRanking != null) return secondPhaseRanking;
+ if (getInherited() != null) return getInherited().getSecondPhaseRanking();
return null;
}
public void setSecondPhaseRanking(RankingExpression rankingExpression) {
- this.secondPhaseRanking=rankingExpression;
+ this.secondPhaseRanking = rankingExpression;
}
- /**
- * Called by parser to store the expression string, for delayed evaluation
- *
- * @param exp ranking expression for second phase
- */
- public void setSecondPhaseRankingString(String exp) {
- this.secondPhaseRankingString = exp;
- }
-
- /**
- * Called by parser to store the expression string, for delayed evaluation
- *
- * @param exp ranking expression for first phase
- */
- public void setFirstPhaseRankingString(String exp) {
- this.firstPhaseRankingString = exp;
+ public void setSecondPhaseRanking(String expression) {
+ try {
+ this.secondPhaseRanking = parseRankingExpression("secondphase", expression);
+ }
+ catch (ParseException e) {
+ throw new IllegalArgumentException("Illegal second phase ranking function", e);
+ }
}
/** Returns a read-only view of the summary features to use in this profile. This is never null */
@@ -412,8 +408,8 @@ public class RankProfile implements Serializable, Cloneable {
}
public void addRankFeature(ReferenceNode feature) {
- if (rankFeatures==null)
- rankFeatures=new LinkedHashSet<>();
+ if (rankFeatures == null)
+ rankFeatures = new LinkedHashSet<>();
rankFeatures.add(feature);
}
@@ -522,55 +518,43 @@ public class RankProfile implements Serializable, Cloneable {
}
public boolean getIgnoreDefaultRankFeatures() {
- if (ignoreDefaultRankFeatures!=null) return ignoreDefaultRankFeatures;
- return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures();
+ if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures;
+ return (getInherited() != null) && getInherited().getIgnoreDefaultRankFeatures();
}
- /**
- * Returns the string form of the second phase ranking expression.
- *
- * @return string form of second phase ranking expression
- */
- public String getSecondPhaseRankingString() {
- if (secondPhaseRankingString != null) return secondPhaseRankingString;
- if (getInherited() != null) return getInherited().getSecondPhaseRankingString();
- return null;
- }
-
- /**
- * Returns the string form of the first phase ranking expression.
- *
- * @return string form of first phase ranking expression
- */
- public String getFirstPhaseRankingString() {
- if (firstPhaseRankingString != null) return firstPhaseRankingString;
- if (getInherited() != null) return getInherited().getFirstPhaseRankingString();
- return null;
+ /** Adds a function */
+ public void addFunction(String name, List<String> arguments, String expression, boolean inline) {
+ try {
+ addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline);
+ }
+ catch (ParseException e) {
+ throw new IllegalArgumentException("Could not parse function '" + name + "'", e);
+ }
}
- /** Creates a new (empty) macro and returns it */
- public Macro addMacro(String name, boolean inline) {
- Macro macro = new Macro(name, inline);
- macros.put(name, macro);
- return macro;
+ /** Adds a function and returns it */
+ public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) {
+ RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline);
+ functions.put(function.getName(), rankingExpressionFunction);
+ return rankingExpressionFunction;
}
- /** Returns an unmodifiable view of the macros in this */
- public Map<String, Macro> getMacros() {
- if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap();
- if (macros.size() == 0) return getInherited().getMacros();
- if (getInherited() == null) return Collections.unmodifiableMap(macros);
+ /** Returns an unmodifiable view of the functions in this */
+ public Map<String, RankingExpressionFunction> getFunctions() {
+ if (functions.size() == 0 && getInherited() == null) return Collections.emptyMap();
+ if (functions.size() == 0) return getInherited().getFunctions();
+ if (getInherited() == null) return Collections.unmodifiableMap(functions);
// Neither is null
- Map<String, Macro> allMacros = new LinkedHashMap<>(getInherited().getMacros());
- allMacros.putAll(macros);
- return Collections.unmodifiableMap(allMacros);
+ Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(getInherited().getFunctions());
+ allFunctions.putAll(functions);
+ return Collections.unmodifiableMap(allFunctions);
}
public int getKeepRankCount() {
- if (keepRankCount>=0) return keepRankCount;
- if (getInherited()!=null) return getInherited().getKeepRankCount();
+ if (keepRankCount >= 0) return keepRankCount;
+ if (getInherited() != null) return getInherited().getKeepRankCount();
return -1;
}
@@ -579,8 +563,8 @@ public class RankProfile implements Serializable, Cloneable {
}
public double getRankScoreDropLimit() {
- if (rankScoreDropLimit>-Double.MAX_VALUE) return rankScoreDropLimit;
- if (getInherited()!=null) return getInherited().getRankScoreDropLimit();
+ if (rankScoreDropLimit >- Double.MAX_VALUE) return rankScoreDropLimit;
+ if (getInherited() != null) return getInherited().getRankScoreDropLimit();
return rankScoreDropLimit;
}
@@ -607,58 +591,15 @@ public class RankProfile implements Serializable, Cloneable {
return retval;
}
- /**
- * Will take the parser-set textual ranking expressions and turn into ranking expression objects,
- * if not already done
- */
- // TODO: There doesn't appear to be any good reason to defer parsing of ranking expressions
- // until this is called. Simplify by parsing them right away.
- public void parseExpressions() {
- try {
- parseRankingExpressions();
- parseMacros();
- } catch (ParseException e) {
- throw new IllegalArgumentException(e);
- }
- }
-
- /**
- * Passes the contents of macros on to parser. Then put all the implied rank properties
- * from those macros into the profile's props map.
- */
- private void parseMacros() throws ParseException {
- for (Map.Entry<String, Macro> e : getMacros().entrySet()) {
- String macroName = e.getKey();
- Macro macro = e.getValue();
- if (macro.getRankingExpression() == null) {
- RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression());
- macro.setRankingExpression(expr);
- macro.setTextualExpression(expr.getRoot().toString());
- }
- }
- }
-
- /**
- * Passes ranking expressions on to parser
- *
- * @throws ParseException if either of the ranking expressions could not be parsed
- */
- private void parseRankingExpressions() throws ParseException {
- if (getFirstPhaseRankingString() != null && firstPhaseRanking == null)
- setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString()));
- if (getSecondPhaseRankingString() != null && secondPhaseRanking == null)
- setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString()));
- }
-
- private RankingExpression parseRankingExpression(String expressionName, String exp) throws ParseException {
- if (exp.trim().length() == 0)
+ private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException {
+ if (expression.trim().length() == 0)
throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + ".");
- try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, exp.trim())) {
+ try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) {
return new RankingExpression(expressionName, rankingExpressionReader);
}
catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) {
- ParseException exception = new ParseException("Could not parse ranking expression '" + exp.trim() +
+ ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() +
"' in " + getName()+ ", " + expressionName + ".");
throw (ParseException)exception.initCause(e);
}
@@ -686,14 +627,13 @@ public class RankProfile implements Serializable, Cloneable {
@Override
public RankProfile clone() {
try {
- // Note: This treats RankingExpression in Macros as immutables even though they are not
RankProfile clone = (RankProfile)super.clone();
clone.rankSettings = new LinkedHashSet<>(this.rankSettings);
clone.matchPhaseSettings = this.matchPhaseSettings; // hmm?
clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null;
clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null;
clone.rankProperties = new LinkedHashMap<>(this.rankProperties);
- clone.macros = new LinkedHashMap<>(this.macros);
+ clone.functions = new LinkedHashMap<>(this.functions);
clone.filterFields = new HashSet<>(this.filterFields);
clone.constants = new HashMap<>(this.constants);
return clone;
@@ -719,60 +659,59 @@ public class RankProfile implements Serializable, Cloneable {
}
private void compileThis(QueryProfileRegistry queryProfiles, ImportedModels importedModels) {
- parseExpressions();
- checkNameCollisions(getMacros(), getConstants());
+ checkNameCollisions(getFunctions(), getConstants());
ExpressionTransforms expressionTransforms = new ExpressionTransforms();
- // Macro compiling first pass: compile inline macros without resolving other macros
- Map<String, Macro> inlineMacros = compileMacros(getInlineMacros(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms);
+ // Function compiling first pass: compile inline functions without resolving other functions
+ Map<String, RankingExpressionFunction> inlineFunctions =
+ compileFunctions(getInlineFunctions(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms);
- // Macro compiling second pass: compile all macros and insert previously compiled inline macros
- macros = compileMacros(getMacros(), queryProfiles, importedModels, inlineMacros, expressionTransforms);
+ // Function compiling second pass: compile all functions and insert previously compiled inline functions
+ functions = compileFunctions(getFunctions(), queryProfiles, importedModels, inlineFunctions, expressionTransforms);
- firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms);
- secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms);
+ firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms);
+ secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms);
}
- private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) {
- for (Map.Entry<String, Macro> macroEntry : macros.entrySet()) {
- if (constants.get(macroEntry.getKey()) != null)
- throw new IllegalArgumentException("Cannot have both a constant and macro named '" +
- macroEntry.getKey() + "'");
+ private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) {
+ for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) {
+ if (constants.get(functionEntry.getKey()) != null)
+ throw new IllegalArgumentException("Cannot have both a constant and function named '" +
+ functionEntry.getKey() + "'");
}
}
- private Map<String, Macro> getInlineMacros() {
- return getMacros().entrySet().stream().filter(x -> x.getValue().getInline())
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ private Map<String, RankingExpressionFunction> getInlineFunctions() {
+ return getFunctions().entrySet().stream().filter(x -> x.getValue().inline())
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
- private Map<String, Macro> compileMacros(Map<String, Macro> macros,
- QueryProfileRegistry queryProfiles,
- ImportedModels importedModels,
- Map<String, Macro> inlineMacros,
- ExpressionTransforms expressionTransforms) {
- Map<String, Macro> compiledMacros = new LinkedHashMap<>();
- for (Map.Entry<String, Macro> entry : macros.entrySet()) {
- Macro macro = entry.getValue().clone();
- RankingExpression exp = compile(macro.getRankingExpression(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms);
- macro.setRankingExpression(exp);
- compiledMacros.put(entry.getKey(), macro);
+ private Map<String, RankingExpressionFunction> compileFunctions(Map<String, RankingExpressionFunction> functions,
+ QueryProfileRegistry queryProfiles,
+ ImportedModels importedModels,
+ Map<String, RankingExpressionFunction> inlineFunctions,
+ ExpressionTransforms expressionTransforms) {
+ Map<String, RankingExpressionFunction> compiledFunctions = new LinkedHashMap<>();
+ for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) {
+ RankingExpressionFunction rankingExpressionFunction = entry.getValue();
+ RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms);
+ compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withBody(compiled));
}
- return compiledMacros;
+ return compiledFunctions;
}
private RankingExpression compile(RankingExpression expression,
QueryProfileRegistry queryProfiles,
ImportedModels importedModels,
Map<String, Value> constants,
- Map<String, Macro> inlineMacros,
+ Map<String, RankingExpressionFunction> inlineFunctions,
ExpressionTransforms expressionTransforms) {
if (expression == null) return null;
RankProfileTransformContext context = new RankProfileTransformContext(this,
queryProfiles,
importedModels,
constants,
- inlineMacros);
+ inlineFunctions);
expression = expressionTransforms.transform(expression, context);
for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) {
addRankProperty(rankProperty.getKey(), rankProperty.getValue());
@@ -785,9 +724,9 @@ public class RankProfile implements Serializable, Cloneable {
* referable from this rank profile.
*/
public TypeContext<Reference> typeContext(QueryProfileRegistry queryProfiles) {
- MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream()
- .map(Macro::asExpressionFunction)
- .collect(Collectors.toList()));
+ MapEvaluationTypeContext context = new MapEvaluationTypeContext(getFunctions().values().stream()
+ .map(RankingExpressionFunction::function)
+ .collect(Collectors.toList()));
// Add small and large constants, respectively
getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type()));
@@ -854,11 +793,11 @@ public class RankProfile implements Serializable, Cloneable {
/** True if this setting really pertains to an index, not a field within an index */
private boolean isIndexLevel;
- private Type(String name) {
+ Type(String name) {
this(name,false);
}
- private Type(String name,boolean isIndexLevel) {
+ Type(String name,boolean isIndexLevel) {
this.name = name;
this.isIndexLevel=isIndexLevel;
}
@@ -866,7 +805,7 @@ public class RankProfile implements Serializable, Cloneable {
/** True if this setting really pertains to an index, not a field within an index */
public boolean isIndexLevel() { return isIndexLevel; }
- /** @return The name of this type */
+ /** Returns the name of this type */
public String getName() {
return name;
}
@@ -899,10 +838,12 @@ public class RankProfile implements Serializable, Cloneable {
}
}
+ @Override
public int hashCode() {
return fieldName.hashCode() + 17 * type.hashCode();
}
+ @Override
public boolean equals(Object object) {
if (!(object instanceof RankSetting)) {
return false;
@@ -913,6 +854,7 @@ public class RankProfile implements Serializable, Cloneable {
type.equals(other.type);
}
+ @Override
public String toString() {
return type + " setting " + fieldName + ": " + value;
}
@@ -936,7 +878,7 @@ public class RankProfile implements Serializable, Cloneable {
@Override
public int hashCode() {
- return name.hashCode() + 17*value.hashCode();
+ return name.hashCode() + 17 * value.hashCode();
}
@Override
@@ -953,73 +895,32 @@ public class RankProfile implements Serializable, Cloneable {
}
- /**
- * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro
- */
- public static class Macro implements Serializable, Cloneable {
+ /** A function in a rank profile */
+ public static class RankingExpressionFunction {
- private final String name;
- private String textualExpression = null;
- private RankingExpression expression = null;
- private List<String> formalParams = new ArrayList<>();
+ private final ExpressionFunction function;
- /** True if this should be inlined into calling expressions. Useful for very cheap macros. */
+ /** True if this should be inlined into calling expressions. Useful for very cheap functions. */
private final boolean inline;
- public Macro(String name, boolean inline) {
- this.name = name;
+ public RankingExpressionFunction(ExpressionFunction function, boolean inline) {
+ this.function = function;
this.inline = inline;
}
- public void addParam(String name) {
- formalParams.add(name);
- }
-
- public List<String> getFormalParams() {
- return formalParams;
- }
-
- public String getTextualExpression() {
- return textualExpression;
- }
-
- public void setTextualExpression(String textualExpression) {
- this.textualExpression = textualExpression;
- }
-
- public void setRankingExpression(RankingExpression expr) {
- this.expression=expr;
- }
-
- public RankingExpression getRankingExpression() {
- return expression;
- }
-
- public String getName() {
- return name;
- }
+ public ExpressionFunction function() { return function; }
- public boolean getInline() {
- return inline && formalParams.size() == 0; // only inline no-arg macros;
+ public boolean inline() {
+ return inline && function.arguments().isEmpty(); // only inline no-arg functions;
}
- public ExpressionFunction asExpressionFunction() {
- return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression());
- }
-
- @Override
- public Macro clone() {
- try {
- return (Macro)super.clone();
- }
- catch (CloneNotSupportedException e) {
- throw new RuntimeException("Won't happen", e);
- }
+ public RankingExpressionFunction withBody(RankingExpression expression) {
+ return new RankingExpressionFunction(function.withBody(expression), inline);
}
@Override
public String toString() {
- return "macro " + getName() + ": " + expression;
+ return "function " + function;
}
}
@@ -1106,6 +1007,7 @@ public class RankProfile implements Serializable, Cloneable {
public Map<String, String> getTypes() {
return Collections.unmodifiableMap(types);
}
+
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 592efcc2d85..3c2ebc058ac 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -15,7 +15,6 @@ import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.searchdefinition.parser.SDParser;
import com.yahoo.searchdefinition.parser.SimpleCharStream;
import com.yahoo.searchdefinition.parser.TokenMgrException;
-import com.yahoo.searchdefinition.processing.MinimalProcessing;
import com.yahoo.searchdefinition.processing.Processing;
import com.yahoo.vespa.documentmodel.DocumentModel;
import com.yahoo.vespa.model.container.search.QueryProfiles;
@@ -40,17 +39,25 @@ public class SearchBuilder {
private final DocumentTypeManager docTypeMgr = new DocumentTypeManager();
private List<Search> searchList = new LinkedList<>();
- private ApplicationPackage app = null;
+ private ApplicationPackage app;
private boolean isBuilt = false;
private DocumentModel model = new DocumentModel();
private final RankProfileRegistry rankProfileRegistry;
private final QueryProfileRegistry queryProfileRegistry;
+ /** True to build the document aspect only, skipping instantiation of rank profiles */
+ private final boolean documentsOnly;
+
/** For testing only */
public SearchBuilder() {
this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry());
}
+ /** Used for generating documents for typed access to document fields in Java */
+ public SearchBuilder(boolean documentsOnly) {
+ this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly);
+ }
+
/** For testing only */
public SearchBuilder(ApplicationPackage app) {
this(app, new RankProfileRegistry(), new QueryProfileRegistry());
@@ -69,9 +76,16 @@ public class SearchBuilder {
public SearchBuilder(ApplicationPackage app,
RankProfileRegistry rankProfileRegistry,
QueryProfileRegistry queryProfileRegistry) {
+ this(app, rankProfileRegistry, queryProfileRegistry, false);
+ }
+ public SearchBuilder(ApplicationPackage app,
+ RankProfileRegistry rankProfileRegistry,
+ QueryProfileRegistry queryProfileRegistry,
+ boolean documentsOnly) {
this.app = app;
this.rankProfileRegistry = rankProfileRegistry;
this.queryProfileRegistry = queryProfileRegistry;
+ this.documentsOnly = documentsOnly;
}
/**
@@ -151,7 +165,7 @@ public class SearchBuilder {
Search search;
SimpleCharStream stream = new SimpleCharStream(str);
try {
- search = new SDParser(stream, deployLogger, app, rankProfileRegistry).search(docTypeMgr, searchDefDir);
+ search = new SDParser(stream, deployLogger, app, rankProfileRegistry, documentsOnly).search(docTypeMgr, searchDefDir);
} catch (TokenMgrException e) {
throw new ParseException("Unknown symbol: " + e.getMessage());
} catch (ParseException pe) {
@@ -165,9 +179,9 @@ public class SearchBuilder {
* {@link Search} object is considered to be "raw" if it has not already been processed. This is the case for most
* programmatically constructed search objects used in unit tests.
*
- * @param rawSearch The object to import.
- * @return The name of the imported object.
- * @throws IllegalArgumentException Thrown if the given search object has already been processed.
+ * @param rawSearch the object to import.
+ * @return the name of the imported object.
+ * @throws IllegalArgumentException if the given search object has already been processed.
*/
public String importRawSearch(Search rawSearch) {
if (rawSearch.getName() == null) {
@@ -251,15 +265,15 @@ public class SearchBuilder {
* #build()} method so that subclasses can choose not to build anything.
*/
protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) {
- new Processing().process(search, deployLogger, rankProfileRegistry, queryProfiles, validate);
+ new Processing().process(search, deployLogger, rankProfileRegistry, queryProfiles, validate, documentsOnly);
}
/**
* Convenience method to call {@link #getSearch(String)} when there is only a single {@link Search} object
* built. This method will never return null.
*
- * @return The build object.
- * @throws IllegalStateException Thrown if there is not exactly one search.
+ * @return the built object
+ * @throws IllegalStateException if there is not exactly one search.
*/
public Search getSearch() {
if ( ! isBuilt) throw new IllegalStateException("Searches not built.");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java
index c1b05c0fcdf..49b7ad621af 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java
@@ -1,8 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;
+import com.yahoo.collections.Pair;
+
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
@@ -54,22 +58,18 @@ public class FieldRankSettings {
table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY));
}
- public Map<String,String> deriveRankProperties(int part) {
- Map<String,String> ret = new LinkedHashMap<>();
- int i = part;
- for (Iterator<NativeTable> itr = tables.values().iterator(); itr.hasNext(); ++i) {
- NativeTable table = itr.next();
- if (isFieldMatchTable(table)) {
- ret.put("nativeFieldMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
- }
- if (isAttributeMatchTable(table)) {
- ret.put("nativeAttributeMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
- }
- if (isProximityTable(table)) {
- ret.put("nativeProximity." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
- }
+ public List<Pair<String, String>> deriveRankProperties() {
+ List<Pair<String, String>> properties = new ArrayList<>();
+ for (Iterator<NativeTable> i = tables.values().iterator(); i.hasNext();) {
+ NativeTable table = i.next();
+ if (isFieldMatchTable(table))
+ properties.add(new Pair<>("nativeFieldMatch." + table.getType().getName() + "." + fieldName, table.getName()));
+ if (isAttributeMatchTable(table))
+ properties.add(new Pair<>("nativeAttributeMatch." + table.getType().getName() + "." + fieldName, table.getName()));
+ if (isProximityTable(table))
+ properties.add(new Pair<>("nativeProximity." + table.getType().getName() + "." + fieldName, table.getName()));
}
- return ret;
+ return properties;
}
public String toString() {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index 43cc2fad285..c041d5c6a89 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -50,17 +50,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
*/
public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, AttributeFields attributeFields) {
this.name = rankProfile.getName();
- compressedProperties = compress(removePartFromKeys(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive()));
- }
-
- private List<Pair<String, String>> removePartFromKeys(Map<String, String> map) {
- ImmutableList.Builder<Pair<String, String>> replaced = new ImmutableList.Builder<>();
- for (Map.Entry<String, String> e : map.entrySet()) {
- String key = e.getKey().replaceFirst(".part\\d+$", "");
- String val = e.getValue();
- replaced.add(new Pair<>(key, val));
- }
- return replaced.build();
+ compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive());
}
private Compressor.Compression compress(List<Pair<String, String>> properties) {
@@ -185,57 +175,57 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
rankScoreDropLimit = rankProfile.getRankScoreDropLimit();
ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures();
rankProperties = new ArrayList<>(rankProfile.getRankProperties());
- derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros());
+ derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions());
}
- private void derivePropertiesAndSummaryFeaturesFromMacros(Map<String, RankProfile.Macro> macros) {
- if (macros.isEmpty()) return;
- Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>();
- for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) {
- expressionMacros.put(macro.getKey(), macro.getValue().asExpressionFunction());
+ private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) {
+ if (functions.isEmpty()) return;
+ Map<String, ExpressionFunction> expressionFunctions = new LinkedHashMap<>();
+ for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : functions.entrySet()) {
+ expressionFunctions.put(function.getKey(), function.getValue().function());
}
- Map<String, String> macroProperties = new LinkedHashMap<>();
- macroProperties.putAll(deriveMacroProperties(expressionMacros));
+ Map<String, String> functionProperties = new LinkedHashMap<>();
+ functionProperties.putAll(deriveFunctionProperties(expressionFunctions));
if (firstPhaseRanking != null) {
- macroProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values())));
+ functionProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values())));
}
if (secondPhaseRanking != null) {
- macroProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values())));
+ functionProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values())));
}
- for (Map.Entry<String, String> e : macroProperties.entrySet()) {
+
+ for (Map.Entry<String, String> e : functionProperties.entrySet()) {
rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue()));
}
- SerializationContext context = new SerializationContext(expressionMacros.values(), null, macroProperties);
- replaceMacroSummaryFeatures(context);
+ SerializationContext context = new SerializationContext(expressionFunctions.values(), null, functionProperties);
+ replaceFunctionSummaryFeatures(context);
}
- private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) {
- SerializationContext context = new SerializationContext(eMacros);
- for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) {
+ private Map<String, String> deriveFunctionProperties(Map<String, ExpressionFunction> functions) {
+ SerializationContext context = new SerializationContext(functions);
+ for (Map.Entry<String, ExpressionFunction> e : functions.entrySet()) {
String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString();
context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression);
-
}
return context.serializedFunctions();
}
- private void replaceMacroSummaryFeatures(SerializationContext context) {
+ private void replaceFunctionSummaryFeatures(SerializationContext context) {
if (summaryFeatures == null) return;
- Map<String, ReferenceNode> macroSummaryFeatures = new LinkedHashMap<>();
+ Map<String, ReferenceNode> functionSummaryFeatures = new LinkedHashMap<>();
for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) {
ReferenceNode referenceNode = i.next();
- // Is the feature a macro?
+ // Is the feature a function?
if (context.getFunction(referenceNode.getName()) != null) {
context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()),
referenceNode.toString(new StringBuilder(), context, null, null).toString());
ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput());
- macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode);
+ functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode);
i.remove(); // Will add the expanded one in next block
}
}
- // Then, replace the summary features that were macros
- for (Map.Entry<String, ReferenceNode> e : macroSummaryFeatures.entrySet()) {
+ // Then, replace the summary features that were functions
+ for (Map.Entry<String, ReferenceNode> e : functionSummaryFeatures.entrySet()) {
summaryFeatures.add(e.getValue());
}
}
@@ -300,17 +290,12 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
return settings;
}
- /**
- * Derives the properties this produces. Equal keys are suffixed with .part0 etc, remove when exporting to file
- *
- * @return map of the derived properties
- */
- public Map<String, String> derive() {
- Map<String, String> properties = new LinkedHashMap<>();
- int i = 0;
+ /** Derives the properties this produces */
+ public List<Pair<String, String>> derive() {
+ List<Pair<String, String>> properties = new ArrayList<>();
for (RankProfile.RankProperty property : rankProperties) {
if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) {
- // Could have been set by macro expansion. Set expressions, then skip this property.
+ // Could have been set by function expansion. Set expressions, then skip this property.
try {
firstPhaseRanking = new RankingExpression(property.getValue());
} catch (ParseException e) {
@@ -325,100 +310,92 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
}
}
else {
- properties.put(property.getName() + ".part" + i, property.getValue());
- i++;
+ properties.add(new Pair<>(property.getName(), property.getValue()));
}
}
- properties.putAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase"));
- properties.putAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase"));
+ properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase"));
+ properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase"));
for (FieldRankSettings settings : fieldRankSettings.values()) {
- properties.putAll(settings.deriveRankProperties(i));
+ properties.addAll(settings.deriveRankProperties());
}
- i = 0;
for (RankProfile.RankProperty property : boostAndWeightRankProperties) {
- properties.put(property.getName() + ".part" + i, property.getValue());
- i++;
+ properties.add(new Pair<>(property.getName(), property.getValue()));
}
- i = 0;
for (ReferenceNode feature : summaryFeatures) {
- properties.put(summaryFeatureFefPropertyPrefix + ".part" + i, feature.toString());
- i++;
+ properties.add(new Pair<>(summaryFeatureFefPropertyPrefix, feature.toString()));
}
- i = 0;
for (ReferenceNode feature : rankFeatures) {
- properties.put(rankFeatureFefPropertyPrefix + ".part" + i, feature.toString());
- i++;
+ properties.add(new Pair<>(rankFeatureFefPropertyPrefix, feature.toString()));
}
if (numThreadsPerSearch > 0) {
- properties.put("vespa.matching.numthreadspersearch", numThreadsPerSearch + "");
+ properties.add(new Pair<>("vespa.matching.numthreadspersearch", numThreadsPerSearch + ""));
}
if (minHitsPerThread > 0) {
- properties.put("vespa.matching.minhitsperthread", minHitsPerThread + "");
+ properties.add(new Pair<>("vespa.matching.minhitsperthread", minHitsPerThread + ""));
}
if (numSearchPartitions >= 0) {
- properties.put("vespa.matching.numsearchpartitions", numSearchPartitions + "");
+ properties.add(new Pair<>("vespa.matching.numsearchpartitions", numSearchPartitions + ""));
}
if (termwiseLimit < 1.0) {
- properties.put("vespa.matching.termwise_limit", termwiseLimit + "");
+ properties.add(new Pair<>("vespa.matching.termwise_limit", termwiseLimit + ""));
}
if (matchPhaseSettings != null) {
- properties.put("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute());
- properties.put("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "");
- properties.put("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + "");
- properties.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "");
- properties.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "");
- properties.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "");
+ properties.add(new Pair<>("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute()));
+ properties.add(new Pair<>("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + ""));
+ properties.add(new Pair<>("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + ""));
+ properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + ""));
+ properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + ""));
+ properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + ""));
RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity();
if (diversitySettings != null) {
- properties.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute());
- properties.put("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()));
- properties.put("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()));
- properties.put("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()));
+ properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute()));
+ properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups())));
+ properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor())));
+ properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy())));
}
}
if (rerankCount > -1) {
- properties.put("vespa.hitcollector.heapsize", rerankCount + "");
+ properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + ""));
}
if (keepRankCount > -1) {
- properties.put("vespa.hitcollector.arraysize", keepRankCount + "");
+ properties.add(new Pair<>("vespa.hitcollector.arraysize", keepRankCount + ""));
}
if (rankScoreDropLimit > -Double.MAX_VALUE) {
- properties.put("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "");
+ properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + ""));
}
if (ignoreDefaultRankFeatures) {
- properties.put("vespa.dump.ignoredefaultfeatures", String.valueOf(true));
+ properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true)));
}
Iterator filterFieldsIterator = filterFields.iterator();
while (filterFieldsIterator.hasNext()) {
String fieldName = (String) filterFieldsIterator.next();
- properties.put("vespa.isfilterfield." + fieldName + ".part42", String.valueOf(true));
+ properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true)));
}
for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) {
- properties.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue());
+ properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue()));
}
for (Map.Entry<String, String> queryFeatureType : queryFeatureTypes.entrySet()) {
- properties.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue());
+ properties.add(new Pair<>("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue()));
}
if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties");
return properties;
}
- private Map<String, String> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) {
- Map<String, String> ret = new LinkedHashMap<>();
- if (expression == null) {
- return ret;
- }
+ private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) {
+ List<Pair<String, String>> properties = new ArrayList<>();
+ if (expression == null) return properties;
+
String name = expression.getName();
- if ("".equals(name)) {
+ if ("".equals(name))
name = phase;
- }
+
if (expression.getRoot() instanceof ReferenceNode) {
- ret.put("vespa.rank." + phase, expression.getRoot().toString());
+ properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString()));
} else {
- ret.put("vespa.rank." + phase, "rankingExpression(" + name + ")");
- ret.put("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString());
+ properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")"));
+ properties.add(new Pair<>("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString()));
}
- return ret;
+ return properties;
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
index a639165d297..cbabfffb7a1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
@@ -28,8 +28,8 @@ public class ExpressionTransforms {
new XgboostFeatureConverter(),
new ConstantDereferencer(),
new ConstantTensorTransformer(),
- new MacroInliner(),
- new MacroShadower(),
+ new FunctionInliner(),
+ new FunctionShadower(),
new TensorTransformer(),
new Simplifier());
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java
index 6aef39db4da..c15ef20a455 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java
@@ -8,11 +8,11 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
/**
- * Inlines macros in ranking expressions
+ * Inlines functions in ranking expressions
*
* @author bratseth
*/
-public class MacroInliner extends ExpressionTransformer<RankProfileTransformContext> {
+public class FunctionInliner extends ExpressionTransformer<RankProfileTransformContext> {
@Override
public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) {
@@ -24,9 +24,9 @@ public class MacroInliner extends ExpressionTransformer<RankProfileTransformCont
}
private ExpressionNode transformFeatureNode(ReferenceNode feature, RankProfileTransformContext context) {
- RankProfile.Macro macro = context.inlineMacros().get(feature.getName());
- if (macro == null) return feature;
- return transform(macro.getRankingExpression().getRoot(), context); // inline recursively and return
+ RankProfile.RankingExpressionFunction rankingExpressionFunction = context.inlineFunctions().get(feature.getName());
+ if (rankingExpressionFunction == null) return feature;
+ return transform(rankingExpressionFunction.function().getBody().getRoot(), context); // inline recursively and return
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java
index 758d2b2a87d..74b6471d291 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java
@@ -10,19 +10,19 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
/**
- * Transforms function nodes to reference nodes if a macro shadows a built-in function.
- * This has the effect of allowing macros to redefine built-in functions.
- * Another effect is that we can more or less add built-in functions over time
- * without fear of breaking existing users' macros with the same name.
+ * Transforms function nodes to reference nodes if a rank profile function shadows a built-in function.
+ * This has the effect of allowing rank profile functions to redefine built-in functions.
+ * Another effect is that we can add built-in functions over time
+ * without fear of breaking existing users' functions with the same name.
*
- * However, there is a (largish) caveat. If a user has a macro with a certain number
+ * However, there is a (largish) caveat. If a user has a function with a certain number
* of arguments, and we add in a built-in function with a different arity,
* this will cause parse errors as the Java parser gives precedence to
* built-in functions.
*
* @author lesters
*/
-public class MacroShadower extends ExpressionTransformer<RankProfileTransformContext> {
+public class FunctionShadower extends ExpressionTransformer<RankProfileTransformContext> {
@Override
public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) {
@@ -43,16 +43,14 @@ public class MacroShadower extends ExpressionTransformer<RankProfileTransformCon
private ExpressionNode transformFunctionNode(FunctionNode function, RankProfileTransformContext context) {
String name = function.getFunction().toString();
- RankProfile.Macro macro = context.rankProfile().getMacros().get(name);
- if (macro == null) {
+ RankProfile.RankingExpressionFunction rankingExpressionFunction = context.rankProfile().getFunctions().get(name);
+ if (rankingExpressionFunction == null) {
return transformChildren(function, context);
}
int functionArity = function.getFunction().arity();
- int macroArity = macro.getFormalParams() != null ? macro.getFormalParams().size() : 0;
- if (functionArity != macroArity) {
+ if (functionArity != rankingExpressionFunction.function().arguments().size())
return transformChildren(function, context);
- }
ReferenceNode node = new ReferenceNode(name, function.children(), null);
return transformChildren(node, context);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java
index 40c3b997daa..2fe2dacf2ce 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java
@@ -20,25 +20,25 @@ public class RankProfileTransformContext extends TransformContext {
private final RankProfile rankProfile;
private final QueryProfileRegistry queryProfiles;
private final ImportedModels importedModels;
- private final Map<String, RankProfile.Macro> inlineMacros;
+ private final Map<String, RankProfile.RankingExpressionFunction> inlineFunctions;
private final Map<String, String> rankProperties = new HashMap<>();
public RankProfileTransformContext(RankProfile rankProfile,
QueryProfileRegistry queryProfiles,
ImportedModels importedModels,
Map<String, Value> constants,
- Map<String, RankProfile.Macro> inlineMacros) {
+ Map<String, RankProfile.RankingExpressionFunction> inlineFunctions) {
super(constants);
this.rankProfile = rankProfile;
this.queryProfiles = queryProfiles;
this.importedModels = importedModels;
- this.inlineMacros = inlineMacros;
+ this.inlineFunctions = inlineFunctions;
}
public RankProfile rankProfile() { return rankProfile; }
public QueryProfileRegistry queryProfiles() { return queryProfiles; }
public ImportedModels importedModels() { return importedModels; }
- public Map<String, RankProfile.Macro> inlineMacros() { return inlineMacros; }
+ public Map<String, RankProfile.RankingExpressionFunction> inlineFunctions() { return inlineFunctions; }
public Map<String, String> rankProperties() { return rankProperties; }
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java
index cf123d0f7c1..fefb54a7fe3 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java
@@ -27,7 +27,7 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
search.allImportedFields()
.flatMap(this::getSummaryFieldsForImportedField)
.forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
index e0d32ea8ccd..803a6c5ab40 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
@@ -26,7 +26,7 @@ public class AddExtraFieldsToDocument extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
SDDocumentType document = search.getDocument();
if (document != null) {
for (Field field : search.extraFieldList()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java
index 9ec596792fa..94589d94255 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java
@@ -20,7 +20,7 @@ public class AttributeProperties extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
String fieldName = field.getName();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java
index a95f4264dc6..23257e5eafd 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java
@@ -23,7 +23,7 @@ public class AttributesImplicitWord extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if (fieldImplicitlyWordMatch(field)) {
field.getMatching().setType(Matching.Type.WORD);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java
index a3c4c97cf31..b9be30e8485 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java
@@ -22,7 +22,7 @@ public class Bolding extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
for (SummaryField summary : field.getSummaryFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java
index 37d60c1d32e..a0c4c8adb2d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles;
/**
* Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document
+ *
* @author Vegard Havdal
*/
public class BuiltInFieldSets extends Processor {
@@ -23,7 +24,7 @@ public class BuiltInFieldSets extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
addDocumentFieldSet();
addSearchFieldSet();
// "Hook" the field sets on search onto the document types, since we will include them
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
index d7b688be203..ad862ef767f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
@@ -38,7 +38,7 @@ public class CreatePositionZCurve extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
DataType fieldType = field.getDataType();
if ( ! isSupportedPositionType(fieldType)) continue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java
index 7cc9b4e9b52..b34db6febd5 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java
@@ -15,7 +15,7 @@ public class DeprecateAttributePrefetch extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java
index 861ebad7085..076161a8584 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java
@@ -23,7 +23,7 @@ public class DisallowComplexMapAndWsetKeyTypes extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
// TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java
index 6b78da2146b..029892cba1c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java
@@ -18,8 +18,9 @@ public class DiversitySettingsValidator extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
+ if (documentsOnly) return;
for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) {
if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java
index 59bc3dc66f4..a7c0ebd4a07 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java
@@ -26,7 +26,7 @@ public class ExactMatch extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
Matching.Type matching = field.getMatching().getType();
if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java
index 9cfac625da5..cd33e4434e7 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java
@@ -21,7 +21,7 @@ public class FastAccessValidator extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
String invalidAttributes = search.allFields()
@@ -29,7 +29,7 @@ public class FastAccessValidator extends Processor {
.filter(FastAccessValidator::isIncompatibleAttribute)
.map(Attribute::getName)
.collect(Collectors.joining(", "));
- if (!invalidAttributes.isEmpty()) {
+ if ( ! invalidAttributes.isEmpty()) {
throw new IllegalArgumentException(
String.format(
"For search '%s': The following attributes have a type that is incompatible with fast-access: %s. " +
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java
index 15c11589245..4ef9a9733d5 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java
@@ -24,7 +24,7 @@ public class FieldSetValidity extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java
index 0c75314ffa2..adb8ab62aab 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java
@@ -26,7 +26,9 @@ public class FilterFieldNames extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
+ if (documentsOnly) return;
+
for (SDField f : search.allConcreteFields()) {
if (f.getRanking().isFilter()) {
filterField(f.getName());
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
index b51524b7e62..1f795458875 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
@@ -28,7 +28,7 @@ public class ImplicitSummaries extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
DocumentSummary defaultSummary = search.getSummary("default");
if (defaultSummary == null) {
defaultSummary = new DocumentSummary("default");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java
index 7464f574255..0d99c698aca 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java
@@ -21,7 +21,7 @@ public class ImplicitSummaryFields extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (DocumentSummary docsum : search.getSummaries().values()) {
addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate);
addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
index 9f03cdf4b5e..a3efd086c6a 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
@@ -33,7 +33,7 @@ public class ImportedFieldsResolver extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate));
search.setImportedFields(new ImportedFields(importedFields));
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java
index 018183b91d8..210a8e7009c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java
@@ -23,7 +23,7 @@ public class IndexFieldNames extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java
index 0740029de90..41355a76f47 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java
@@ -24,7 +24,7 @@ public class IndexSettingsNonFieldNames extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
index 419268468c2..aeab2bb6638 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
@@ -27,7 +27,7 @@ public class IndexingInputs extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
ScriptExpression script = field.getIndexingScript();
if (script == null) continue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java
index 6f04184c512..11d69bf6c75 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java
@@ -29,7 +29,7 @@ public class IndexingOutputs extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
ScriptExpression script = field.getIndexingScript();
if (script == null) continue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
index b73151768fd..27520647e3b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
@@ -25,7 +25,7 @@ public class IndexingValidation extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
VerificationContext context = new VerificationContext(new MyAdapter(search));
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
index f3907ae68cb..72777b7dfb4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
@@ -22,7 +22,7 @@ public class IndexingValues extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (Field field : search.getDocument().fieldSet()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java
index c119dc2660b..baaf145dbce 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java
@@ -30,7 +30,7 @@ public class IntegerIndex2Attribute extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) {
if (field.getIndex(field.getName()) != null
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java
index 507a0e87cff..fe94ac9849f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java
@@ -31,7 +31,7 @@ public class LiteralBoost extends Processor {
/** Adds extra search fields and indices to express literal boosts */
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
checkRankModifierRankType(search);
addLiteralBoostsToFields(search);
reduceFieldLiteralBoosts(search);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java
index f853b99bbb2..0daf7265daa 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java
@@ -25,7 +25,7 @@ public class MakeAliases extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
List<String> usedAliases = new ArrayList<>();
for (SDField field : search.allConcreteFields()) {
for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java
index 2c43a65da99..6f67c22d9d2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java
@@ -34,7 +34,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
DocumentSummary defaultSummary=search.getSummary("default");
for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) {
if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java
index fe584fa41c0..ff8a5c2eb0b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java
@@ -30,7 +30,7 @@ public class MatchConsistency extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
Map<String, Matching.Type> types = new HashMap<>();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java
index 479384e09ef..b1728b9bd89 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java
@@ -20,8 +20,9 @@ public class MatchPhaseSettingsValidator extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
+ if (documentsOnly) return;
for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) {
RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java
deleted file mode 100644
index 0fee844ecbc..00000000000
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.yahoo.searchdefinition.processing;
-
-import java.util.Collection;
-
-public class MinimalProcessing extends Processing {
- @Override
- protected Collection<ProcessorFactory> createProcessorFactories() {
- return minimalSetOfProcessors();
- }
-}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java
index 45e018f8fc3..a52a8ab74e6 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java
@@ -31,7 +31,7 @@ public class MultifieldIndexHarmonizer extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
populateIndexToFields(search);
resolveAllConflicts(search);
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java
index 9bcb3929b3d..4d8f0032a78 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java
@@ -15,12 +15,12 @@ public class MutableAttributes extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
- if (!field.isExtraField() && field.getAttributes().containsKey(field.getName())) {
+ if ( ! field.isExtraField() && field.getAttributes().containsKey(field.getName())) {
if (field.getAttributes().get(field.getName()).isMutable()) {
throw new IllegalArgumentException("Field '" + field.getName() + "' in '" + search.getDocument().getName() +
- "' can not be marked mutable as it is inside the document clause.");
+ "' can not be marked mutable as it is inside the document clause.");
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java
index cdfba54ee5a..3f3fc11380b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java
@@ -26,7 +26,7 @@ public class NGramMatch extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if (field.getMatching().getType().equals(Matching.Type.GRAM))
implementGramMatch(search, field, validate);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
index c89c709ffbf..8f2a29abcb6 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
@@ -22,7 +22,7 @@ public class OptimizeIlscript extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
ScriptExpression script = field.getIndexingScript();
if (script == null) continue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java
index 3583c4a0162..79f19efe422 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java
@@ -33,7 +33,7 @@ public class PredicateProcessor extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if (field.getDataType() == DataType.PREDICATE) {
if (validate && field.doesIndexing()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
index 9b4b74d07d8..8c8c32389e2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -20,7 +20,7 @@ import java.util.List;
*/
public class Processing {
- protected Collection<ProcessorFactory> minimalSetOfProcessors() {
+ private Collection<ProcessorFactory> processors() {
return Arrays.asList(
SearchMustHaveDocument::new,
UrlFieldValidator::new,
@@ -74,24 +74,13 @@ public class Processing {
RankProfileTypeSettingsProcessor::new,
ReferenceFieldsProcessor::new,
FastAccessValidator::new,
- ReservedMacroNames::new);
- }
-
- private Collection<ProcessorFactory> extendedSetOfProcessors() {
- return Arrays.asList(
+ ReservedFunctionNames::new,
RankingExpressionTypeValidator::new,
-
- // These should be last.
+ // These should be last:
IndexingValidation::new,
IndexingValues::new);
}
- protected Collection<ProcessorFactory> createProcessorFactories() {
- List<ProcessorFactory> processorFactories = new ArrayList<>(minimalSetOfProcessors());
- processorFactories.addAll(extendedSetOfProcessors());
- return processorFactories;
- }
-
/**
* Runs all search processors on the given {@link Search} object. These will modify the search object, <b>possibly
* exchanging it with another</b>, as well as its document types.
@@ -102,17 +91,17 @@ public class Processing {
* @param queryProfiles The query profiles contained in the application this search is part of.
*/
public void process(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
- QueryProfiles queryProfiles, boolean validate)
- {
- Collection<ProcessorFactory> factories = createProcessorFactories();
+ QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) {
+ Collection<ProcessorFactory> factories = processors();
search.process();
factories.stream()
.map(factory -> factory.create(search, deployLogger, rankProfileRegistry, queryProfiles))
- .forEach(processor -> processor.process(validate));
+ .forEach(processor -> processor.process(validate, documentsOnly));
}
@FunctionalInterface
public interface ProcessorFactory {
Processor create(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles);
}
+
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
index b938e40d9a2..6bfd0ef29ea 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
@@ -53,8 +53,10 @@ public abstract class Processor {
* @param validate true to throw exceptions on validation errors, false to make the best possible effort
* at completing processing without throwing an exception.
* If we are not validating, emitting warnings have no effect and can (but must not) be skipped.
+ * @param documentsOnly true to skip processing (including validation, regardless of the validate setting)
+ * of aspects not relating to document definitions (e.g rank profiles)
*/
- public abstract void process(boolean validate);
+ public abstract void process(boolean validate, boolean documentsOnly);
/**
* Convenience method for adding a no-strings-attached implementation field for a regular field
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java
index 81455991cc9..102d1910360 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java
@@ -33,8 +33,9 @@ public class RankingExpressionTypeValidator extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
+ if (documentsOnly) return;
for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) {
try {
@@ -48,7 +49,6 @@ public class RankingExpressionTypeValidator extends Processor {
/** Throws an IllegalArgumentException if the given rank profile does not produce valid type */
private void validate(RankProfile profile) {
- profile.parseExpressions();
TypeContext context = profile.typeContext(queryProfiles);
profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context));
ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java
index 76afcd5d520..0418538922b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java
@@ -28,7 +28,7 @@ public class ReferenceFieldsProcessor extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
clearSummaryAttributeAspectForConcreteFields();
clearSummaryAttributeAspectForExplicitSummaryFields();
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java
index f2aa31bb9c3..805cbaced0f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java
@@ -28,7 +28,7 @@ public class ReservedDocumentNames extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
String docName = search.getDocument().getName();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java
index adcebed9254..d7099215f17 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java
@@ -13,29 +13,30 @@ import java.util.Set;
import java.util.logging.Level;
/**
- * Issues a warning if some macro has a reserved name. This is not necessarily
- * an error, as a macro can shadow a built-in function.
+ * Issues a warning if some function has a reserved name. This is not necessarily
+ * an error, as a rank profile function can shadow a built-in function.
*
* @author lesters
*/
-public class ReservedMacroNames extends Processor {
+public class ReservedFunctionNames extends Processor {
private static Set<String> reservedNames = getReservedNames();
- public ReservedMacroNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ public ReservedFunctionNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
super(search, deployLogger, rankProfileRegistry, queryProfiles);
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
+ if (documentsOnly) return;
for (RankProfile rp : rankProfileRegistry.all()) {
- for (String macroName : rp.getMacros().keySet()) {
- if (reservedNames.contains(macroName)) {
- deployLogger.log(Level.WARNING, "Macro \"" + macroName + "\" " +
- "in rank profile \"" + rp.getName() + "\" " +
- "has a reserved name. This might mean that the macro shadows " +
+ for (String functionName : rp.getFunctions().keySet()) {
+ if (reservedNames.contains(functionName)) {
+ deployLogger.log(Level.WARNING, "Function '" + functionName + "' " +
+ "in rank profile '" + rp.getName() + "' " +
+ "has a reserved name. This might mean that the function shadows " +
"the built-in function with the same name."
);
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java
index 403de1253b4..2d8eaff7762 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java
@@ -19,7 +19,7 @@ public class SearchMustHaveDocument extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
if (search.getDocument() == null)
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
index a0c884d25f9..8a4795c4dd2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
@@ -24,7 +24,8 @@ public class SetLanguage extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
+ if ( ! validate) return;
List<String> textFieldsWithoutLanguage = new ArrayList<>();
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java
index a19ea8d7068..715828f808a 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java
@@ -20,7 +20,7 @@ public class SetRankTypeEmptyOnFilters extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if (field.getRanking().isFilter()) {
field.setRankType(RankType.EMPTY);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java
index 6426c724a07..defcf761649 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java
@@ -21,7 +21,7 @@ public class SortingSettings extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java
index d56b0272f06..e133f88f45c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java
@@ -16,7 +16,7 @@ public class StringSettingsOnNonStringFields extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java
index a952d3732b3..d2c4968ca26 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java
@@ -26,7 +26,7 @@ public class SummaryConsistency extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (DocumentSummary summary : search.getSummaries().values()) {
if (summary.getName().equals("default")) continue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java
index 647a433f201..b8d170c07f6 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java
@@ -26,7 +26,7 @@ public class SummaryDynamicStructsArrays extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java
index 7bcbd9a267c..9b51c7c473e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java
@@ -22,7 +22,7 @@ public class SummaryFieldsMustHaveValidSource extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (DocumentSummary summary : search.getSummaries().values()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java
index 23569cf39ae..678d5324e38 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java
@@ -26,7 +26,7 @@ public class SummaryNamesFieldCollisions extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
Map<String, Pair<String, String>> fieldToClassAndSource = new HashMap<>();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java
index 177fc7f2326..79b7a6067b9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java
@@ -25,7 +25,7 @@ public class TagType extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag())
implementTagType(field);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
index 08571168336..8e54d7c00d6 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -22,7 +22,7 @@ public class TensorFieldProcessor extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java
index 645ed5121ea..74f30d6a730 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java
@@ -34,7 +34,7 @@ public class TextMatch extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if (field.getMatching().getType() != Matching.Type.TEXT) continue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
index ac376982cfa..d81fdf70d20 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
@@ -28,7 +28,7 @@ public class UriHack extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if (field.doesIndexing()) {
DataType fieldType = field.getDataType();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java
index ed813b42fff..c6b83349691 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java
@@ -18,7 +18,7 @@ public class UrlFieldValidator extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
for (SDField field : search.allConcreteFields()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
index 54ad9f13f6f..21b7f1d2675 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
@@ -28,7 +28,7 @@ public class ValidateFieldTypes extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
String searchName = search.getName();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java
index a0b204a25f2..408d60e1cff 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java
@@ -21,7 +21,7 @@ public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
Matching defaultMatching = new Matching();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java
index 892bcdad12c..13fe3f24d69 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java
@@ -25,7 +25,7 @@ public class WordMatch extends Processor {
super(search, deployLogger, rankProfileRegistry, queryProfiles);
}
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
for (SDField field : search.allConcreteFields()) {
if ( ! field.getMatching().getType().equals(Matching.Type.WORD)) continue;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
index cc1638347f6..ec4cbdfe58b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
@@ -22,7 +22,7 @@ import java.util.Map;
import java.util.Optional;
/**
- * Class that processes a search instance and sets type settings on all rank profiles.
+ * This processes a search instance and sets type settings on all rank profiles.
*
* Currently, type settings are limited to the type of tensor attribute fields and tensor query features.
*
@@ -35,7 +35,9 @@ public class RankProfileTypeSettingsProcessor extends Processor {
}
@Override
- public void process(boolean validate) {
+ public void process(boolean validate, boolean documentsOnly) {
+ if (documentsOnly) return;
+
processAttributeFields();
processImportedFields();
processQueryProfileTypes();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index 282e5a29962..4b70b1b5ae2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -32,6 +32,7 @@ import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.RankingConstants;
import com.yahoo.searchdefinition.derived.AttributeFields;
import com.yahoo.searchdefinition.derived.RankProfileList;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.vespa.model.ml.ConvertedModel;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModel;
@@ -236,7 +237,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()),
model.name(), profile, queryProfiles, model);
for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) {
- profile.addMacro(entry.getKey(), false).setRankingExpression(entry.getValue());
+ profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false);
}
}
}
@@ -248,7 +249,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
rankProfileRegistry.add(profile);
ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile);
for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) {
- profile.addMacro(entry.getKey(), false).setRankingExpression(entry.getValue());
+ profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false);
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
index e2236feb336..adf5c81283e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
@@ -13,6 +13,7 @@ import com.yahoo.searchdefinition.FeatureNames;
import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankingConstant;
import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.Reference;
import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
@@ -139,7 +140,7 @@ public class ConvertedModel {
public ExpressionNode expression(FeatureArguments arguments, RankProfileTransformContext context) {
RankingExpression expression = selectExpression(arguments);
if (sourceModel.isPresent()) // we can verify
- verifyRequiredMacros(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles());
+ verifyRequiredFunctions(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles());
return expression.getRoot();
}
@@ -183,41 +184,41 @@ public class ConvertedModel {
QueryProfileRegistry queryProfiles,
ModelStore store) {
// Add constants
- Set<String> constantsReplacedByMacros = new HashSet<>();
+ Set<String> constantsReplacedByFunctions = new HashSet<>();
model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v));
model.largeConstants().forEach((k, v) -> transformLargeConstant(store, profile, queryProfiles,
- constantsReplacedByMacros, k, v));
+ constantsReplacedByFunctions, k, v));
- // Add macros
- addGeneratedMacros(model, profile);
+ // Add functions
+ addGeneratedFunctions(model, profile);
// Add expressions
Map<String, RankingExpression> expressions = new HashMap<>();
for (Pair<String, RankingExpression> output : model.outputExpressions()) {
addExpression(output.getSecond(), output.getFirst(),
- constantsReplacedByMacros,
+ constantsReplacedByFunctions,
model, store, profile, queryProfiles,
expressions);
}
- // Transform and save macro - must come after reading expressions due to optimization transforms
- // and must use the macro expression added to the profile, which may differ from the one saved in the model,
+ // Transform and save function - must come after reading expressions due to optimization transforms
+ // and must use the function expression added to the profile, which may differ from the one saved in the model,
// after rewrite
- model.macros().forEach((k, v) -> transformGeneratedMacro(store, constantsReplacedByMacros, k,
- profile.getMacros().get(k).getRankingExpression()));
+ model.functions().forEach((k, v) -> transformGeneratedFunction(store, constantsReplacedByFunctions, k,
+ profile.getFunctions().get(k).function().getBody()));
return expressions;
}
private static void addExpression(RankingExpression expression,
String expressionName,
- Set<String> constantsReplacedByMacros,
+ Set<String> constantsReplacedByFunctions,
ImportedModel model,
ModelStore store,
RankProfile profile,
QueryProfileRegistry queryProfiles,
Map<String, RankingExpression> expressions) {
- expression = replaceConstantsByMacros(expression, constantsReplacedByMacros);
+ expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions);
reduceBatchDimensions(expression, model, profile, queryProfiles);
store.writeExpression(expressionName, expression);
expressions.put(expressionName, expression);
@@ -232,8 +233,8 @@ public class ConvertedModel {
profile.rankingConstants().add(constant);
}
- for (Pair<String, RankingExpression> macro : store.readMacros()) {
- addGeneratedMacroToProfile(profile, macro.getFirst(), macro.getSecond());
+ for (Pair<String, RankingExpression> function : store.readFunctions()) {
+ addGeneratedFunctionToProfile(profile, function.getFirst(), function.getSecond());
}
return store.readExpressions();
@@ -247,16 +248,16 @@ public class ConvertedModel {
private static void transformLargeConstant(ModelStore store,
RankProfile profile,
QueryProfileRegistry queryProfiles,
- Set<String> constantsReplacedByMacros,
+ Set<String> constantsReplacedByFunctions,
String constantName,
Tensor constantValue) {
- RankProfile.Macro macroOverridingConstant = profile.getMacros().get(constantName);
- if (macroOverridingConstant != null) {
- TensorType macroType = macroOverridingConstant.getRankingExpression().type(profile.typeContext(queryProfiles));
- if ( ! macroType.equals(constantValue.type()))
- throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " +
- typeMismatchExplanation(constantValue.type(), macroType));
- constantsReplacedByMacros.add(constantName); // will replace constant(constantName) by constantName later
+ RankProfile.RankingExpressionFunction rankingExpressionFunctionOverridingConstant = profile.getFunctions().get(constantName);
+ if (rankingExpressionFunctionOverridingConstant != null) {
+ TensorType functionType = rankingExpressionFunctionOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles));
+ if ( ! functionType.equals(constantValue.type()))
+ throw new IllegalArgumentException("Function '" + constantName + "' replaces the constant with this name. " +
+ typeMismatchExplanation(constantValue.type(), functionType));
+ constantsReplacedByFunctions.add(constantName); // will replace constant(constantName) by constantName later
}
else {
Path constantPath = store.writeLargeConstant(constantName, constantValue);
@@ -267,79 +268,75 @@ public class ConvertedModel {
}
}
- private static void transformGeneratedMacro(ModelStore store,
- Set<String> constantsReplacedByMacros,
- String macroName,
- RankingExpression expression) {
+ private static void transformGeneratedFunction(ModelStore store,
+ Set<String> constantsReplacedByFunctions,
+ String functionName,
+ RankingExpression expression) {
- expression = replaceConstantsByMacros(expression, constantsReplacedByMacros);
- store.writeMacro(macroName, expression);
+ expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions);
+ store.writeFunction(functionName, expression);
}
- private static void addGeneratedMacroToProfile(RankProfile profile, String macroName, RankingExpression expression) {
- if (profile.getMacros().containsKey(macroName)) {
- if ( ! profile.getMacros().get(macroName).getRankingExpression().equals(expression))
- throw new IllegalArgumentException("Generated macro '" + macroName + "' already exists in " + profile +
+ private static void addGeneratedFunctionToProfile(RankProfile profile, String functionName, RankingExpression expression) {
+ if (profile.getFunctions().containsKey(functionName)) {
+ if ( ! profile.getFunctions().get(functionName).function().getBody().equals(expression))
+ throw new IllegalArgumentException("Generated function '" + functionName + "' already exists in " + profile +
" - with a different definition" +
- ": Has\n" + profile.getMacros().get(macroName).getRankingExpression() +
+ ": Has\n" + profile.getFunctions().get(functionName).function().getBody() +
"\nwant to add " + expression + "\n");
return;
}
- RankProfile.Macro macro = profile.addMacro(macroName, false); // TODO: Inline if only used once
- macro.setRankingExpression(expression);
- macro.setTextualExpression(expression.getRoot().toString());
+ profile.addFunction(new ExpressionFunction(functionName, expression), false); // TODO: Inline if only used once
}
/**
- * Verify that the macros referred in the given expression exists in the given rank profile,
- * and return tensors of the types specified in requiredMacros.
+ * Verify that the functions referred in the given expression exists in the given rank profile,
+ * and return tensors of the types specified in requiredFunctions.
*/
- private static void verifyRequiredMacros(RankingExpression expression, ImportedModel model,
- RankProfile profile, QueryProfileRegistry queryProfiles) {
- Set<String> macroNames = new HashSet<>();
- addMacroNamesIn(expression.getRoot(), macroNames, model);
- for (String macroName : macroNames) {
- TensorType requiredType = model.requiredMacros().get(macroName);
- if (requiredType == null) continue; // Not a required macro
-
- RankProfile.Macro macro = profile.getMacros().get(macroName);
- if (macro == null)
- throw new IllegalArgumentException("Model refers input '" + macroName +
- "' of type " + requiredType + " but this macro is not present in " +
+ private static void verifyRequiredFunctions(RankingExpression expression, ImportedModel model,
+ RankProfile profile, QueryProfileRegistry queryProfiles) {
+ Set<String> functionNames = new HashSet<>();
+ addFunctionNamesIn(expression.getRoot(), functionNames, model);
+ for (String functionName : functionNames) {
+ TensorType requiredType = model.requiredFunctions().get(functionName);
+ if (requiredType == null) continue; // Not a required function
+
+ RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName);
+ if (rankingExpressionFunction == null)
+ throw new IllegalArgumentException("Model refers input '" + functionName +
+ "' of type " + requiredType + " but this function is not present in " +
profile);
// TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second
// phase and summary features), as it may only resolve correctly given those bindings
- // Or, probably better, annotate the macros with type constraints here and verify during general
+ // Or, probably better, annotate the functions with type constraints here and verify during general
// type verification
- TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles));
+ TensorType actualType = rankingExpressionFunction.function().getBody().getRoot().type(profile.typeContext(queryProfiles));
if ( actualType == null)
- throw new IllegalArgumentException("Model refers input '" + macroName +
+ throw new IllegalArgumentException("Model refers input '" + functionName +
"' of type " + requiredType +
- " which must be produced by a macro in the rank profile, but " +
- "this macro references a feature which is not declared");
+ " which must be produced by a function in the rank profile, but " +
+ "this function references a feature which is not declared");
if ( ! actualType.isAssignableTo(requiredType))
- throw new IllegalArgumentException("Model refers input '" + macroName + "'. " +
+ throw new IllegalArgumentException("Model refers input '" + functionName + "'. " +
typeMismatchExplanation(requiredType, actualType));
}
}
private static String typeMismatchExplanation(TensorType requiredType, TensorType actualType) {
- return "The required type of this is " + requiredType + ", but this macro returns " + actualType +
+ return "The required type of this is " + requiredType + ", but this function returns " + actualType +
(actualType.rank() == 0 ? ". This is often due to missing declaration of query tensor features " +
"in query profile types - see the documentation."
: "");
}
- /**
- * Add the generated macros to the rank profile
- */
- private static void addGeneratedMacros(ImportedModel model, RankProfile profile) {
- model.macros().forEach((k, v) -> addGeneratedMacroToProfile(profile, k, v.copy()));
+ /** Add the generated functions to the rank profile */
+ private static void addGeneratedFunctions(ImportedModel model, RankProfile profile) {
+ model.functions().forEach((k, v) -> addGeneratedFunctionToProfile(profile, k, v.copy()));
}
/**
* Check if batch dimensions of inputs can be reduced out. If the input
- * macro specifies that a single exemplar should be evaluated, we can
+ * function specifies that a single exemplar should be evaluated, we can
* reduce the batch dimension out.
*/
private static void reduceBatchDimensions(RankingExpression expression, ImportedModel model,
@@ -347,19 +344,19 @@ public class ConvertedModel {
TypeContext<Reference> typeContext = profile.typeContext(queryProfiles);
TensorType typeBeforeReducing = expression.getRoot().type(typeContext);
- // Check generated macros for inputs to reduce
- Set<String> macroNames = new HashSet<>();
- addMacroNamesIn(expression.getRoot(), macroNames, model);
- for (String macroName : macroNames) {
- if ( ! model.macros().containsKey(macroName)) continue;
+ // Check generated functions for inputs to reduce
+ Set<String> functionNames = new HashSet<>();
+ addFunctionNamesIn(expression.getRoot(), functionNames, model);
+ for (String functionName : functionNames) {
+ if ( ! model.functions().containsKey(functionName)) continue;
- RankProfile.Macro macro = profile.getMacros().get(macroName);
- if (macro == null) {
- throw new IllegalArgumentException("Model refers to generated macro '" + macroName +
- "but this macro is not present in " + profile);
+ RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName);
+ if (rankingExpressionFunction == null) {
+ throw new IllegalArgumentException("Model refers to generated function '" + functionName +
+ "but this function is not present in " + profile);
}
- RankingExpression macroExpression = macro.getRankingExpression();
- macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext));
+ RankingExpression functionExpression = rankingExpressionFunction.function().getBody();
+ functionExpression.setRoot(reduceBatchDimensionsAtInput(functionExpression.getRoot(), model, typeContext));
}
// Check expression for inputs to reduce
@@ -378,7 +375,7 @@ public class ConvertedModel {
List<ExpressionNode> children = ((TensorFunctionNode)node).children();
if (children.size() == 1 && children.get(0) instanceof ReferenceNode) {
ReferenceNode referenceNode = (ReferenceNode) children.get(0);
- if (model.requiredMacros().containsKey(referenceNode.getName())) {
+ if (model.requiredFunctions().containsKey(referenceNode.getName())) {
return reduceBatchDimensionExpression(tensorFunction, typeContext);
}
}
@@ -386,7 +383,7 @@ public class ConvertedModel {
}
if (node instanceof ReferenceNode) {
ReferenceNode referenceNode = (ReferenceNode) node;
- if (model.requiredMacros().containsKey(referenceNode.getName())) {
+ if (model.requiredFunctions().containsKey(referenceNode.getName())) {
return reduceBatchDimensionExpression(TensorFunctionNode.wrapArgument(node), typeContext);
}
}
@@ -447,47 +444,47 @@ public class ConvertedModel {
}
/**
- * If a constant c is overridden by a macro, we need to replace instances of "constant(c)" by "c" in expressions.
+ * If a constant c is overridden by a function, we need to replace instances of "constant(c)" by "c" in expressions.
* This method does that for the given expression and returns the result.
*/
- private static RankingExpression replaceConstantsByMacros(RankingExpression expression,
- Set<String> constantsReplacedByMacros) {
- if (constantsReplacedByMacros.isEmpty()) return expression;
+ private static RankingExpression replaceConstantsByFunctions(RankingExpression expression,
+ Set<String> constantsReplacedByFunctions) {
+ if (constantsReplacedByFunctions.isEmpty()) return expression;
return new RankingExpression(expression.getName(),
- replaceConstantsByMacros(expression.getRoot(), constantsReplacedByMacros));
+ replaceConstantsByFunctions(expression.getRoot(), constantsReplacedByFunctions));
}
- private static ExpressionNode replaceConstantsByMacros(ExpressionNode node, Set<String> constantsReplacedByMacros) {
+ private static ExpressionNode replaceConstantsByFunctions(ExpressionNode node, Set<String> constantsReplacedByFunctions) {
if (node instanceof ReferenceNode) {
Reference reference = ((ReferenceNode)node).reference();
if (FeatureNames.isSimpleFeature(reference) && reference.name().equals("constant")) {
String argument = reference.simpleArgument().get();
- if (constantsReplacedByMacros.contains(argument))
+ if (constantsReplacedByFunctions.contains(argument))
return new ReferenceNode(argument);
}
}
if (node instanceof CompositeNode) { // not else: this matches some of the same nodes as the outer if above
CompositeNode composite = (CompositeNode)node;
return composite.setChildren(composite.children().stream()
- .map(child -> replaceConstantsByMacros(child, constantsReplacedByMacros))
+ .map(child -> replaceConstantsByFunctions(child, constantsReplacedByFunctions))
.collect(Collectors.toList()));
}
return node;
}
- private static void addMacroNamesIn(ExpressionNode node, Set<String> names, ImportedModel model) {
+ private static void addFunctionNamesIn(ExpressionNode node, Set<String> names, ImportedModel model) {
if (node instanceof ReferenceNode) {
ReferenceNode referenceNode = (ReferenceNode)node;
- if (referenceNode.getOutput() == null) { // macro references cannot specify outputs
+ if (referenceNode.getOutput() == null) { // function references cannot specify outputs
names.add(referenceNode.getName());
- if (model.macros().containsKey(referenceNode.getName())) {
- addMacroNamesIn(model.macros().get(referenceNode.getName()).getRoot(), names, model);
+ if (model.functions().containsKey(referenceNode.getName())) {
+ addFunctionNamesIn(model.functions().get(referenceNode.getName()).getRoot(), names, model);
}
}
}
else if (node instanceof CompositeNode) {
for (ExpressionNode child : ((CompositeNode)node).children())
- addMacroNamesIn(child, names, model);
+ addFunctionNamesIn(child, names, model);
}
}
@@ -551,19 +548,19 @@ public class ConvertedModel {
return expressions;
}
- /** Adds this macro expression to the application package so it can be read later. */
- void writeMacro(String name, RankingExpression expression) {
- application.getFile(modelFiles.macrosPath()).appendFile(name + "\t" +
- expression.getRoot().toString() + "\n");
+ /** Adds this function expression to the application package so it can be read later. */
+ void writeFunction(String name, RankingExpression expression) {
+ application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" +
+ expression.getRoot().toString() + "\n");
}
- /** Reads the previously stored macro expressions for these arguments */
- List<Pair<String, RankingExpression>> readMacros() {
+ /** Reads the previously stored function expressions for these arguments */
+ List<Pair<String, RankingExpression>> readFunctions() {
try {
- ApplicationFile file = application.getFile(modelFiles.macrosPath());
+ ApplicationFile file = application.getFile(modelFiles.functionsPath());
if ( ! file.exists()) return Collections.emptyList();
- List<Pair<String, RankingExpression>> macros = new ArrayList<>();
+ List<Pair<String, RankingExpression>> functions = new ArrayList<>();
BufferedReader reader = new BufferedReader(file.createReader());
String line;
while (null != (line = reader.readLine())) {
@@ -571,13 +568,13 @@ public class ConvertedModel {
String name = parts[0];
try {
RankingExpression expression = new RankingExpression(parts[0], parts[1]);
- macros.add(new Pair<>(name, expression));
+ functions.add(new Pair<>(name, expression));
}
catch (ParseException e) {
throw new IllegalStateException("Could not parse " + name, e);
}
}
- return macros;
+ return functions;
}
catch (IOException e) {
throw new UncheckedIOException(e);
@@ -725,9 +722,9 @@ public class ConvertedModel {
return storedModelReplicatedPath().append("constants");
}
- /** Path to the macros file */
- public Path macrosPath() {
- return storedModelReplicatedPath().append("macros.txt");
+ /** Path to the functions file */
+ public Path functionsPath() {
+ return storedModelReplicatedPath().append("functions.txt");
}
}
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 63d3926afad..813d1d47533 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -34,6 +34,7 @@ import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferen
import com.yahoo.searchdefinition.RankingConstant;
import com.yahoo.searchdefinition.Index;
import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.DocumentsOnlyRankProfile;
import com.yahoo.searchdefinition.DefaultRankProfile;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings;
@@ -58,6 +59,8 @@ import com.yahoo.language.Linguistics;
import com.yahoo.language.simple.SimpleLinguistics;
import com.yahoo.search.query.ranking.Diversity;
import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.logging.Level;
import org.apache.commons.lang.StringUtils;
@@ -73,6 +76,7 @@ public class SDParser {
private ApplicationPackage app;
private DeployLogger deployLogger;
private RankProfileRegistry rankProfileRegistry;
+ private boolean documentsOnly;
/** For testing only */
public SDParser(String input, DeployLogger deployLogger) {
@@ -81,17 +85,24 @@ public class SDParser {
/** For testing only */
public SDParser(SimpleCharStream stream, DeployLogger deployLogger) {
- this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry());
+ this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry(), false);
}
+ /**
+ * Creates a parser
+ *
+ * @param documentsOnly true to only parse the document aspect of a search definition (e.g skip rank profiles)
+ */
public SDParser(SimpleCharStream stream,
DeployLogger deployLogger,
ApplicationPackage applicationPackage,
- RankProfileRegistry rankProfileRegistry) {
+ RankProfileRegistry rankProfileRegistry,
+ boolean documentsOnly) {
this(stream);
this.deployLogger = deployLogger;
this.app = applicationPackage;
this.rankProfileRegistry = rankProfileRegistry;
+ this.documentsOnly = documentsOnly;
}
/**
@@ -1804,6 +1815,7 @@ void rankingConstant(Search search) :
}
lbrace() (rankingConstantItem(constant) (<NL>)*)+ <RBRACE> )
{
+ if (documentsOnly) return;
search.rankingConstants().add(constant);
}
}
@@ -1859,7 +1871,10 @@ void rankProfile(Search search) :
{
( <RANKPROFILE> name = identifier()
{
- if ("default".equals(name)) {
+ if (documentsOnly) {
+ profile = new DocumentsOnlyRankProfile(name, search, rankProfileRegistry);
+ }
+ else if ("default".equals(name)) {
profile = rankProfileRegistry.get(search, "default");
} else {
profile = new RankProfile(name, search, rankProfileRegistry);
@@ -1868,6 +1883,7 @@ void rankProfile(Search search) :
[inheritsRankProfile(profile)]
lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> )
{
+ if (documentsOnly) return;
rankProfileRegistry.add(profile);
}
}
@@ -1885,7 +1901,7 @@ Object rankProfileItem(RankProfile profile) : { }
| fieldRankFilter(profile)
| firstPhase(profile)
| matchPhase(profile)
- | macro(profile)
+ | function(profile)
| ignoreRankFeatures(profile)
| numThreadsPerSearch(profile)
| minHitsPerThread(profile)
@@ -1910,7 +1926,8 @@ void inheritsRankProfile(RankProfile profile) :
String str;
}
{
- <INHERITS> str = identifier() { profile.setInherited(str); }
+ <INHERITS> str = identifier()
+ { profile.setInherited(str); }
}
/**
@@ -1918,19 +1935,20 @@ void inheritsRankProfile(RankProfile profile) :
*
* @param profile The profile to modify.
*/
-void macro(RankProfile profile) :
+void function(RankProfile profile) :
{
- String macro, param, expr;
+ String name, expression, parameter;
+ List parameters = new ArrayList();
boolean inline = false;
}
{
- ( <MACRO> inline = inline() macro = identifier() [ "$" { macro = macro + token.image; } ]
- "(" { profile.addMacro(macro, inline); }
- [ param = identifier() { profile.getMacros().get(macro).addParam(param); }
- ( <COMMA> param = identifier() { profile.getMacros().get(macro).addParam(param); } )* ]
+ ( ( <FUNCTION> | <MACRO> ) inline = inline() name = identifier() [ "$" { name = name + token.image; } ]
+ "("
+ [ parameter = identifier() { parameters.add(parameter); }
+ ( <COMMA> parameter = identifier() { parameters.add(parameter); } )* ]
")"
- lbrace() expr = expression() (<NL>)* <RBRACE> )
- { profile.getMacros().get(macro).setTextualExpression(expr); }
+ lbrace() expression = expression() (<NL>)* <RBRACE> )
+ { profile.addFunction(name, parameters, expression, inline); }
}
boolean inline() :
@@ -2034,7 +2052,7 @@ Object firstPhaseItem(RankProfile profile) :
double dropLimit;
}
{
- ( expression = expression() { profile.setFirstPhaseRankingString(expression); }
+ ( expression = expression() { profile.setFirstPhaseRanking(expression); }
| (<KEEPRANKCOUNT> <COLON> rerankCount = integer()) { profile.setKeepRankCount(rerankCount); }
| (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); }
)
@@ -2063,7 +2081,7 @@ Object secondPhaseItem(RankProfile profile) :
int rerankCount;
}
{
- ( expression = expression() { profile.setSecondPhaseRankingString(expression); }
+ ( expression = expression() { profile.setSecondPhaseRanking(expression); }
| (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); }
)
{ return null; }
diff --git a/config-model/src/test/derived/gemini2/gemini.sd b/config-model/src/test/derived/gemini2/gemini.sd
index 18be346a758..01e20c1b30a 100644
--- a/config-model/src/test/derived/gemini2/gemini.sd
+++ b/config-model/src/test/derived/gemini2/gemini.sd
@@ -6,19 +6,19 @@ search gemini {
rank-profile test {
- macro wrapper2(x) {
+ function wrapper2(x) {
expression: x
}
- macro wrapper1(x) {
+ function wrapper1(x) {
expression: wrapper2(x)
}
- macro toplevel() {
+ function toplevel() {
expression: wrapper1(attribute(right))
}
- macro interfering() {
+ function interfering() {
expression: wrapper1(attribute(wrong))
}
diff --git a/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd b/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd
index 7f09095c5e7..6e399c03a2c 100644
--- a/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd
+++ b/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd
@@ -22,11 +22,11 @@ search rankexpression {
}
rank-profile macros {
- macro titlematch$(var1, var2) {
+ function titlematch$(var1, var2) {
expression: file: titlematch
}
- macro artistmatch() {
+ function artistmatch() {
expression: 78+closeness(distance)
}
diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd
index 96b0fa98098..0435ea439df 100644
--- a/config-model/src/test/examples/simple.sd
+++ b/config-model/src/test/examples/simple.sd
@@ -121,7 +121,7 @@ search simple {
second-phase {
rerank-count: 99
}
- macro openTicket() {
+ function openTicket() {
expression: if(attribute(status) == "accepted",1, if(attribute(status) == "new",1,if(attribute(status) == "reopened",1,0)))
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
index 03fa92f5cb9..07a36832094 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
@@ -5,10 +5,12 @@ import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.derived.DerivedConfiguration;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels;
+import com.yahoo.yolean.Exceptions;
import org.junit.Test;
import java.io.IOException;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -27,9 +29,9 @@ public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionT
new DerivedConfiguration(search, registry, new QueryProfileRegistry(), new ImportedModels()); // cause rank profile parsing
fail("parsing should have failed");
} catch (IllegalArgumentException e) {
- e.printStackTrace();
- assertTrue(e.getCause().getMessage().contains("Could not read ranking expression file"));
- assertTrue(e.getCause().getMessage().contains("wrongending.expr.expression"));
+ String message = Exceptions.toMessageString(e);
+ assertTrue(message.contains("Could not read ranking expression file"));
+ assertTrue(message.contains("wrongending.expr.expression"));
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java
index 28559f351ac..02ec597c3ed 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java
@@ -4,6 +4,8 @@ package com.yahoo.searchdefinition;
import com.yahoo.config.model.application.provider.FilesApplicationPackage;
import com.yahoo.config.model.test.TestDriver;
import com.yahoo.config.model.test.TestRoot;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import org.junit.Test;
@@ -17,6 +19,7 @@ import static org.junit.Assert.assertNull;
* @author Ulf Lilleengen
*/
public class RankProfileRegistryTest {
+
private static final String TESTDIR = "src/test/cfg/search/data/v2/inherited_rankprofiles";
@Test
@@ -43,11 +46,11 @@ public class RankProfileRegistryTest {
RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search);
for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) {
- assertNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo"));
- RankProfile rankProfileWithAddedMacro = new RankProfile(rankProfileName, search, rankProfileRegistry);
- rankProfileWithAddedMacro.addMacro("foo", true);
- rankProfileRegistry.add(rankProfileWithAddedMacro);
- assertNotNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo"));
+ assertNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo"));
+ RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, search, rankProfileRegistry);
+ rankProfileWithAddedFunction.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true);
+ rankProfileRegistry.add(rankProfileWithAddedFunction);
+ assertNotNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo"));
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
index a524a26cbef..150469cc928 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
@@ -59,7 +59,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
" constants {\n" +
" p2: 2.0 \n" +
" }\n" +
- " macro foo() {\n" +
+ " function foo() {\n" +
" expression: p2*p1\n" +
" }\n" +
" }\n" +
@@ -76,7 +76,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedModels());
assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString());
- assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString());
+ assertEquals("foo: 14.0", child2.getFunctions().get("foo").function().getBody().toString());
List<Pair<String, String>> rankProperties = new RawRankProfile(child2,
queryProfileRegistry,
new ImportedModels(),
@@ -101,7 +101,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
" constants {\n" +
" c: 7 \n" +
" }\n" +
- " macro c() {\n" +
+ " function c() {\n" +
" expression: p2*p1\n" +
" }\n" +
" }\n" +
@@ -114,7 +114,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
fail("Should have caused an exception");
}
catch (IllegalArgumentException e) {
- assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and macro named 'c'",
+ assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and function named 'c'",
Exceptions.toMessageString(e));
}
}
@@ -132,7 +132,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
" }\n" +
" \n" +
" rank-profile test {\n" +
- " macro POP_SLOW_SCORE() {\n" +
+ " function POP_SLOW_SCORE() {\n" +
" expression: safeLog(popShareSlowDecaySignal, -9.21034037)\n" +
" }\n" +
" }\n" +
@@ -141,8 +141,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
builder.build();
Search s = builder.getSearch();
RankProfile profile = rankProfileRegistry.get(s, "test");
- profile.parseExpressions(); // TODO: Do differently
- assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString());
+ assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString());
}
@Test
@@ -161,7 +160,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
" constants {\n" +
" myValue: -9.21034037\n" +
" }\n" +
- " macro POP_SLOW_SCORE() {\n" +
+ " function POP_SLOW_SCORE() {\n" +
" expression: safeLog(popShareSlowDecaySignal, myValue)\n" +
" }\n" +
" }\n" +
@@ -170,14 +169,13 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
builder.build();
Search s = builder.getSearch();
RankProfile profile = rankProfileRegistry.get(s, "test");
- profile.parseExpressions(); // TODO: Do differently
- assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString());
+ assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString());
assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)",
- profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString());
+ profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString());
}
@Test
- public void testConstantDivisorInMacro() throws ParseException {
+ public void testConstantDivisorInFunction() throws ParseException {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
builder.importString(
@@ -186,7 +184,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
" }\n" +
" \n" +
" rank-profile test {\n" +
- " macro rank_default(){\n" +
+ " function rank_default(){\n" +
" expression: k1 + (k2 + k3) / 100000000.0\n\n" +
" }\n" +
" }\n" +
@@ -196,7 +194,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
Search s = builder.getSearch();
RankProfile profile = rankProfileRegistry.get(s, "test");
assertEquals("k1 + (k2 + k3) / 100000000.0",
- profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString());
+ profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString());
}
@Test
@@ -212,7 +210,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
" }\n" +
" \n" +
" rank-profile test {\n" +
- " macro rank_default(){\n" +
+ " function rank_default(){\n" +
" expression: 0.5+50*(attribute(rating_yelp)-3)\n\n" +
" }\n" +
" }\n" +
@@ -222,7 +220,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
Search s = builder.getSearch();
RankProfile profile = rankProfileRegistry.get(s, "test");
assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)",
- profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString());
+ profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString());
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
index b13ffabda77..e507a6c48e4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.fail;
public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase {
@Test
- public void testMacroInliningPreserveArithemticOrdering() throws ParseException {
+ public void testFunctionInliningPreserveArithemticOrdering() throws ParseException {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
builder.importString(
@@ -44,18 +44,18 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase
" first-phase {\n" +
" expression: p1 * add\n" +
" }\n" +
- " macro inline add() {\n" +
+ " function inline add() {\n" +
" expression: 3 + attribute(a) + attribute(b) * mul3\n" +
" }\n" +
- " macro inline mul3() {\n" +
+ " function inline mul3() {\n" +
" expression: attribute(a) * 3 + singleif\n" +
" }\n" +
- " macro inline singleif() {\n" +
+ " function inline singleif() {\n" +
" expression: if (p1 < attribute(a), 1, 2) == 0\n" +
" }\n" +
" }\n" +
" rank-profile child inherits parent {\n" +
- " macro inline add() {\n" +
+ " function inline add() {\n" +
" expression: 9 + attribute(a)\n" +
" }\n" +
" }\n" +
@@ -95,7 +95,7 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase
" second-phase {\n" +
" expression: p2 * foo\n" +
" }\n" +
- " macro inline foo() {\n" +
+ " function inline foo() {\n" +
" expression: 3 + p1 + p2\n" +
" }\n" +
" }\n" +
@@ -106,16 +106,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase
" constants {\n" +
" p2: 2.0 \n" +
" }\n" +
- " macro bar() {\n" +
+ " function bar() {\n" +
" expression: p2*p1\n" +
" }\n" +
- " macro inline baz() {\n" +
+ " function inline baz() {\n" +
" expression: p2+p1+boz\n" +
" }\n" +
- " macro inline boz() {\n" +
+ " function inline boz() {\n" +
" expression: 3.0\n" +
" }\n" +
- " macro inline arg(a1) {\n" +
+ " function inline arg(a1) {\n" +
" expression: a1*2\n" +
" }\n" +
" }\n" +
@@ -162,16 +162,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase
" first-phase {\n" +
" expression: A + C + D\n" +
" }\n" +
- " macro inline D() {\n" +
+ " function inline D() {\n" +
" expression: B + 1\n" +
" }\n" +
- " macro C() {\n" +
+ " function C() {\n" +
" expression: A + B\n" +
" }\n" +
- " macro inline B() {\n" +
+ " function inline B() {\n" +
" expression: attribute(b)\n" +
" }\n" +
- " macro inline A() {\n" +
+ " function inline A() {\n" +
" expression: attribute(a)\n" +
" }\n" +
" }\n" +
@@ -187,8 +187,8 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase
}
/**
- * Expression evaluation has no stack so macro arguments are bound at config time creating a separate version of
- * each macro for each binding, using hashes to name the bound variants of the macro.
+ * Expression evaluation has no stack so function arguments are bound at config time creating a separate version of
+ * each function for each binding, using hashes to name the bound variants of the function.
* This method censors those hashes for string comparison.
*/
private String censorBindingHash(String s) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java
index df9a40d29e2..17bebcba70e 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java
@@ -29,7 +29,7 @@ public class RankingExpressionLoopDetectionTestCase {
" first-phase {\n" +
" expression: foo\n" +
" }\n" +
- " macro foo() {\n" +
+ " function foo() {\n" +
" expression: foo\n" +
" }\n" +
" }\n" +
@@ -61,10 +61,10 @@ public class RankingExpressionLoopDetectionTestCase {
" first-phase {\n" +
" expression: foo\n" +
" }\n" +
- " macro foo() {\n" +
+ " function foo() {\n" +
" expression: arg(5)\n" +
" }\n" +
- " macro arg(a1) {\n" +
+ " function arg(a1) {\n" +
" expression: foo + a1*2\n" +
" }\n" +
" }\n" +
@@ -96,10 +96,10 @@ public class RankingExpressionLoopDetectionTestCase {
" first-phase {\n" +
" expression: foo\n" +
" }\n" +
- " macro foo() {\n" +
+ " function foo() {\n" +
" expression: arg(foo)\n" +
" }\n" +
- " macro arg(a1) {\n" +
+ " function arg(a1) {\n" +
" expression: a1*2\n" +
" }\n" +
" }\n" +
@@ -131,10 +131,10 @@ public class RankingExpressionLoopDetectionTestCase {
" first-phase {\n" +
" expression: foo(3)\n" +
" }\n" +
- " macro foo(a1) {\n" +
+ " function foo(a1) {\n" +
" expression: bar(3)\n" +
" }\n" +
- " macro bar(a1) {\n" +
+ " function bar(a1) {\n" +
" expression: a1*2\n" +
" }\n" +
" }\n" +
@@ -159,10 +159,10 @@ public class RankingExpressionLoopDetectionTestCase {
" first-phase {\n" +
" expression: foo(3)\n" +
" }\n" +
- " macro foo(a1) {\n" +
+ " function foo(a1) {\n" +
" expression: bar(3) + bar(a1)\n" +
" }\n" +
- " macro bar(a1) {\n" +
+ " function bar(a1) {\n" +
" expression: a1*2\n" +
" }\n" +
" }\n" +
@@ -183,10 +183,10 @@ public class RankingExpressionLoopDetectionTestCase {
" first-phase {\n" +
" expression: foo(bar(2))\n" +
" }\n" +
- " macro foo(x) {\n" +
+ " function foo(x) {\n" +
" expression: x * x\n" +
" }\n" +
- " macro bar(x) {\n" +
+ " function bar(x) {\n" +
" expression: x + x\n" +
" }\n" +
" }\n" +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
index 1ece2355a92..e15d4075b19 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
@@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals;
public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase {
@Test
- public void testBasicMacroShadowing() throws ParseException {
+ public void testBasicFunctionShadowing() throws ParseException {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
builder.importString(
@@ -31,7 +31,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
" }\n" +
" \n" +
" rank-profile test {\n" +
- " macro sin(x) {\n" +
+ " function sin(x) {\n" +
" expression: x * x\n" +
" }\n" +
" first-phase {\n" +
@@ -57,7 +57,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
@Test
- public void testMultiLevelMacroShadowing() throws ParseException {
+ public void testMultiLevelFunctionShadowing() throws ParseException {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
builder.importString(
@@ -69,13 +69,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
" }\n" +
" \n" +
" rank-profile test {\n" +
- " macro tan(x) {\n" +
+ " function tan(x) {\n" +
" expression: x * x\n" +
" }\n" +
- " macro cos(x) {\n" +
+ " function cos(x) {\n" +
" expression: tan(x)\n" +
" }\n" +
- " macro sin(x) {\n" +
+ " function sin(x) {\n" +
" expression: cos(x)\n" +
" }\n" +
" first-phase {\n" +
@@ -113,7 +113,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
@Test
- public void testMacroShadowingArguments() throws ParseException {
+ public void testFunctionShadowingArguments() throws ParseException {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
builder.importString(
@@ -125,7 +125,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
" }\n" +
" \n" +
" rank-profile test {\n" +
- " macro sin(x) {\n" +
+ " function sin(x) {\n" +
" expression: x * x\n" +
" }\n" +
" first-phase {\n" +
@@ -168,13 +168,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
" }\n" +
" \n" +
" rank-profile test {\n" +
- " macro relu(x) {\n" + // relu is a built in function, redefined here
+ " function relu(x) {\n" + // relu is a built in function, redefined here
" expression: max(1.0, x)\n" +
" }\n" +
- " macro hidden_layer() {\n" +
+ " function hidden_layer() {\n" +
" expression: relu(sum(query(q) * constant(W_hidden), input) + constant(b_input))\n" +
" }\n" +
- " macro final_layer() {\n" +
+ " function final_layer() {\n" +
" expression: sigmoid(sum(hidden_layer * constant(W_final), hidden) + constant(b_final))\n" +
" }\n" +
" second-phase {\n" +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
index 3fe3a7c3de1..5e649c2e551 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
@@ -5,9 +5,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.derived.DerivedConfiguration;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels;
+import com.yahoo.yolean.Exceptions;
import org.junit.Ignore;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
@@ -29,9 +31,7 @@ public class RankingExpressionValidationTestCase extends SearchDefinitionTestCas
fail("No exception on incorrect ranking expression " + expression);
} catch (IllegalArgumentException e) {
// Success
- // TODO: Where's the "com.yahoo.searchdefinition.parser.ParseException:" nonsense coming from?
- assertTrue("Got unexpected error message: " + e.getCause().getMessage(),
- e.getCause().getMessage().startsWith("com.yahoo.searchdefinition.parser.ParseException: Could not parse ranking expression '" + expression + "'"));
+ assertTrue(Exceptions.toMessageString(e).startsWith("Illegal first phase ranking function: Could not parse ranking expression '" + expression + "' in default, firstphase.:"));
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
index 8cdfdd51637..82c03c02f61 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
@@ -34,20 +34,19 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase {
assertEquals("simple",search.getName());
assertTrue(search.hasDocument());
- SDDocumentType document=search.getDocument();
- assertEquals("simple",document.getName());
- assertEquals(12,document.getFieldCount());
+ SDDocumentType document = search.getDocument();
+ assertEquals("simple", document.getName());
+ assertEquals(12, document.getFieldCount());
SDField field;
Attribute attribute;
- new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true);
+ new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false);
// First field
field=(SDField) document.getField("title");
assertEquals(DataType.STRING,field.getDataType());
- assertEquals("{ summary | index; }",
- field.getIndexingScript().toString());
+ assertEquals("{ summary | index; }", field.getIndexingScript().toString());
assertTrue(!search.getIndex("default").isPrefix());
assertTrue(search.getIndex("title").isPrefix());
Iterator<String> titleAliases=search.getIndex("title").aliasIterator();
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
index 3ef8591d203..f87a26c6f79 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java
@@ -31,7 +31,7 @@ public class IdTestCase extends AbstractExportingTestCase {
uri.parseIndexingScript("{ summary | index }");
document.addField(uri);
- new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true);
+ new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false);
assertNull(document.getField("uri"));
assertNull(document.getField("Uri"));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java
index fad80164135..2bae285301c 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java
@@ -41,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase {
rankProfileRegistry.add(other);
other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333));
- new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true);
+ new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false);
DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels());
// Check attribute fields
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
index 9d19d4b154c..ba19a8312f6 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
@@ -69,14 +69,14 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase {
assertTrue(!transforms.hasNext());
}
@Test
- public void testPositionDeriving() throws IOException, ParseException {
+ public void testPositionDeriving() {
Search search = new Search("store", null);
SDDocumentType document = new SDDocumentType("store");
search.addDocument(document);
String fieldName = "location";
SDField field = document.addField(fieldName, PositionDataType.INSTANCE);
field.parseIndexingScript("{ attribute | summary }");
- new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true);
+ new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false);
SummaryMap summaryMap = new SummaryMap(search, new Summaries(search, new BaseDeployLogger()));
Iterator transforms = summaryMap.resultTransformIterator();
@@ -141,7 +141,7 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase {
}
@Test
- public void testFailOnSummaryFieldSourceCollision() throws IOException, ParseException {
+ public void testFailOnSummaryFieldSourceCollision() {
try {
Search search = SearchBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd");
} catch (Exception e) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
index 17cc9aaae4c..8941b07101d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
@@ -33,7 +33,7 @@ public class TypeConversionTestCase extends SearchDefinitionTestCase {
a.parseIndexingScript("{ index }");
document.addField(a);
- new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true);
+ new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false);
DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels());
IndexInfo indexInfo = derived.getIndexInfo();
assertFalse(indexInfo.hasCommand("default", "compact-to-term"));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
index a0556a156b5..48adc0eefc5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java
@@ -36,7 +36,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest {
AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields(
search,null,null,null);
- processor.process(true);
+ processor.process(true, false);
SummaryField summaryField = search.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME);
SummaryTransform actualTransform = summaryField.getTransform();
assertEquals(SummaryTransform.ATTRIBUTE, actualTransform);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java
index 8f1cc9c3de8..3a0fedfd550 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java
@@ -24,7 +24,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase {
public void testInvalidAttributeProperties() throws IOException, ParseException {
try {
Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties1.sd");
- new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true);
+ new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false);
fail("attribute property should not be set");
} catch (RuntimeException e) {
// empty
@@ -34,7 +34,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase {
@Test
public void testValidAttributeProperties() throws IOException, ParseException {
Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties2.sd");
- new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true);
+ new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false);
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
index 4dee939e3da..1ab8b054cb7 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
@@ -24,7 +24,7 @@ public class BoldingTestCase extends SearchDefinitionTestCase {
public void testBoldingNonString() throws IOException, ParseException {
try {
Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/processing/boldnonstring.sd");
- new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true);
+ new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false);
fail();
} catch (IllegalArgumentException e) {
assertTrue(e.getMessage().contains("'bolding: on' for non-text field"));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
index 7230f176048..de08bf66548 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
@@ -188,7 +188,7 @@ public class ImportedFieldsResolverTestCase {
private static ImportedFields resolve(Search search) {
assertNotNull(search.temporaryImportedFields().get());
assertFalse(search.importedFields().isPresent());
- new ImportedFieldsResolver(search, null, null, null).process(true);
+ new ImportedFieldsResolver(search, null, null, null).process(true, false);
assertFalse(search.temporaryImportedFields().isPresent());
assertNotNull(search.importedFields().get());
return search.importedFields().get();
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
index 2af3477fa62..e078d91f248 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
@@ -155,7 +155,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase {
sdoc.addField(unprocessedField);
Search search = new Search("test", null);
search.addDocument(sdoc);
- new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true);
+ new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false);
return unprocessedField.getIndexingScript();
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
index 1e0270f293d..29bba224f46 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
@@ -19,11 +19,12 @@ import static org.junit.Assert.assertTrue;
* @author baldersheim
*/
public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase {
+
@Test
public void testIntegerIndex2Attribute() throws IOException, ParseException {
Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/integerindex2attribute.sd");
search.process();
- new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true);
+ new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false);
SDField f;
f = search.getConcreteField("s1");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
index f67c85e2881..cff9abb08ed 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
@@ -72,9 +72,9 @@ class RankProfileSearchFixture {
assertEquals(expValue, rankPropertyList.get(0).getValue());
}
- public void assertMacro(String expexctedExpression, String macroName, String rankProfile) {
+ public void assertFunction(String expexctedExpression, String functionName, String rankProfile) {
assertEquals(expexctedExpression,
- compiledRankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString());
+ compiledRankProfile(rankProfile).getFunctions().get(functionName).function().getBody().getRoot().toString());
}
public RankProfile compileRankProfile(String rankProfile) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java
index d8eb4368b57..0d8cbbf2e6a 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java
@@ -109,7 +109,7 @@ public class RankingExpressionTypeValidatorTestCase {
}
@Test
- public void testMacroInvocationTypes() throws Exception {
+ public void testFunctionInvocationTypes() throws Exception {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
builder.importString(joinLines(
@@ -123,7 +123,7 @@ public class RankingExpressionTypeValidatorTestCase {
" }",
" }",
" rank-profile my_rank_profile {",
- " macro macro1(attribute_to_use) {",
+ " function macro1(attribute_to_use) {",
" expression: attribute(attribute_to_use)",
" }",
" summary-features {",
@@ -143,7 +143,7 @@ public class RankingExpressionTypeValidatorTestCase {
}
@Test
- public void testTensorMacroInvocationTypes_Nested() throws Exception {
+ public void testTensorFunctionInvocationTypes_Nested() throws Exception {
SearchBuilder builder = new SearchBuilder();
builder.importString(joinLines(
"search test {",
@@ -156,16 +156,16 @@ public class RankingExpressionTypeValidatorTestCase {
" }",
" }",
" rank-profile my_rank_profile {",
- " macro return_a() {",
+ " function return_a() {",
" expression: return_first(attribute(a), attribute(b))",
" }",
- " macro return_b() {",
+ " function return_b() {",
" expression: return_second(attribute(a), attribute(b))",
" }",
- " macro return_first(e1, e2) {",
+ " function return_first(e1, e2) {",
" expression: e1",
" }",
- " macro return_second(e1, e2) {",
+ " function return_second(e1, e2) {",
" expression: return_first(e2, e1)",
" }",
" summary-features {",
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
index b046d60f948..8944409e1e9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
@@ -128,7 +128,7 @@ public class RankingExpressionWithOnnxTestCase {
}
@Test
- public void testOnnxReferenceMissingMacro() throws ParseException {
+ public void testOnnxReferenceMissingFunction() throws ParseException {
try {
RankProfileSearchFixture search = new RankProfileSearchFixture(
new StoringApplicationPackage(applicationDir),
@@ -145,14 +145,14 @@ public class RankingExpressionWithOnnxTestCase {
catch (IllegalArgumentException expected) {
assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " +
"onnx('mnist_softmax.onnx'): " +
- "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " +
+ "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " +
"not present in rank profile 'my_profile'",
Exceptions.toMessageString(expected));
}
}
@Test
- public void testOnnxReferenceWithWrongMacroType() {
+ public void testOnnxReferenceWithWrongFunctionType() {
try {
RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)",
"onnx('mnist_softmax.onnx')");
@@ -163,7 +163,7 @@ public class RankingExpressionWithOnnxTestCase {
assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " +
"onnx('mnist_softmax.onnx'): " +
"Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " +
- "but this macro returns tensor(d0[2],d5[10])",
+ "but this function returns tensor(d0[2],d5[10])",
Exceptions.toMessageString(expected));
}
}
@@ -213,13 +213,13 @@ public class RankingExpressionWithOnnxTestCase {
}
@Test
- public void testImportingFromStoredExpressionsWithMacroOverridingConstant() throws IOException {
+ public void testImportingFromStoredExpressionsWithFunctionOverridingConstant() throws IOException {
String rankProfile =
" rank-profile my_profile {\n" +
- " macro Placeholder() {\n" +
+ " function Placeholder() {\n" +
" expression: tensor(d0[2],d1[784])(0.0)\n" +
" }\n" +
- " macro " + name + "_Variable() {\n" +
+ " function " + name + "_Variable() {\n" +
" expression: tensor(d1[10],d2[784])(0.0)\n" +
" }\n" +
" first-phase {\n" +
@@ -234,7 +234,7 @@ public class RankingExpressionWithOnnxTestCase {
search.compileRankProfile("my_profile", applicationDir.append("models"));
search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
- assertNull("Constant overridden by macro is not added",
+ assertNull("Constant overridden by function is not added",
search.search().rankingConstants().get( name + "_Variable"));
// At this point the expression is stored - copy application to another location which do not have a models dir
@@ -247,7 +247,7 @@ public class RankingExpressionWithOnnxTestCase {
RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication);
searchFromStored.compileRankProfile("my_profile", applicationDir.append("models"));
searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
- assertNull("Constant overridden by macro is not added",
+ assertNull("Constant overridden by function is not added",
searchFromStored.search().rankingConstants().get( name + "_Variable"));
} finally {
IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile());
@@ -275,19 +275,19 @@ public class RankingExpressionWithOnnxTestCase {
}
}
- private RankProfileSearchFixture fixtureWith(String macroExpression,
+ private RankProfileSearchFixture fixtureWith(String functionExpression,
String firstPhaseExpression,
String constant,
String field,
- String macroName,
+ String functionName,
StoringApplicationPackage application) {
try {
RankProfileSearchFixture fixture = new RankProfileSearchFixture(
application,
application.getQueryProfiles(),
" rank-profile my_profile {\n" +
- " macro " + macroName + "() {\n" +
- " expression: " + macroExpression +
+ " function " + functionName + "() {\n" +
+ " expression: " + functionExpression +
" }\n" +
" first-phase {\n" +
" expression: " + firstPhaseExpression +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
index 14632a568ea..cba931e81f0 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
@@ -160,7 +160,7 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testTensorFlowReferenceMissingMacro() throws ParseException {
+ public void testTensorFlowReferenceMissingFunction() throws ParseException {
try {
RankProfileSearchFixture search = new RankProfileSearchFixture(
new StoringApplicationPackage(applicationDir),
@@ -177,14 +177,14 @@ public class RankingExpressionWithTensorFlowTestCase {
catch (IllegalArgumentException expected) {
assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " +
"tensorflow('mnist_softmax/saved'): " +
- "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " +
+ "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " +
"not present in rank profile 'my_profile'",
Exceptions.toMessageString(expected));
}
}
@Test
- public void testTensorFlowReferenceWithWrongMacroType() {
+ public void testTensorFlowReferenceWithWrongFunctionType() {
try {
RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)",
"tensorflow('mnist_softmax/saved')");
@@ -195,7 +195,7 @@ public class RankingExpressionWithTensorFlowTestCase {
assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " +
"tensorflow('mnist_softmax/saved'): " +
"Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " +
- "but this macro returns tensor(d0[2],d5[10])",
+ "but this function returns tensor(d0[2],d5[10])",
Exceptions.toMessageString(expected));
}
}
@@ -261,13 +261,13 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testImportingFromStoredExpressionsWithMacroOverridingConstantAndInheritance() throws IOException {
+ public void testImportingFromStoredExpressionsWithFunctionOverridingConstantAndInheritance() throws IOException {
String rankProfiles =
" rank-profile my_profile {\n" +
- " macro Placeholder() {\n" +
+ " function Placeholder() {\n" +
" expression: tensor(d0[2],d1[784])(0.0)\n" +
" }\n" +
- " macro " + name + "_layer_Variable_read() {\n" +
+ " function " + name + "_layer_Variable_read() {\n" +
" expression: tensor(d1[10],d2[784])(0.0)\n" +
" }\n" +
" first-phase {\n" +
@@ -285,7 +285,7 @@ public class RankingExpressionWithTensorFlowTestCase {
search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child");
- assertNull("Constant overridden by macro is not added",
+ assertNull("Constant overridden by function is not added",
search.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read"));
// At this point the expression is stored - copy application to another location which do not have a models dir
@@ -300,7 +300,7 @@ public class RankingExpressionWithTensorFlowTestCase {
searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models"));
searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child");
- assertNull("Constant overridden by macro is not added",
+ assertNull("Constant overridden by function is not added",
searchFromStored.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read"));
}
finally {
@@ -317,11 +317,11 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testMacroGeneration() {
+ public void testFunctionGeneration() {
final String name = "mnist_saved";
- final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))";
- final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))";
- final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))";
+ final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))";
+ final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))";
+ final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))";
RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)",
"tensorflow('mnist/saved')",
@@ -330,8 +330,8 @@ public class RankingExpressionWithTensorFlowTestCase {
"input",
new StoringApplicationPackage(applicationDir));
search.assertFirstPhaseExpression(expression, "my_profile");
- search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile");
- search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile");
+ search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile");
+ search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile");
}
@Test
@@ -339,7 +339,7 @@ public class RankingExpressionWithTensorFlowTestCase {
final String name = "mnist_saved";
final String rankProfiles =
" rank-profile my_profile {\n" +
- " macro input() {\n" +
+ " function input() {\n" +
" expression: tensor(d0[1],d1[784])(0.0)\n" +
" }\n" +
" first-phase {\n" +
@@ -349,9 +349,9 @@ public class RankingExpressionWithTensorFlowTestCase {
" rank-profile my_profile_child inherits my_profile {\n" +
" }";
- final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))";
- final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))";
- final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))";
+ final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))";
+ final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))";
+ final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))";
RankProfileSearchFixture search = fixtureWithUncompiled(rankProfiles, new StoringApplicationPackage(applicationDir));
search.compileRankProfile("my_profile", applicationDir.append("models"));
@@ -359,10 +359,10 @@ public class RankingExpressionWithTensorFlowTestCase {
search.assertFirstPhaseExpression(expression, "my_profile");
search.assertFirstPhaseExpression(expression, "my_profile_child");
assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search);
- search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile");
- search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child");
- search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile");
- search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child");
+ search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile");
+ search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child");
+ search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile");
+ search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child");
// At this point the expression is stored - copy application to another location which do not have a models dir
Path storedApplicationDirectory = applicationDir.getParentPath().append("copy");
@@ -377,10 +377,10 @@ public class RankingExpressionWithTensorFlowTestCase {
searchFromStored.assertFirstPhaseExpression(expression, "my_profile");
searchFromStored.assertFirstPhaseExpression(expression, "my_profile_child");
assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search);
- searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile");
- searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child");
- searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile");
- searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child");
+ searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile");
+ searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child");
+ searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile");
+ searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child");
}
finally {
IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile());
@@ -429,19 +429,19 @@ public class RankingExpressionWithTensorFlowTestCase {
new StoringApplicationPackage(applicationDir));
}
- private RankProfileSearchFixture fixtureWith(String macroExpression,
+ private RankProfileSearchFixture fixtureWith(String functionExpression,
String firstPhaseExpression,
String constant,
String field,
- String macroName,
+ String functionName,
StoringApplicationPackage application) {
try {
RankProfileSearchFixture fixture = new RankProfileSearchFixture(
application,
application.getQueryProfiles(),
" rank-profile my_profile {\n" +
- " macro " + macroName + "() {\n" +
- " expression: " + macroExpression +
+ " function " + functionName + "() {\n" +
+ " expression: " + functionExpression +
" }\n" +
" first-phase {\n" +
" expression: " + firstPhaseExpression +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
index 0866d3192cf..48f2bf43e81 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
@@ -95,10 +95,10 @@ public class RankingExpressionWithTensorTestCase {
}
@Test
- public void requireThatConstantTensorsCanBeUsedInMacro() throws ParseException {
+ public void requireThatConstantTensorsCanBeUsedInFunction() throws ParseException {
RankProfileSearchFixture f = new RankProfileSearchFixture(
" rank-profile my_profile {\n" +
- " macro my_macro() {\n" +
+ " function my_macro() {\n" +
" expression: sum(my_tensor)\n" +
" }\n" +
" first-phase {\n" +
@@ -112,7 +112,7 @@ public class RankingExpressionWithTensorTestCase {
" }");
f.compileRankProfile("my_profile");
f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile");
- f.assertMacro("reduce(constant(my_tensor), sum)", "my_macro", "my_profile");
+ f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile");
f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile");
f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile");
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
index 86127a260c5..fd048737b43 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
@@ -20,24 +20,24 @@ import static org.junit.Assert.assertEquals;
public class RankingExpressionsTestCase extends SearchDefinitionTestCase {
@Test
- public void testMacros() throws IOException, ParseException {
+ public void testFunctions() throws IOException, ParseException {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction",
rankProfileRegistry,
new QueryProfileRegistry()).getSearch();
- final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros");
- macrosRankProfile.parseExpressions();
- final Map<String, RankProfile.Macro> macros = macrosRankProfile.getMacros();
- assertEquals(2, macros.get("titlematch$").getFormalParams().size());
- assertEquals("var1", macros.get("titlematch$").getFormalParams().get(0));
- assertEquals("var2", macros.get("titlematch$").getFormalParams().get(1));
- assertEquals("var1 * var2 + 890", macros.get("titlematch$").getTextualExpression().trim());
- assertEquals("var1 * var2 + 890", macros.get("titlematch$").getRankingExpression().getRoot().toString());
- assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim());
- assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim());
- assertEquals(0, macros.get("artistmatch").getFormalParams().size());
+ RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros");
+ Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions();
+ assertEquals(2, functions.get("titlematch$").function().arguments().size());
+ assertEquals("var1", functions.get("titlematch$").function().arguments().get(0));
+ assertEquals("var2", functions.get("titlematch$").function().arguments().get(1));
+ assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString());
+ assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)",
+ functionsRankProfile.getFirstPhaseRanking().getRoot().toString());
+ assertEquals("78 + closeness(distance)",
+ functions.get("artistmatch").function().getBody().getRoot().toString());
+ assertEquals(0, functions.get("artistmatch").function().arguments().size());
- List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile,
+ List<Pair<String, String>> rankProperties = new RawRankProfile(functionsRankProfile,
new QueryProfileRegistry(),
new ImportedModels(),
new AttributeFields(search)).configProperties();
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java
index 8a07e99101c..b39c48b67bf 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java
@@ -16,10 +16,10 @@ import static org.junit.Assert.assertTrue;
/**
* @author lesters
*/
-public class ReservedMacroNamesTestCase {
+public class ReservedRankingExpressionFunctionNamesTestCase {
@Test
- public void requireThatMacrosWithReservedNamesIssueAWarning() throws ParseException {
+ public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException {
TestDeployLogger deployLogger = new TestDeployLogger();
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
@@ -32,10 +32,10 @@ public class ReservedMacroNamesTestCase {
" }\n" +
" \n" +
" rank-profile test_rank_profile {\n" +
- " macro not_a_reserved_name(x) {\n" +
+ " function not_a_reserved_name(x) {\n" +
" expression: x + x\n" +
" }\n" +
- " macro sigmoid(x) {\n" +
+ " function sigmoid(x) {\n" +
" expression: x * x\n" +
" }\n" +
" first-phase {\n" +
@@ -43,7 +43,7 @@ public class ReservedMacroNamesTestCase {
" }\n" +
" }\n" +
" rank-profile test_rank_profile_2 inherits test_rank_profile {\n" +
- " macro sin(x) {\n" +
+ " function sin(x) {\n" +
" expression: x * x\n" +
" }\n" +
" first-phase {\n" +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
index 85bee61c1b4..d0c1bf8b0ca 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
@@ -12,6 +12,7 @@ import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase {
@@ -20,41 +21,44 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTe
Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidsummarysource.sd");
search.process();
try {
- new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true);
- assertTrue("This should throw and never get here", false);
+ new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false);
+ fail("This should throw and never get here");
} catch (IllegalArgumentException e) {
assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage());
}
}
+
@Test
public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException {
Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidimplicitsummarysource.sd");
search.process();
try {
- new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true);
- assertTrue("This should throw and never get here", false);
+ new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false);
+ fail("This should throw and never get here");
} catch (IllegalArgumentException e) {
assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage());
}
}
+
@Test
public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException {
Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidselfreferringsummary.sd");
search.process();
try {
- new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true);
- assertTrue("This should throw and never get here", false);
+ new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false);
+ fail("This should throw and never get here");
} catch (IllegalArgumentException e) {
assertEquals("For search 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage());
}
}
+
@Test
public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException {
Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/documentidinsummary.sd");
search.process();
BaseDeployLogger deployLogger = new BaseDeployLogger();
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
- new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true);
+ new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false);
assertEquals("documentid", search.getSummary("withid").getSummaryField("w").getSingleSource());
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
index 76c50821cb9..8e721dbe503 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
@@ -113,7 +113,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase {
}
@Test
- public void requireThatMaxAndMinWithTensorsReturnedFromMacrosAreReplaced() throws ParseException {
+ public void requireThatMaxAndMinWithTensorsReturnedFromFunctionsAreReplaced() throws ParseException {
assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)",
"max(returns_tensor,x)");
assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)",
@@ -171,7 +171,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase {
" value: { {x:0}:0 }\n" +
" }\n" +
" }\n" +
- " macro base_tensor() {\n" +
+ " function base_tensor() {\n" +
" expression: constant(base_constant_tensor)\n" +
" }\n" +
" }\n" +
@@ -181,19 +181,19 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase {
" value: { {x:0}:1 }\n" +
" }\n" +
" }\n" +
- " macro returns_tensor_with_arg(arg1) {\n" +
+ " function returns_tensor_with_arg(arg1) {\n" +
" expression: 2.0 * arg1\n" +
" }\n" +
- " macro wraps_returns_tensor() {\n" +
+ " function wraps_returns_tensor() {\n" +
" expression: returns_tensor\n" +
" }\n" +
- " macro returns_tensor() {\n" +
+ " function returns_tensor() {\n" +
" expression: attribute(tensor_field_2)\n" +
" }\n" +
- " macro tensor_inheriting() {\n" +
+ " function tensor_inheriting() {\n" +
" expression: base_tensor\n" +
" }\n" +
- " macro testexpression() {\n" +
+ " function testexpression() {\n" +
" expression: " + expression + "\n" +
" }\n" +
" }\n" +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
index ab3bb113727..d0b6524a7e1 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java
@@ -40,7 +40,7 @@ public class ValidateFieldTypesTest {
exceptionRule.expectMessage(
"For search '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " +
"Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string.");
- validator.process(true);
+ validator.process(true, false);
}
private static Search createSearchWithDocument(String documentName) {
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java
index 334228333c5..ea2ce324a27 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java
@@ -6,8 +6,7 @@ import java.util.Objects;
/**
* Represents a tenant in the provision API.
*
- * @author lulf
- * @since 5.12
+ * @author Ulf Lilleengen
*/
public class TenantName implements Comparable<TenantName> {
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
index d2fca309a19..7bea2d0825a 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
@@ -81,11 +81,11 @@ public class RankProfilesConfigImporter {
referencedFunctions.put(reference.get(),
new ExpressionFunction(reference.get().serialForm(), arguments, expression));
}
- else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to macros
+ else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to functions
firstPhase = new ExpressionFunction("firstphase", new ArrayList<>(),
new RankingExpression("first-phase", property.value()));
}
- else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to macros
+ else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to functions
secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(),
new RankingExpression("second-phase", property.value()));
}
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
index c6d8f70fde8..da34ab8822d 100755
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
@@ -3,13 +3,16 @@ package com.yahoo.searchlib.rankingexpression;
import com.google.common.collect.ImmutableList;
import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
-import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext;
import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
import com.yahoo.text.Utf8;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.util.*;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* A function defined by a ranking expression
@@ -24,6 +27,16 @@ public class ExpressionFunction {
private final RankingExpression body;
/**
+ * Constructs a new function with no arguments
+ *
+ * @param name the name of this function
+ * @param body the ranking expression that defines this function
+ */
+ public ExpressionFunction(String name, RankingExpression body) {
+ this(name, Collections.emptyList(), body);
+ }
+
+ /**
* Constructs a new function
*
* @param name the name of this function
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
index ed82ba20fbe..722520fea08 100755
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
@@ -250,12 +250,12 @@ public class RankingExpression implements Serializable {
/**
* Creates the necessary rank properties required to implement this expression.
*
- * @param macros the expression macros to expand.
- * @return a list of named rank properties required to implement this expression.
+ * @param functions the expression functions to expand
+ * @return a list of named rank properties required to implement this expression
*/
- public Map<String, String> getRankProperties(List<ExpressionFunction> macros) {
+ public Map<String, String> getRankProperties(List<ExpressionFunction> functions) {
Deque<String> path = new LinkedList<>();
- SerializationContext context = new SerializationContext(macros);
+ SerializationContext context = new SerializationContext(functions);
String serializedRoot = root.toString(new StringBuilder(), context, path, null).toString();
Map<String, String> serializedExpressions = context.serializedFunctions();
serializedExpressions.put(propertyName(name), serializedRoot);
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java
index ac5eefcc5b2..282a4c5e0a9 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java
@@ -30,8 +30,8 @@ public class ImportedModel {
private final Map<String, Tensor> smallConstants = new HashMap<>();
private final Map<String, Tensor> largeConstants = new HashMap<>();
private final Map<String, RankingExpression> expressions = new HashMap<>();
- private final Map<String, RankingExpression> macros = new HashMap<>();
- private final Map<String, TensorType> requiredMacros = new HashMap<>();
+ private final Map<String, RankingExpression> functions = new HashMap<>();
+ private final Map<String, TensorType> requiredFunctions = new HashMap<>();
/**
* Creates a new imported model.
@@ -77,13 +77,13 @@ public class ImportedModel {
public Map<String, RankingExpression> expressions() { return Collections.unmodifiableMap(expressions); }
/**
- * Returns an immutable map of macros that are part of this model.
- * Note that the macros themselves are *not* copies and *not* immutable - they must be copied before modification.
+ * Returns an immutable map of the functions that are part of this model.
+ * Note that the functions themselves are *not* copies and *not* immutable - they must be copied before modification.
*/
- public Map<String, RankingExpression> macros() { return Collections.unmodifiableMap(macros); }
+ public Map<String, RankingExpression> functions() { return Collections.unmodifiableMap(functions); }
- /** Returns an immutable map of the macros that must be provided by the environment running this model */
- public Map<String, TensorType> requiredMacros() { return Collections.unmodifiableMap(requiredMacros); }
+ /** Returns an immutable map of the functions that must be provided by the environment running this model */
+ public Map<String, TensorType> requiredFunctions() { return Collections.unmodifiableMap(requiredFunctions); }
/** Returns an immutable map of the signatures of this */
public Map<String, Signature> signatures() { return Collections.unmodifiableMap(signatures); }
@@ -100,8 +100,8 @@ public class ImportedModel {
void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); }
void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); }
void expression(String name, RankingExpression expression) { expressions.put(name, expression); }
- void macro(String name, RankingExpression expression) { macros.put(name, expression); }
- void requiredMacro(String name, TensorType type) { requiredMacros.put(name, type); }
+ void function(String name, RankingExpression expression) { functions.put(name, expression); }
+ void requiredFunction(String name, TensorType type) { requiredFunctions.put(name, type); }
/**
* Returns all the output expressions of this indexed by name. The names consist of one or two parts
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java
index 2ae107a5770..d25502fd149 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java
@@ -24,7 +24,7 @@ import java.util.logging.Logger;
* ranking expressions. The general mechanism for import is for the
* specific ML platform import implementations to create an
* IntermediateGraph. This class offers common code to convert the
- * IntermediateGraph to Vespa ranking expressions and macros.
+ * IntermediateGraph to Vespa ranking expressions and functions.
*
* @author lesters
*/
@@ -122,7 +122,7 @@ public abstract class ModelImporter {
importExpressionInputs(operation, model);
importRankingExpression(operation, model);
importArgumentExpression(operation, model);
- importMacroExpression(operation, model);
+ importFunctionExpression(operation, model);
return operation.function();
}
@@ -188,15 +188,15 @@ public abstract class ModelImporter {
// All inputs must have dimensions with standard naming convention: d0, d1, ...
OrderedTensorType standardNamingConvention = OrderedTensorType.standardType(operation.type().get());
model.argument(operation.vespaName(), standardNamingConvention.type());
- model.requiredMacro(operation.vespaName(), standardNamingConvention.type());
+ model.requiredFunction(operation.vespaName(), standardNamingConvention.type());
}
}
- private static void importMacroExpression(IntermediateOperation operation, ImportedModel model) {
- if (operation.macro().isPresent()) {
- TensorFunction function = operation.macro().get();
+ private static void importFunctionExpression(IntermediateOperation operation, ImportedModel model) {
+ if (operation.rankingExpressionFunction().isPresent()) {
+ TensorFunction function = operation.rankingExpressionFunction().get();
try {
- model.macro(operation.macroName(), new RankingExpression(operation.macroName(), function.toString()));
+ model.function(operation.rankingExpressionFunctionName(), new RankingExpression(operation.rankingExpressionFunctionName(), function.toString()));
}
catch (ParseException e) {
throw new RuntimeException("Tensorflow function " + function +
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java
index 43de29cedd5..34f5f1365a1 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java
@@ -29,7 +29,7 @@ import java.util.function.Function;
*/
public abstract class IntermediateOperation {
- private final static String MACRO_PREFIX = "imported_ml_macro_";
+ private final static String FUNCTION_PREFIX = "imported_ml_function_";
protected final String name;
protected final String modelName;
@@ -38,7 +38,7 @@ public abstract class IntermediateOperation {
protected OrderedTensorType type;
protected TensorFunction function;
- protected TensorFunction macro = null;
+ protected TensorFunction rankingExpressionFunction = null;
private final List<String> importWarnings = new ArrayList<>();
private Value constantValue = null;
@@ -71,8 +71,8 @@ public abstract class IntermediateOperation {
ExpressionNode constant = new ReferenceNode(Reference.simple("constant", vespaName()));
function = new TensorFunctionNode.TensorFunctionExpressionNode(constant);
} else if (outputs.size() > 1) {
- macro = lazyGetFunction();
- function = new VariableTensor(macroName(), type.type());
+ rankingExpressionFunction = lazyGetFunction();
+ function = new VariableTensor(rankingExpressionFunctionName(), type.type());
} else {
function = lazyGetFunction();
}
@@ -86,11 +86,13 @@ public abstract class IntermediateOperation {
/** Return unmodifiable list of inputs */
public List<IntermediateOperation> inputs() { return inputs; }
- /** Return unmodifiable list of outputs. If a node has multiple outputs, consider adding a macro. */
+ /** Return unmodifiable list of outputs. If a node has multiple outputs, consider adding a function. */
public List<IntermediateOperation> outputs() { return Collections.unmodifiableList(outputs); }
- /** Returns a Vespa ranking expression that should be added as a macro */
- public Optional<TensorFunction> macro() { return Optional.ofNullable(macro); }
+ /** Returns a function that should be added as a ranking expression function */
+ public Optional<TensorFunction> rankingExpressionFunction() {
+ return Optional.ofNullable(rankingExpressionFunction);
+ }
/** Add dimension name constraints for this operation */
public void addDimensionNameConstraints(DimensionRenamer renamer) { }
@@ -131,8 +133,10 @@ public abstract class IntermediateOperation {
public String vespaName() { return vespaName(name); }
public String vespaName(String name) { return name != null ? namePartOf(name).replace('/', '_') : null; }
- /** Retrieve the valid Vespa name of this node if it is a macro */
- public String macroName() { return vespaName() != null ? MACRO_PREFIX + modelName + "_" + vespaName() : null; }
+ /** Retrieve the valid Vespa name of this node if it is a ranking expression function */
+ public String rankingExpressionFunctionName() {
+ return vespaName() != null ? FUNCTION_PREFIX + modelName + "_" + vespaName() : null;
+ }
/** Retrieve the list of warnings produced during its lifetime */
public List<String> warnings() { return Collections.unmodifiableList(importWarnings); }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java
index 9299ae9be12..b335fd7e1c5 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java
@@ -26,13 +26,13 @@ public class PlaceholderWithDefault extends IntermediateOperation {
if (!allInputFunctionsPresent(1)) {
return null;
}
- // This should be a call to the macro we add below, but for now
+ // This should be a call to the function we add below, but for now
// we treat this as as identity function and just pass the constant.
return inputs.get(0).function().orElse(null);
}
@Override
- public Optional<TensorFunction> macro() {
+ public Optional<TensorFunction> rankingExpressionFunction() {
// For now, it is much more efficient to assume we always will return
// the default value, as we can prune away large parts of the expression
// tree by having it calculated as a constant. If a case arises where
@@ -42,7 +42,7 @@ public class PlaceholderWithDefault extends IntermediateOperation {
@Override
public boolean isConstant() {
- return true; // not true if we add to macro
+ return true; // not true if we add to function
}
}
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java
index e2dc170c168..eb8d2229a6d 100755
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java
@@ -15,7 +15,7 @@ import java.util.Deque;
import java.util.List;
/**
- * A node referring either to a value in the context or to a named ranking expression (function aka macro).
+ * A node referring either to a value in the context or to a named ranking expression function.
*
* @author bratseth
*/
@@ -64,7 +64,7 @@ public final class ReferenceNode extends CompositeNode {
@Override
public StringBuilder toString(StringBuilder string, SerializationContext context, Deque<String> path, CompositeNode parent) {
- // A reference to a macro argument?
+ // A reference to a function argument?
if (reference.isIdentifier() && context.getBinding(getName()) != null) {
// a bound identifier: replace by the value it is bound to
return string.append(context.getBinding(getName()));
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
index 7c929ae24b3..571e1f4d608 100755
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
@@ -91,26 +91,26 @@ public class RankingExpressionTestCase {
@Test
public void testSelfRecursionSerialization() throws ParseException {
- List<ExpressionFunction> macros = new ArrayList<>();
- macros.add(new ExpressionFunction("foo", null, new RankingExpression("foo")));
+ List<ExpressionFunction> functions = new ArrayList<>();
+ functions.add(new ExpressionFunction("foo", null, new RankingExpression("foo")));
RankingExpression exp = new RankingExpression("foo");
try {
- exp.getRankProperties(macros);
+ exp.getRankProperties(functions);
} catch (RuntimeException e) {
assertEquals("Cycle in ranking expression function: [foo[]]", e.getMessage());
}
}
@Test
- public void testMacroCycleSerialization() throws ParseException {
- List<ExpressionFunction> macros = new ArrayList<>();
- macros.add(new ExpressionFunction("foo", null, new RankingExpression("bar")));
- macros.add(new ExpressionFunction("bar", null, new RankingExpression("foo")));
+ public void testFunctionCycleSerialization() throws ParseException {
+ List<ExpressionFunction> funnctions = new ArrayList<>();
+ funnctions.add(new ExpressionFunction("foo", null, new RankingExpression("bar")));
+ funnctions.add(new ExpressionFunction("bar", null, new RankingExpression("foo")));
RankingExpression exp = new RankingExpression("foo");
try {
- exp.getRankProperties(macros);
+ exp.getRankProperties(funnctions);
} catch (RuntimeException e) {
assertEquals("Cycle in ranking expression function: [foo[], bar[]]", e.getMessage());
}
@@ -118,11 +118,11 @@ public class RankingExpressionTestCase {
@Test
public void testSerialization() throws ParseException {
- List<ExpressionFunction> macros = new ArrayList<>();
- macros.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))")));
- macros.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2")));
- macros.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)")));
- macros.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977")));
+ List<ExpressionFunction> functions = new ArrayList<>();
+ functions.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))")));
+ functions.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2")));
+ functions.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)")));
+ functions.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977")));
assertSerialization(Arrays.asList(
"rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(foo@af74e3fd9070bd18.a368ed0a5ba3a5d0) * rankingExpression(foo@dbab346efdad5362.e5c39e42ebd91c30)",
@@ -130,19 +130,19 @@ public class RankingExpressionTestCase {
"min(6,pow(7,2))",
"min(1,pow(2,2))",
"min(3,pow(4,2))",
- "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", macros);
+ "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", functions);
assertSerialization(Arrays.asList(
"rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(bar@af74e3fd9070bd18.a368ed0a5ba3a5d0)",
"min(1,pow(2,2))",
- "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", macros);
+ "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", functions);
assertSerialization(Arrays.asList(
"rankingExpression(baz@e2dc17a89864aed0.12232eb692c6c502)",
"min(1,pow(2,2))",
"rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) / rankingExpression(bar@e2dc17a89864aed0.12232eb692c6c502)",
- "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", macros);
+ "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", functions);
assertSerialization(Arrays.asList(
"rankingExpression(cox)",
- "10 + 08 * 1977"), "cox", macros
+ "10 + 08 * 1977"), "cox", functions
);
}
@@ -159,8 +159,8 @@ public class RankingExpressionTestCase {
@Test
public void testBug3464208() throws ParseException {
- List<ExpressionFunction> macros = new ArrayList<>();
- macros.add(new ExpressionFunction("log10tweetage", null, new RankingExpression("69")));
+ List<ExpressionFunction> functions = new ArrayList<>();
+ functions.add(new ExpressionFunction("log10tweetage", null, new RankingExpression("69")));
String lhs = "log10(0.01+attribute(user_followers_count)) * log10(socialratio) * " +
"log10(userage/(0.01+attribute(user_statuses_count)))";
@@ -172,8 +172,8 @@ public class RankingExpressionTestCase {
String expRhs = "(rankingExpression(log10tweetage) * rankingExpression(log10tweetage) * " +
"rankingExpression(log10tweetage)) + 5.0 * attribute(ythl)";
- assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, macros);
- assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, macros);
+ assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, functions);
+ assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, functions);
}
@Test
@@ -295,12 +295,12 @@ public class RankingExpressionTestCase {
assertEquals(expected, new RankingExpression(expression).toString());
}
- /** Test serialization with no macros */
+ /** Test serialization with no functions */
private void assertSerialization(String expectedSerialization, String expressionString) {
String serializedExpression;
try {
RankingExpression expression = new RankingExpression(expressionString);
- // No macros -> expect one rank property
+ // No functions -> expect one rank property
serializedExpression = expression.getRankProperties(Collections.emptyList()).values().iterator().next();
assertEquals(expectedSerialization, serializedExpression);
}
@@ -309,7 +309,7 @@ public class RankingExpressionTestCase {
}
try {
- // No macros -> output should be parseable to a ranking expression
+ // No functions -> output should be parseable to a ranking expression
// (but not the same one due to primitivization)
RankingExpression reparsedExpression = new RankingExpression(serializedExpression);
// Serializing the primitivized expression should yield the same expression again
@@ -323,17 +323,17 @@ public class RankingExpressionTestCase {
}
private void assertSerialization(List<String> expectedSerialization, String expressionString,
- List<ExpressionFunction> macros) {
- assertSerialization(expectedSerialization, expressionString, macros, false);
+ List<ExpressionFunction> functions) {
+ assertSerialization(expectedSerialization, expressionString, functions, false);
}
private void assertSerialization(List<String> expectedSerialization, String expressionString,
- List<ExpressionFunction> macros, boolean print) {
+ List<ExpressionFunction> functions, boolean print) {
try {
if (print)
System.out.println("Parsing expression '" + expressionString + "'.");
RankingExpression expression = new RankingExpression(expressionString);
- Map<String, String> rankProperties = expression.getRankProperties(macros);
+ Map<String, String> rankProperties = expression.getRankProperties(functions);
if (print) {
for (String key : rankProperties.keySet())
System.out.println("Property '" + key + "': " + rankProperties.get(key));
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java
index a63c7346335..a8f7542f3a4 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java
@@ -18,11 +18,11 @@ public class DropoutImportTestCase {
public void testDropoutImport() {
TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/files/integration/tensorflow/dropout/saved");
- // Check required macros
- assertEquals(1, model.get().requiredMacros().size());
- assertTrue(model.get().requiredMacros().containsKey("X"));
+ // Check required functions
+ assertEquals(1, model.get().requiredFunctions().size());
+ assertTrue(model.get().requiredFunctions().containsKey("X"));
assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(),
- model.get().requiredMacros().get("X"));
+ model.get().requiredFunctions().get("X"));
ImportedModel.Signature signature = model.get().signature("serving_default");
@@ -32,7 +32,7 @@ public class DropoutImportTestCase {
RankingExpression output = signature.outputExpression("y");
assertNotNull(output);
assertEquals("outputs/Maximum", output.getName());
- assertEquals("join(join(imported_ml_macro_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_macro_test_outputs_BiasAdd, f(a,b)(max(a,b)))",
+ assertEquals("join(join(imported_ml_function_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_function_test_outputs_BiasAdd, f(a,b)(max(a,b)))",
output.getRoot().toString());
model.assertEqualResult("X", output.getName());
}
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java
index bcfc6ce0a04..e20ac16a691 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java
@@ -36,11 +36,11 @@ public class OnnxMnistSoftmaxImportTestCase {
constant1.type());
assertEquals(10, constant1.size());
- // Check required macros (inputs)
- assertEquals(1, model.requiredMacros().size());
- assertTrue(model.requiredMacros().containsKey("Placeholder"));
+ // Check required functions (inputs)
+ assertEquals(1, model.requiredFunctions().size());
+ assertTrue(model.requiredFunctions().containsKey("Placeholder"));
assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(),
- model.requiredMacros().get("Placeholder"));
+ model.requiredFunctions().get("Placeholder"));
// Check outputs
RankingExpression output = model.defaultSignature().outputExpression("add");
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java
index dd6c8095e3c..ef28eb4678f 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java
@@ -34,14 +34,14 @@ public class TensorFlowMnistSoftmaxImportTestCase {
constant1.type());
assertEquals(10, constant1.size());
- // Check (provided) macros
- assertEquals(0, model.get().macros().size());
+ // Check (provided) functions
+ assertEquals(0, model.get().functions().size());
- // Check required macros
- assertEquals(1, model.get().requiredMacros().size());
- assertTrue(model.get().requiredMacros().containsKey("Placeholder"));
+ // Check required functions
+ assertEquals(1, model.get().requiredFunctions().size());
+ assertTrue(model.get().requiredFunctions().containsKey("Placeholder"));
assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(),
- model.get().requiredMacros().get("Placeholder"));
+ model.get().requiredFunctions().get("Placeholder"));
// Check signatures
assertEquals(1, model.get().signatures().size());
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java
index 4de3aa5d635..5447e5240f7 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java
@@ -48,7 +48,7 @@ public class TestableTensorFlowModel {
Tensor placeholder = placeholderArgument();
context.put(inputName, new TensorValue(placeholder));
- model.macros().forEach((k,v) -> evaluateMacro(context, model, k));
+ model.functions().forEach((k, v) -> evaluateFunction(context, model, k));
Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor();
assertEquals("Operation '" + operationName + "' produces equal results",
@@ -62,7 +62,7 @@ public class TestableTensorFlowModel {
Tensor placeholder = placeholderArgument();
context.put(inputName, new TensorValue(placeholder));
- model.macros().forEach((k,v) -> evaluateMacro(context, model, k));
+ model.functions().forEach((k, v) -> evaluateFunction(context, model, k));
Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor();
assertEquals("Operation '" + operationName + "' produces equal results", tfResult, vespaResult);
@@ -96,24 +96,24 @@ public class TestableTensorFlowModel {
return b.build();
}
- private void evaluateMacro(Context context, ImportedModel model, String macroName) {
- if (!context.names().contains(macroName)) {
- RankingExpression e = model.macros().get(macroName);
- evaluateMacroDependencies(context, model, e.getRoot());
- context.put(macroName, new TensorValue(e.evaluate(context).asTensor()));
+ private void evaluateFunction(Context context, ImportedModel model, String functionName) {
+ if (!context.names().contains(functionName)) {
+ RankingExpression e = model.functions().get(functionName);
+ evaluateFunctionDependencies(context, model, e.getRoot());
+ context.put(functionName, new TensorValue(e.evaluate(context).asTensor()));
}
}
- private void evaluateMacroDependencies(Context context, ImportedModel model, ExpressionNode node) {
+ private void evaluateFunctionDependencies(Context context, ImportedModel model, ExpressionNode node) {
if (node instanceof ReferenceNode) {
String name = node.toString();
- if (model.macros().containsKey(name)) {
- evaluateMacro(context, model, name);
+ if (model.functions().containsKey(name)) {
+ evaluateFunction(context, model, name);
}
}
else if (node instanceof CompositeNode) {
for (ExpressionNode child : ((CompositeNode)node).children()) {
- evaluateMacroDependencies(context, model, child);
+ evaluateFunctionDependencies(context, model, child);
}
}
}
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java
index 84e51835458..1f28f0b0129 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java
@@ -27,7 +27,7 @@ public class ConstantDereferencerTestCase {
TransformContext context = new TransformContext(constants);
assertEquals("1.0 + 2.0 + 3.5", c.transform(new RankingExpression("a + b + c"), context).toString());
- assertEquals("myMacro(1.0,2.0)", c.transform(new RankingExpression("myMacro(a, b)"), context).toString());
+ assertEquals("myFunction(1.0,2.0)", c.transform(new RankingExpression("myFunction(a, b)"), context).toString());
}
}
diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
index b7c955bd83b..d27352b8ea7 100644
--- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
+++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
@@ -7,12 +7,9 @@ import com.yahoo.document.annotation.AnnotationReferenceDataType;
import com.yahoo.document.annotation.AnnotationType;
import com.yahoo.documentmodel.NewDocumentType;
import com.yahoo.documentmodel.VespaDocumentType;
-import com.yahoo.searchdefinition.MinimalProcessingSearchBuilder;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.UnprocessingSearchBuilder;
import com.yahoo.searchdefinition.parser.ParseException;
-import com.yahoo.tensor.TensorType;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
@@ -80,10 +77,10 @@ public class DocumentGenMojo extends AbstractMojo {
void execute(File sdDir, File outputDir, String packageName) throws MojoFailureException {
if ("".equals(packageName)) throw new IllegalArgumentException("You may not use empty package for generated types.");
- searches = new HashMap<String, Search>();
- docTypes = new HashMap<String, String>();
- structTypes = new HashMap<String, String>();
- annotationTypes = new HashMap<String, String>();
+ searches = new HashMap<>();
+ docTypes = new HashMap<>();
+ structTypes = new HashMap<>();
+ annotationTypes = new HashMap<>();
outputDir.mkdirs();
SearchBuilder builder = buildSearches(sdDir);
@@ -111,7 +108,7 @@ public class DocumentGenMojo extends AbstractMojo {
public boolean accept(File dir, String name) {
return name.endsWith(".sd");
}});
- SearchBuilder builder = new MinimalProcessingSearchBuilder();
+ SearchBuilder builder = new SearchBuilder(true);
for (File f : sdFiles) {
try {
long modTime = f.lastModified();