Merge pull request #18080 from vespa-engine/balder/do-not-use-file-as-trigger

Balder/do not use file as trigger

4 files changed, 77 insertions, 155 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index 7e5b52f6af1..68e03ecb188 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -73,12 +73,10 @@ public class RankProfile implements Cloneable {
     protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>();
 
     /** The ranking expression to be used for first phase */
-    private RankingExpression firstPhaseRanking = null;
+    private RankingExpressionFunction firstPhaseRanking = null;
 
     /** The ranking expression to be used for second phase */
-    private RankingExpression secondPhaseRanking = null;
-
-    private Set<String> externalFileExpressions = new HashSet<>();
+    private RankingExpressionFunction secondPhaseRanking = null;
 
     /** Number of hits to be reranked in second phase, -1 means use default */
     private int rerankCount = -1;
@@ -128,8 +126,8 @@ public class RankProfile implements Cloneable {
     /**
      * Creates a new rank profile for a particular search definition
      *
-     * @param name   the name of the new profile
-     * @param search the search definition owning this profile
+     * @param name                the name of the new profile
+     * @param search              the search definition owning this profile
      * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing
      *                            and looking up rank profiles.
      */
@@ -169,10 +167,6 @@ public class RankProfile implements Cloneable {
         return search != null ? search.rankingConstants() : model.rankingConstants();
     }
 
-    public RankExpressionFiles rankExpressionFiles() {
-        return search != null ? search.rankExpressionFiles() : model.rankExpressionFiles();
-    }
-
     public Map<String, OnnxModel> onnxModels() {
         return search != null ? search.onnxModels().asMap() : onnxModels.asMap();
     }
@@ -198,9 +192,7 @@ public class RankProfile implements Cloneable {
     }
 
     /** Returns the name of the profile this one inherits, or null if none is inherited */
-    public String getInheritedName() {
-        return inheritedName;
-    }
+    public String getInheritedName() { return inheritedName; }
 
     /** Returns the inherited rank profile, or null if there is none */
     public RankProfile getInherited() {
@@ -243,7 +235,7 @@ public class RankProfile implements Cloneable {
 
     public MatchPhaseSettings getMatchPhaseSettings() {
         MatchPhaseSettings settings = this.matchPhaseSettings;
-        if (settings != null ) return settings;
+        if (settings != null) return settings;
         if (getInherited() != null) return getInherited().getMatchPhaseSettings();
         return null;
     }
@@ -264,7 +256,7 @@ public class RankProfile implements Cloneable {
      * @return the rank setting found, or null.
      */
     RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) {
-        for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext();) {
+        for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext(); ) {
             RankSetting setting = i.next();
             if (setting.getFieldName().equals(field) &&
                 setting.getType().equals(type)) {
@@ -369,77 +361,26 @@ public class RankProfile implements Cloneable {
      * Returns null if no expression is set.
      */
     public RankingExpression getFirstPhaseRanking() {
+        RankingExpressionFunction function = getFirstPhase();
+        if (function == null) return null;
+        return function.function.getBody();
+    }
+
+    public RankingExpressionFunction getFirstPhase() {
         if (firstPhaseRanking != null) return firstPhaseRanking;
         RankProfile inherited = getInherited();
-        if (inherited != null) return inherited.getFirstPhaseRanking();
+        if (inherited != null) return inherited.getFirstPhase();
         return null;
     }
 
     void setFirstPhaseRanking(RankingExpression rankingExpression) {
-        this.firstPhaseRanking = rankingExpression;
-    }
-
-    public String getUniqueExpressionName(String name) {
-        return getName().replace('-', '_') + "_" + name;
-    }
-    public String resolveExpressionName(String name) {
-        if (externalFileExpressions.contains(name)) {
-            return getUniqueExpressionName(name);
-        }
-        if (functions.get(name) == null) {
-            RankProfile inherited = getInherited();
-            if (inherited != null) {
-                return inherited.resolveExpressionName(name);
-            }
-        }
-        return name;
-    }
-    public String getFirstPhaseFile() {
-        String name = FIRST_PHASE;
-        if (externalFileExpressions.contains(name)) {
-            return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName();
-        }
-        if (firstPhaseRanking == null) {
-            RankProfile inherited = getInherited();
-            if (inherited != null) {
-                return getInherited().getFirstPhaseFile();
-            }
-        }
-        return null;
-    }
-
-    public String getSecondPhaseFile() {
-        String name = SECOND_PHASE;
-        if (externalFileExpressions.contains(name)) {
-            return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName();
-        }
-        if (secondPhaseRanking == null) {
-            RankProfile inherited = getInherited();
-            if (inherited != null) {
-                return getInherited().getSecondPhaseFile();
-            }
-        }
-        return null;
-    }
-
-    public String getExpressionFile(String name) {
-        if (externalFileExpressions.contains(name)) {
-            return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName();
-        }
-        if (functions.get(name) == null) {
-            RankProfile inherited = getInherited();
-            if (inherited != null) {
-                return inherited.getExpressionFile(name);
-            }
-        }
-        return null;
+        this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, Collections.emptyList(), rankingExpression), false);
     }
 
     public void setFirstPhaseRanking(String expression) {
         try {
-            this.firstPhaseRanking = parseRankingExpression(FIRST_PHASE, expression);
-        }
-        catch (ParseException e) {
+            firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, Collections.emptyList(), expression), false);
+        } catch (ParseException e) {
             throw new IllegalArgumentException("Illegal first phase ranking function", e);
         }
     }
@@ -449,15 +390,21 @@ public class RankProfile implements Cloneable {
      * Returns null if no expression is set.
      */
     public RankingExpression getSecondPhaseRanking() {
+        RankingExpressionFunction function = getSecondPhase();
+        if (function == null) return null;
+        return function.function().getBody();
+    }
+
+    public RankingExpressionFunction getSecondPhase() {
         if (secondPhaseRanking != null) return secondPhaseRanking;
         RankProfile inherited = getInherited();
-        if (inherited != null) return inherited.getSecondPhaseRanking();
+        if (inherited != null) return inherited.getSecondPhase();
         return null;
     }
 
     public void setSecondPhaseRanking(String expression) {
         try {
-            this.secondPhaseRanking = parseRankingExpression(SECOND_PHASE, expression);
+            secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, Collections.emptyList(), expression), false);
         }
         catch (ParseException e) {
             throw new IllegalArgumentException("Illegal second phase ranking function", e);
@@ -626,7 +573,7 @@ public class RankProfile implements Cloneable {
     /** Adds a function */
     public void addFunction(String name, List<String> arguments, String expression, boolean inline) {
         try {
-            addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline);
+            addFunction(parseRankingExpression(name, arguments, expression), inline);
         }
         catch (ParseException e) {
             throw new IllegalArgumentException("Could not parse function '" + name + "'", e);
@@ -714,20 +661,20 @@ public class RankProfile implements Cloneable {
         return retval;
     }
 
-    private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException {
+    private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException {
         if (expression.trim().length() == 0)
-            throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + ".");
+            throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + name + ".");
 
-        try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) {
-            return new RankingExpression(expressionName, rankingExpressionReader);
+        try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) {
+            return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader));
         }
         catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) {
             ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() +
-                                                          "' in " + getName()+ ", " + expressionName + ".");
+                                                          "' in " + getName()+ ", " + name + ".");
             throw (ParseException)exception.initCause(e);
         }
         catch (IOException e) {
-            throw new RuntimeException("IOException parsing ranking expression '" + expressionName + "'");
+            throw new RuntimeException("IOException parsing ranking expression '" + name + "'");
         }
     }
 
@@ -748,10 +695,6 @@ public class RankProfile implements Cloneable {
             throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file +
                     "' in subdirectory, which is not supported.");
 
-        if (search.getDeployProperties().featureFlags().distributeExternalRankExpressions()) {
-            rankExpressionFiles().add(new RankExpressionFile(getUniqueExpressionName(expName), fileName), search.getDeployLogger());
-            externalFileExpressions.add(expName);
-        }
         return search.getRankingExpression(fileName);
     }
 
@@ -769,7 +712,6 @@ public class RankProfile implements Cloneable {
             clone.functions = new LinkedHashMap<>(this.functions);
             clone.filterFields = new HashSet<>(this.filterFields);
             clone.constants = new HashMap<>(this.constants);
-            clone.externalFileExpressions = new HashSet(this.externalFileExpressions);
             return clone;
         }
         catch (CloneNotSupportedException e) {
@@ -801,8 +743,8 @@ public class RankProfile implements Cloneable {
         Map<String, RankingExpressionFunction> inlineFunctions =
                 compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms);
 
-        firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms);
-        secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms);
+        firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms);
+        secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms);
 
         // Function compiling second pass: compile all functions and insert previously compiled inline functions
         // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good.
@@ -811,7 +753,7 @@ public class RankProfile implements Cloneable {
 
     private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) {
         for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) {
-            if (constants.get(functionEntry.getKey()) != null)
+            if (constants.containsKey(functionEntry.getKey()))
                 throw new IllegalArgumentException("Cannot have both a constant and function named '" +
                                                    functionEntry.getKey() + "'");
         }
@@ -835,9 +777,9 @@ public class RankProfile implements Cloneable {
         // A straightforward iteration will either miss those functions, or may cause a ConcurrentModificationException
         while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) {
             RankingExpressionFunction rankingExpressionFunction = entry.getValue();
-            RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, featureTypes,
+            RankingExpressionFunction compiled = compile(rankingExpressionFunction, queryProfiles, featureTypes,
                                                  importedModels, getConstants(), inlineFunctions, expressionTransforms);
-            compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withExpression(compiled));
+            compiledFunctions.put(entry.getKey(), compiled);
         }
         return compiledFunctions;
     }
@@ -851,25 +793,25 @@ public class RankProfile implements Cloneable {
         return null;
     }
 
-    private RankingExpression compile(RankingExpression expression,
+    private RankingExpressionFunction compile(RankingExpressionFunction function,
                                       QueryProfileRegistry queryProfiles,
                                       Map<Reference, TensorType> featureTypes,
                                       ImportedMlModels importedModels,
                                       Map<String, Value> constants,
                                       Map<String, RankingExpressionFunction> inlineFunctions,
                                       ExpressionTransforms expressionTransforms) {
-        if (expression == null) return null;
+        if (function == null) return null;
         RankProfileTransformContext context = new RankProfileTransformContext(this,
                                                                               queryProfiles,
                                                                               featureTypes,
                                                                               importedModels,
                                                                               constants,
                                                                               inlineFunctions);
-        expression = expressionTransforms.transform(expression, context);
+        RankingExpression expression = expressionTransforms.transform(function.function().getBody(), context);
         for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) {
             addRankProperty(rankProperty.getKey(), rankProperty.getValue());
         }
-        return expression;
+        return function.withExpression(expression);
     }
 
     /**
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index 1115a4ce405..6b589a22de5 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -22,7 +22,6 @@ import com.yahoo.vespa.config.search.RankProfilesConfig;
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -59,7 +58,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
     public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels,
                           AttributeFields attributeFields, ModelContext.Properties deployProperties) {
         this.name = rankProfile.getName();
-        compressedProperties = compress(new Deriver(rankProfile, rankProfile.compile(queryProfiles, importedModels),
+        compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels),
                                         attributeFields, deployProperties).derive());
     }
 
@@ -124,10 +123,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
 
     private static class Deriver {
 
-        // Due to compiled rankprofiles flattening inheritance we need to use the uncompiled
-        // to sort out what comes from external files and not.
-        private final RankProfile unCompiledRankProfile;
-
         private final Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>();
         private final Set<ReferenceNode> summaryFeatures;
         private final Set<ReferenceNode> rankFeatures;
@@ -154,7 +149,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
         private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default");
         private final Map<String, String> attributeTypes;
         private final Map<String, String> queryFeatureTypes;
-        private final boolean useExternalExpressionFiles;
         private final Set<String> filterFields = new java.util.LinkedHashSet<>();
 
         private RankingExpression firstPhaseRanking;
@@ -163,12 +157,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
         /**
          * Creates a raw rank profile from the given rank profile
          */
-        Deriver(RankProfile unCompiledRankProfile, RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties)
+        Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties)
         {
-            this.unCompiledRankProfile = unCompiledRankProfile;
             attributeTypes = compiled.getAttributeTypes();
             queryFeatureTypes = compiled.getQueryFeatureTypes();
-            useExternalExpressionFiles = deployProperties.featureFlags().useExternalRankExpressions();
             firstPhaseRanking = compiled.getFirstPhaseRanking();
             secondPhaseRanking = compiled.getSecondPhaseRanking();
             summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures());
@@ -187,9 +179,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions();
             List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList());
             Map<String, String> functionProperties = new LinkedHashMap<>();
-            SerializationContext functionSerializationContext = useExternalExpressionFiles
-                    ? new FunctionSerializationContext(unCompiledRankProfile, functionExpressions, functionProperties)
-                    : new SerializationContext(functionExpressions);
+            SerializationContext functionSerializationContext = new SerializationContext(functionExpressions);
 
             if (firstPhaseRanking != null) {
                 functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext));
@@ -210,25 +200,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             filterFields.addAll(rp.allFilterFields());
         }
 
-        private static class FunctionSerializationContext extends SerializationContext {
-            private final RankProfile rankProfile;
-            FunctionSerializationContext(RankProfile rankProfile, Collection<ExpressionFunction> functions,
-                                         Map<String, String> serializedFunctions)
-            {
-                super(functions, null, serializedFunctions);
-                this.rankProfile = rankProfile;
-            }
-
-            @Override
-            public String uniqueName(String functionName) {
-                return rankProfile.resolveExpressionName(functionName);
-            }
-            @Override
-            public boolean needSerialization(String functionName) {
-                return functionName.equals(uniqueName(functionName)) && super.needSerialization(functionName);
-            }
-        }
-
         private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions,
                                                                      Map<String, String> functionProperties,
                                                                      SerializationContext functionContext) {
@@ -249,7 +220,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
                                               Map<String, String> functionProperties,
                                               SerializationContext context) {
             for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) {
-                if (useExternalExpressionFiles && unCompiledRankProfile.getExpressionFile(e.getKey()) != null) continue;
                 String propertyName = RankingExpression.propertyName(e.getKey());
                 if (context.serializedFunctions().containsKey(propertyName)) continue;
 
@@ -371,8 +341,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
                     properties.add(new Pair<>(property.getName(), property.getValue()));
                 }
             }
-            properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, unCompiledRankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE));
-            properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, unCompiledRankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE));
+            properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, RankProfile.FIRST_PHASE));
+            properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, RankProfile.SECOND_PHASE));
             for (FieldRankSettings settings : fieldRankSettings.values()) {
                 properties.addAll(settings.deriveRankProperties());
             }
@@ -437,7 +407,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             return properties;
         }
 
-        private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String fileName, String phase) {
+        private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) {
             List<Pair<String, String>> properties = new ArrayList<>();
             if (expression == null) return properties;
 
@@ -445,9 +415,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             if ("".equals(name))
                 name = phase;
 
-            if (useExternalExpressionFiles && (fileName != null)) {
-                properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + unCompiledRankProfile.getUniqueExpressionName(name) + ")"));
-            } else if (expression.getRoot() instanceof ReferenceNode) {
+            if (expression.getRoot() instanceof ReferenceNode) {
                 properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString()));
             } else {
                 properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")"));
diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg
index d1f6701e7f1..c6263d658c5 100644
--- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg
+++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg
@@ -58,7 +58,9 @@ rankprofile[].name "static"
 rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "attribute"
 rankprofile[].fef.property[].name "vespa.rank.secondphase"
-rankprofile[].fef.property[].value "rankingExpression(static_secondphase)"
+rankprofile[].fef.property[].value "rankingExpression(secondphase)"
+rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
+rankprofile[].fef.property[].value "10 + feature(arg1).out.out"
 rankprofile[].fef.property[].name "vespa.summary.feature"
 rankprofile[].fef.property[].value "attribute(foo1).out"
 rankprofile[].fef.property[].name "vespa.summary.feature"
@@ -77,7 +79,9 @@ rankprofile[].fef.property[].name "vespa.summary.feature"
 rankprofile[].fef.property[].value "attribute(bar4).out"
 rankprofile[].name "overflow"
 rankprofile[].fef.property[].name "vespa.rank.firstphase"
-rankprofile[].fef.property[].value "rankingExpression(overflow_firstphase)"
+rankprofile[].fef.property[].value "rankingExpression(firstphase)"
+rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
+rankprofile[].fef.property[].value "feature1(argument1,argument2,argument3,argument4).output + feature2(argument1,argument2,argument3,argument4).output + feature3(argument1,argument2,argument3,argument4).output + feature4(argument1,argument2,argument3,argument4).output + feature5(argument1,argument2,argument3,argument4).output + feature6(argument1,argument2,argument3,argument4).output + feature7(argument1,argument2,argument3,argument4).output + feature8(argument1,argument2,argument3,argument4).output + feature9(argument1,argument2,argument3,argument4).output + feature10(argument1,argument2,argument3,argument4).output + feature11(argument1,argument2,argument3,argument4).output + feature12(argument1,argument2,argument3,argument4).output + feature13(argument1,argument2,argument3,argument4).output + feature14(argument1,argument2,argument3,argument4).output + feature15(argument1,argument2,argument3,argument4).output + feature16(argument1,argument2,argument3,argument4).output + feature17(argument1,argument2,argument3,argument4).output + feature18(argument1,argument2,argument3,argument4).output + feature19(argument1,argument2,argument3,argument4).output + feature20(argument1,argument2,argument3,argument4).output + feature21(argument1,argument2,argument3,argument4).output + feature22(argument1,argument2,argument3,argument4).output + feature23(argument1,argument2,argument3,argument4).output + feature24(argument1,argument2,argument3,argument4).output + feature25(argument1,argument2,argument3,argument4).output + feature26(argument1,argument2,argument3,argument4).output + feature27(argument1,argument2,argument3,argument4).output + feature28(argument1,argument2,argument3,argument4).output + feature29(argument1,argument2,argument3,argument4).output + feature30(argument1,argument2,argument3,argument4).output + feature31(argument1,argument2,argument3,argument4).output + feature32(argument1,argument2,argument3,argument4).output + feature33(argument1,argument2,argument3,argument4).output + feature34(argument1,argument2,argument3,argument4).output + feature35(argument1,argument2,argument3,argument4).output + feature36(argument1,argument2,argument3,argument4).output + feature37(argument1,argument2,argument3,argument4).output + feature38(argument1,argument2,argument3,argument4).output + feature39(argument1,argument2,argument3,argument4).output + feature40(argument1,argument2,argument3,argument4).output + feature41(argument1,argument2,argument3,argument4).output + feature42(argument1,argument2,argument3,argument4).output + feature43(argument1,argument2,argument3,argument4).output + feature44(argument1,argument2,argument3,argument4).output + feature45(argument1,argument2,argument3,argument4).output + feature46(argument1,argument2,argument3,argument4).output + feature47(argument1,argument2,argument3,argument4).output + feature48(argument1,argument2,argument3,argument4).output + feature49(argument1,argument2,argument3,argument4).output + feature50(argument1,argument2,argument3,argument4).output + feature51(argument1,argument2,argument3,argument4).output + feature52(argument1,argument2,argument3,argument4).output + feature53(argument1,argument2,argument3,argument4).output + feature54(argument1,argument2,argument3,argument4).output + feature55(argument1,argument2,argument3,argument4).output + feature56(argument1,argument2,argument3,argument4).output + feature57(argument1,argument2,argument3,argument4).output + feature58(argument1,argument2,argument3,argument4).output + feature59(argument1,argument2,argument3,argument4).output + feature60(argument1,argument2,argument3,argument4).output + feature61(argument1,argument2,argument3,argument4).output + feature62(argument1,argument2,argument3,argument4).output + feature63(argument1,argument2,argument3,argument4).output + feature64(argument1,argument2,argument3,argument4).output + feature65(argument1,argument2,argument3,argument4).output + feature66(argument1,argument2,argument3,argument4).output + feature67(argument1,argument2,argument3,argument4).output + feature68(argument1,argument2,argument3,argument4).output + feature69(argument1,argument2,argument3,argument4).output + feature70(argument1,argument2,argument3,argument4).output + feature71(argument1,argument2,argument3,argument4).output + feature72(argument1,argument2,argument3,argument4).output + feature73(argument1,argument2,argument3,argument4).output + feature74(argument1,argument2,argument3,argument4).output + feature75(argument1,argument2,argument3,argument4).output + feature76(argument1,argument2,argument3,argument4).output + feature77(argument1,argument2,argument3,argument4).output + feature78(argument1,argument2,argument3,argument4).output + feature79(argument1,argument2,argument3,argument4).output + feature80(argument1,argument2,argument3,argument4).output + feature81(argument1,argument2,argument3,argument4).output + feature82(argument1,argument2,argument3,argument4).output + feature83(argument1,argument2,argument3,argument4).output + feature84(argument1,argument2,argument3,argument4).output + feature85(argument1,argument2,argument3,argument4).output + feature86(argument1,argument2,argument3,argument4).output + feature87(argument1,argument2,argument3,argument4).output + feature88(argument1,argument2,argument3,argument4).output + feature89(argument1,argument2,argument3,argument4).output + feature90(argument1,argument2,argument3,argument4).output + feature91(argument1,argument2,argument3,argument4).output + feature92(argument1,argument2,argument3,argument4).output + feature93(argument1,argument2,argument3,argument4).output + feature94(argument1,argument2,argument3,argument4).output + feature95(argument1,argument2,argument3,argument4).output + feature96(argument1,argument2,argument3,argument4).output + feature97(argument1,argument2,argument3,argument4).output + feature98(argument1,argument2,argument3,argument4).output + feature99(argument1,argument2,argument3,argument4).output + feature100(argument1,argument2,argument3,argument4).output + feature101(argument1,argument2,argument3,argument4).output + feature102(argument1,argument2,argument3,argument4).output + feature103(argument1,argument2,argument3,argument4).output + feature104(argument1,argument2,argument3,argument4).output + feature105(argument1,argument2,argument3,argument4).output + feature106(argument1,argument2,argument3,argument4).output + feature107(argument1,argument2,argument3,argument4).output + feature108(argument1,argument2,argument3,argument4).output + feature109(argument1,argument2,argument3,argument4).output + feature110(argument1,argument2,argument3,argument4).output + feature111(argument1,argument2,argument3,argument4).output + feature112(argument1,argument2,argument3,argument4).output + feature113(argument1,argument2,argument3,argument4).output + feature114(argument1,argument2,argument3,argument4).output + feature115(argument1,argument2,argument3,argument4).output + feature116(argument1,argument2,argument3,argument4).output + feature117(argument1,argument2,argument3,argument4).output + feature118(argument1,argument2,argument3,argument4).output + feature119(argument1,argument2,argument3,argument4).output + feature120(argument1,argument2,argument3,argument4).output + feature121(argument1,argument2,argument3,argument4).output + feature122(argument1,argument2,argument3,argument4).output + feature123(argument1,argument2,argument3,argument4).output + feature124(argument1,argument2,argument3,argument4).output + feature125(argument1,argument2,argument3,argument4).output + feature126(argument1,argument2,argument3,argument4).output + feature127(argument1,argument2,argument3,argument4).output + feature128(argument1,argument2,argument3,argument4).output + feature129(argument1,argument2,argument3,argument4).output + feature130(argument1,argument2,argument3,argument4).output + feature131(argument1,argument2,argument3,argument4).output + feature132(argument1,argument2,argument3,argument4).output + feature133(argument1,argument2,argument3,argument4).output + feature134(argument1,argument2,argument3,argument4).output + feature135(argument1,argument2,argument3,argument4).output + feature136(argument1,argument2,argument3,argument4).output + feature137(argument1,argument2,argument3,argument4).output + feature138(argument1,argument2,argument3,argument4).output + feature139(argument1,argument2,argument3,argument4).output + feature140(argument1,argument2,argument3,argument4).output + feature141(argument1,argument2,argument3,argument4).output + feature142(argument1,argument2,argument3,argument4).output + feature143(argument1,argument2,argument3,argument4).output + feature144(argument1,argument2,argument3,argument4).output + feature145(argument1,argument2,argument3,argument4).output + feature146(argument1,argument2,argument3,argument4).output + feature147(argument1,argument2,argument3,argument4).output + feature148(argument1,argument2,argument3,argument4).output + feature149(argument1,argument2,argument3,argument4).output + feature150(argument1,argument2,argument3,argument4).output + feature151(argument1,argument2,argument3,argument4).output + feature152(argument1,argument2,argument3,argument4).output + feature153(argument1,argument2,argument3,argument4).output + feature154(argument1,argument2,argument3,argument4).output + feature155(argument1,argument2,argument3,argument4).output + feature156(argument1,argument2,argument3,argument4).output + feature157(argument1,argument2,argument3,argument4).output + feature158(argument1,argument2,argument3,argument4).output + feature159(argument1,argument2,argument3,argument4).output + feature160(argument1,argument2,argument3,argument4).output + feature161(argument1,argument2,argument3,argument4).output + feature162(argument1,argument2,argument3,argument4).output + feature163(argument1,argument2,argument3,argument4).output + feature164(argument1,argument2,argument3,argument4).output + feature165(argument1,argument2,argument3,argument4).output + feature166(argument1,argument2,argument3,argument4).output + feature167(argument1,argument2,argument3,argument4).output + feature168(argument1,argument2,argument3,argument4).output + feature169(argument1,argument2,argument3,argument4).output + feature170(argument1,argument2,argument3,argument4).output + feature171(argument1,argument2,argument3,argument4).output + feature172(argument1,argument2,argument3,argument4).output + feature173(argument1,argument2,argument3,argument4).output + feature174(argument1,argument2,argument3,argument4).output + feature175(argument1,argument2,argument3,argument4).output + feature176(argument1,argument2,argument3,argument4).output + feature177(argument1,argument2,argument3,argument4).output + feature178(argument1,argument2,argument3,argument4).output + feature179(argument1,argument2,argument3,argument4).output + feature180(argument1,argument2,argument3,argument4).output + feature181(argument1,argument2,argument3,argument4).output + feature182(argument1,argument2,argument3,argument4).output + feature183(argument1,argument2,argument3,argument4).output + feature184(argument1,argument2,argument3,argument4).output + feature185(argument1,argument2,argument3,argument4).output + feature186(argument1,argument2,argument3,argument4).output + feature187(argument1,argument2,argument3,argument4).output + feature188(argument1,argument2,argument3,argument4).output + feature189(argument1,argument2,argument3,argument4).output + feature190(argument1,argument2,argument3,argument4).output + feature191(argument1,argument2,argument3,argument4).output + feature192(argument1,argument2,argument3,argument4).output + feature193(argument1,argument2,argument3,argument4).output + feature194(argument1,argument2,argument3,argument4).output + feature195(argument1,argument2,argument3,argument4).output + feature196(argument1,argument2,argument3,argument4).output + feature197(argument1,argument2,argument3,argument4).output + feature198(argument1,argument2,argument3,argument4).output + feature199(argument1,argument2,argument3,argument4).output + feature200(argument1,argument2,argument3,argument4).output + feature201(argument1,argument2,argument3,argument4).output + feature202(argument1,argument2,argument3,argument4).output + feature203(argument1,argument2,argument3,argument4).output + feature204(argument1,argument2,argument3,argument4).output + feature205(argument1,argument2,argument3,argument4).output + feature206(argument1,argument2,argument3,argument4).output + feature207(argument1,argument2,argument3,argument4).output + feature208(argument1,argument2,argument3,argument4).output + feature209(argument1,argument2,argument3,argument4).output + feature210(argument1,argument2,argument3,argument4).output + feature211(argument1,argument2,argument3,argument4).output + feature212(argument1,argument2,argument3,argument4).output + feature213(argument1,argument2,argument3,argument4).output + feature214(argument1,argument2,argument3,argument4).output + feature215(argument1,argument2,argument3,argument4).output + feature216(argument1,argument2,argument3,argument4).output + feature217(argument1,argument2,argument3,argument4).output + feature218(argument1,argument2,argument3,argument4).output + feature219(argument1,argument2,argument3,argument4).output + feature220(argument1,argument2,argument3,argument4).output + feature221(argument1,argument2,argument3,argument4).output + feature222(argument1,argument2,argument3,argument4).output + feature223(argument1,argument2,argument3,argument4).output + feature224(argument1,argument2,argument3,argument4).output + feature225(argument1,argument2,argument3,argument4).output + feature226(argument1,argument2,argument3,argument4).output + feature227(argument1,argument2,argument3,argument4).output + feature228(argument1,argument2,argument3,argument4).output + feature229(argument1,argument2,argument3,argument4).output + feature230(argument1,argument2,argument3,argument4).output + feature231(argument1,argument2,argument3,argument4).output + feature232(argument1,argument2,argument3,argument4).output + feature233(argument1,argument2,argument3,argument4).output + feature234(argument1,argument2,argument3,argument4).output + feature235(argument1,argument2,argument3,argument4).output + feature236(argument1,argument2,argument3,argument4).output + feature237(argument1,argument2,argument3,argument4).output + feature238(argument1,argument2,argument3,argument4).output + feature239(argument1,argument2,argument3,argument4).output + feature240(argument1,argument2,argument3,argument4).output + feature241(argument1,argument2,argument3,argument4).output + feature242(argument1,argument2,argument3,argument4).output + feature243(argument1,argument2,argument3,argument4).output + feature244(argument1,argument2,argument3,argument4).output + feature245(argument1,argument2,argument3,argument4).output + feature246(argument1,argument2,argument3,argument4).output + feature247(argument1,argument2,argument3,argument4).output + feature248(argument1,argument2,argument3,argument4).output + feature249(argument1,argument2,argument3,argument4).output + feature250(argument1,argument2,argument3,argument4).output + feature251(argument1,argument2,argument3,argument4).output + feature252(argument1,argument2,argument3,argument4).output + feature253(argument1,argument2,argument3,argument4).output + feature254(argument1,argument2,argument3,argument4).output + feature255(argument1,argument2,argument3,argument4).output + feature256(argument1,argument2,argument3,argument4).output + feature257(argument1,argument2,argument3,argument4).output + feature258(argument1,argument2,argument3,argument4).output + feature259(argument1,argument2,argument3,argument4).output + feature260(argument1,argument2,argument3,argument4).output + feature261(argument1,argument2,argument3,argument4).output + feature262(argument1,argument2,argument3,argument4).output + feature263(argument1,argument2,argument3,argument4).output + feature264(argument1,argument2,argument3,argument4).output + feature265(argument1,argument2,argument3,argument4).output + feature266(argument1,argument2,argument3,argument4).output + feature267(argument1,argument2,argument3,argument4).output + feature268(argument1,argument2,argument3,argument4).output + feature269(argument1,argument2,argument3,argument4).output + feature270(argument1,argument2,argument3,argument4).output + feature271(argument1,argument2,argument3,argument4).output + feature272(argument1,argument2,argument3,argument4).output + feature273(argument1,argument2,argument3,argument4).output + feature274(argument1,argument2,argument3,argument4).output + feature275(argument1,argument2,argument3,argument4).output + feature276(argument1,argument2,argument3,argument4).output + feature277(argument1,argument2,argument3,argument4).output + feature278(argument1,argument2,argument3,argument4).output + feature279(argument1,argument2,argument3,argument4).output + feature280(argument1,argument2,argument3,argument4).output + feature281(argument1,argument2,argument3,argument4).output + feature282(argument1,argument2,argument3,argument4).output + feature283(argument1,argument2,argument3,argument4).output + feature284(argument1,argument2,argument3,argument4).output + feature285(argument1,argument2,argument3,argument4).output + feature286(argument1,argument2,argument3,argument4).output + feature287(argument1,argument2,argument3,argument4).output + feature288(argument1,argument2,argument3,argument4).output + feature289(argument1,argument2,argument3,argument4).output + feature290(argument1,argument2,argument3,argument4).output + feature291(argument1,argument2,argument3,argument4).output + feature292(argument1,argument2,argument3,argument4).output + feature293(argument1,argument2,argument3,argument4).output + feature294(argument1,argument2,argument3,argument4).output + feature295(argument1,argument2,argument3,argument4).output + feature296(argument1,argument2,argument3,argument4).output + feature297(argument1,argument2,argument3,argument4).output + feature298(argument1,argument2,argument3,argument4).output + feature299(argument1,argument2,argument3,argument4).output + feature300(argument1,argument2,argument3,argument4).output"
 rankprofile[].fef.property[].name "vespa.rank.secondphase"
 rankprofile[].fef.property[].value "rankingExpression(secondphase)"
 rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
@@ -245,26 +249,32 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript"
 rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
 rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript"
 rankprofile[].fef.property[].value "rankingExpression(m1) * 67"
+rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript"
+rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness"
 rankprofile[].fef.property[].name "vespa.rank.secondphase"
 rankprofile[].fef.property[].value "rankingExpression(secondphase)"
 rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
-rankprofile[].fef.property[].value "40000 * rankingExpression(m2) * rankingExpression(macros_refering_macros_m4)"
+rankprofile[].fef.property[].value "40000 * rankingExpression(m2) * rankingExpression(m4)"
 rankprofile[].name "macros-refering-macros-inherited"
 rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript"
 rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
 rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript"
 rankprofile[].fef.property[].value "rankingExpression(m1) * 67"
+rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript"
+rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness"
 rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript"
 rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))"
 rankprofile[].fef.property[].name "vespa.rank.secondphase"
 rankprofile[].fef.property[].value "rankingExpression(secondphase)"
 rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
-rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(macros_refering_macros_m4)"
+rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(m4)"
 rankprofile[].name "macros-refering-macros-inherited2"
 rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript"
 rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
 rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript"
 rankprofile[].fef.property[].value "rankingExpression(m1) * 67"
+rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript"
+rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness"
 rankprofile[].fef.property[].name "vespa.rank.secondphase"
 rankprofile[].fef.property[].value "rankingExpression(secondphase)"
 rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
@@ -274,11 +284,13 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript"
 rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
 rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript"
 rankprofile[].fef.property[].value "rankingExpression(m1) * 67"
+rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript"
+rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness"
 rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript"
 rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))"
 rankprofile[].fef.property[].name "rankingExpression(m5).rankingScript"
-rankprofile[].fef.property[].value "if (isNan(attribute(glmpfw)) == 1, rankingExpression(m1), rankingExpression(macros_refering_macros_m4))"
+rankprofile[].fef.property[].value "if (isNan(attribute(glmpfw)) == 1, rankingExpression(m1), rankingExpression(m4))"
 rankprofile[].fef.property[].name "vespa.rank.secondphase"
 rankprofile[].fef.property[].value "rankingExpression(secondphase)"
 rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
-rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(macros_refering_macros_m4)"
+rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(m4)"
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
index ff17a211e17..85ef70132b5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
@@ -52,22 +52,22 @@ public class RankingExpressionsTestCase extends SchemaTestCase {
         RawRankProfile rawRankProfile = new RawRankProfile(functionsRankProfile, new QueryProfileRegistry(),
                 new ImportedMlModels(), new AttributeFields(search), deployProperties);
         List<Pair<String, String>> rankProperties = rawRankProfile.configProperties();
-        assertEquals(5, rankProperties.size());
+        assertEquals(6, rankProperties.size());
 
-        assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst());
-        assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond());
+        assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst());
+        assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond());
 
-        assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst());
-        assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond());
+        assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst());
+        assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond());
 
-        assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(2).getFirst());
-        assertEquals("78 + closeness(distance)", rankProperties.get(2).getSecond());
+        assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst());
+        assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond());
 
-        assertEquals("vespa.rank.firstphase", rankProperties.get(3).getFirst());
-        assertEquals("rankingExpression(firstphase)", rankProperties.get(3).getSecond());
+        assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst());
+        assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond());
 
-        assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(4).getFirst());
-        assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(4).getSecond());
+        assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst());
+        assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond());
     }
 
     @Test(expected = IllegalArgumentException.class)