Revert "Merge pull request #6626 from vespa-engine/revert-6625-bratseth/generate-rank-profiles-for-all-models-part-2-3"

This reverts commit c6d642b55558203a6eddd79efbc232bb34ec8e52, reversing
changes made to 59ffc5091b8246674843deea81a11a917abf801e.

5 files changed, 82 insertions, 24 deletions

diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
index 0ce6129ef7f..ab689b88993 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
@@ -10,7 +10,9 @@ import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
 import com.yahoo.searchdefinition.parser.ParseException;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 
@@ -25,6 +27,7 @@ class RankProfileSearchFixture {
     private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
     private final QueryProfileRegistry queryProfileRegistry;
     private Search search;
+    private Map<String, RankProfile> compiledRankProfiles = new HashMap<>();
 
     RankProfileSearchFixture(String rankProfiles) throws ParseException {
         this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles);
@@ -54,25 +57,38 @@ class RankProfileSearchFixture {
     }
 
     public void assertFirstPhaseExpression(String expExpression, String rankProfile) {
-        assertEquals(expExpression, rankProfile(rankProfile).getFirstPhaseRanking().getRoot().toString());
+        assertEquals(expExpression, compiledRankProfile(rankProfile).getFirstPhaseRanking().getRoot().toString());
     }
 
     public void assertSecondPhaseExpression(String expExpression, String rankProfile) {
-        assertEquals(expExpression, rankProfile(rankProfile).getSecondPhaseRanking().getRoot().toString());
+        assertEquals(expExpression, compiledRankProfile(rankProfile).getSecondPhaseRanking().getRoot().toString());
     }
 
     public void assertRankProperty(String expValue, String name, String rankProfile) {
-        List<RankProfile.RankProperty> rankPropertyList = rankProfile(rankProfile).getRankPropertyMap().get(name);
+        List<RankProfile.RankProperty> rankPropertyList = compiledRankProfile(rankProfile).getRankPropertyMap().get(name);
         assertEquals(1, rankPropertyList.size());
         assertEquals(expValue, rankPropertyList.get(0).getValue());
     }
 
-    public void assertMacro(String expExpression, String macroName, String rankProfile) {
-        assertEquals(expExpression, rankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString());
+    public void assertMacro(String expexctedExpression, String macroName, String rankProfile) {
+        assertEquals(expexctedExpression,
+                     compiledRankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString());
     }
 
+    public RankProfile compileRankProfile(String rankProfile) {
+        RankProfile compiled = rankProfileRegistry.getRankProfile(search, rankProfile).compile(queryProfileRegistry);
+        compiledRankProfiles.put(rankProfile, compiled);
+        return compiled;
+    }
+
+    /** Returns the given uncompiled profile */
     public RankProfile rankProfile(String rankProfile) {
-        return rankProfileRegistry.getRankProfile(search, rankProfile).compile(queryProfileRegistry);
+        return rankProfileRegistry.getRankProfile(search, rankProfile);
+    }
+
+    /** Returns the given compiled profile, or null if not compiled yet or not present at all */
+    public RankProfile compiledRankProfile(String rankProfile) {
+        return compiledRankProfiles.get(rankProfile);
     }
 
     public Search search() { return search; }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
index b2ef08dcc36..a7465fa9695 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
@@ -123,6 +123,7 @@ public class RankingExpressionWithOnnxTestCase {
                             "      expression: onnx('mnist_softmax.onnx')" +
                             "    }\n" +
                             "  }");
+            search.compileRankProfile("my_profile");
             search.assertFirstPhaseExpression(vespaExpression, "my_profile");
             fail("Expecting exception");
         }
@@ -164,7 +165,8 @@ public class RankingExpressionWithOnnxTestCase {
         catch (IllegalArgumentException expected) {
             assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " +
                             "onnx('mnist_softmax.onnx','y'): " +
-                            "Model does not have the specified signature 'y'",
+                         "No expressions available in model 'mnist_softmax.onnx'",
+//                            "No expressions named 'y' in model 'mnist_softmax.onnx'. Available expressions: mnist_softmax.onnx.default.add",
                     Exceptions.toMessageString(expected));
         }
     }
@@ -220,7 +222,8 @@ public class RankingExpressionWithOnnxTestCase {
 
         String vespaExpressionWithoutConstant =
                 "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), mnist_softmax_onnx_Variable, f(a,b)(a * b)), sum, d2), constant(mnist_softmax_onnx_Variable_1), f(a,b)(a + b))";
-        RankProfileSearchFixture search = fixtureWith(rankProfile, new StoringApplicationPackage(applicationDir));
+        RankProfileSearchFixture search = uncompiledFixtureWith(rankProfile, new StoringApplicationPackage(applicationDir));
+        search.compileRankProfile("my_profile");
         search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
 
         assertNull("Constant overridden by macro is not added",
@@ -234,7 +237,8 @@ public class RankingExpressionWithOnnxTestCase {
             IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(),
                     storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
             StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory);
-            RankProfileSearchFixture searchFromStored = fixtureWith(rankProfile, storedApplication);
+            RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication);
+            searchFromStored.compileRankProfile("my_profile");
             searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
             assertNull("Constant overridden by macro is not added",
                     searchFromStored.search().getRankingConstants().get("mnist_softmax_onnx_Variable"));
@@ -271,19 +275,19 @@ public class RankingExpressionWithOnnxTestCase {
 
     private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression) {
         return fixtureWith(placeholderExpression, firstPhaseExpression, null, null, "Placeholder",
-                new StoringApplicationPackage(applicationDir));
+                           new StoringApplicationPackage(applicationDir));
     }
 
     private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression,
                                                  String constant, String field) {
         return fixtureWith(placeholderExpression, firstPhaseExpression, constant, field, "Placeholder",
-                new StoringApplicationPackage(applicationDir));
+                           new StoringApplicationPackage(applicationDir));
     }
 
-    private RankProfileSearchFixture fixtureWith(String rankProfile, StoringApplicationPackage application) {
+    private RankProfileSearchFixture uncompiledFixtureWith(String rankProfile, StoringApplicationPackage application) {
         try {
             return new RankProfileSearchFixture(application, application.getQueryProfiles(),
-                    rankProfile, null, null);
+                                                rankProfile, null, null);
         }
         catch (ParseException e) {
             throw new IllegalArgumentException(e);
@@ -297,7 +301,7 @@ public class RankingExpressionWithOnnxTestCase {
                                                  String macroName,
                                                  StoringApplicationPackage application) {
         try {
-            return new RankProfileSearchFixture(
+            RankProfileSearchFixture fixture = new RankProfileSearchFixture(
                     application,
                     application.getQueryProfiles(),
                     "  rank-profile my_profile {\n" +
@@ -310,6 +314,8 @@ public class RankingExpressionWithOnnxTestCase {
                             "  }",
                     constant,
                     field);
+            fixture.compileRankProfile("my_profile");
+            return fixture;
         }
         catch (ParseException e) {
             throw new IllegalArgumentException(e);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
index 7228af2b0de..29859817736 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
@@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.io.GrowableByteBuffer;
 import com.yahoo.io.IOUtils;
+import com.yahoo.io.reader.NamedReader;
 import com.yahoo.path.Path;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.RankingConstant;
@@ -22,10 +23,12 @@ import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Reader;
 import java.io.UncheckedIOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
@@ -156,6 +159,7 @@ public class RankingExpressionWithTensorFlowTestCase {
                     "      expression: tensorflow('mnist_softmax/saved')" +
                     "    }\n" +
                     "  }");
+            search.compileRankProfile("my_profile");
             search.assertFirstPhaseExpression(vespaExpression, "my_profile");
             fail("Expecting exception");
         }
@@ -196,7 +200,9 @@ public class RankingExpressionWithTensorFlowTestCase {
         catch (IllegalArgumentException expected) {
             assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " +
                          "tensorflow('mnist_softmax/saved','serving_defaultz'): " +
-                         "Model does not have the specified signature 'serving_defaultz'",
+                         "No expressions available in model 'mnist_softmax_saved'",
+//                         "No expressions named 'serving_defaultz' in model 'mnist_softmax/saved'. "+
+//                         "Available expressions: mnist_softmax_saved.serving_default.y",
                          Exceptions.toMessageString(expected));
         }
     }
@@ -212,7 +218,9 @@ public class RankingExpressionWithTensorFlowTestCase {
         catch (IllegalArgumentException expected) {
             assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " +
                          "tensorflow('mnist_softmax/saved','serving_default','x'): " +
-                         "Model does not have the specified output 'x'",
+                         "No expressions available in model 'mnist_softmax_saved'",
+//                         "No expression 'mnist_softmax_saved.serving_default.x' in model 'mnist_softmax/saved'. " +
+//                         "Available expressions: mnist_softmax_saved.serving_default.y",
                          Exceptions.toMessageString(expected));
         }
     }
@@ -268,7 +276,8 @@ public class RankingExpressionWithTensorFlowTestCase {
 
         String vespaExpressionWithoutConstant =
                 "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), mnist_softmax_saved_layer_Variable_read, f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))";
-        RankProfileSearchFixture search = fixtureWith(rankProfile, new StoringApplicationPackage(applicationDir));
+        RankProfileSearchFixture search = fixtureWithUncompiled(rankProfile, new StoringApplicationPackage(applicationDir));
+        search.compileRankProfile("my_profile");
         search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
 
         assertNull("Constant overridden by macro is not added",
@@ -282,7 +291,8 @@ public class RankingExpressionWithTensorFlowTestCase {
             IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(),
                                   storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
             StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory);
-            RankProfileSearchFixture searchFromStored = fixtureWith(rankProfile, storedApplication);
+            RankProfileSearchFixture searchFromStored = fixtureWithUncompiled(rankProfile, storedApplication);
+            searchFromStored.compileRankProfile("my_profile");
             searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
             assertNull("Constant overridden by macro is not added",
                        searchFromStored.search().getRankingConstants().get("mnist_softmax_saved_layer_Variable_read"));
@@ -297,7 +307,7 @@ public class RankingExpressionWithTensorFlowTestCase {
     public void testTensorFlowReduceBatchDimension() {
         final String expression = "join(join(reduce(join(reduce(rename(Placeholder, (d0, d1), (d0, d2)), sum, d0), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))";
         RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)",
-                "tensorflow('mnist_softmax/saved')");
+                                                      "tensorflow('mnist_softmax/saved')");
         search.assertFirstPhaseExpression(expression, "my_profile");
         assertLargeConstant("mnist_softmax_saved_layer_Variable_1_read", search, Optional.of(10L));
         assertLargeConstant("mnist_softmax_saved_layer_Variable_read", search, Optional.of(7840L));
@@ -362,7 +372,7 @@ public class RankingExpressionWithTensorFlowTestCase {
     }
 
     private void assertSmallConstant(String name, TensorType type, RankProfileSearchFixture search) {
-        Value value = search.rankProfile("my_profile").getConstants().get(name);
+        Value value = search.compiledRankProfile("my_profile").getConstants().get(name);
         assertNotNull(value);
         assertEquals(type, value.type());
     }
@@ -410,7 +420,7 @@ public class RankingExpressionWithTensorFlowTestCase {
                                                  String macroName,
                                                  StoringApplicationPackage application) {
         try {
-            return new RankProfileSearchFixture(
+            RankProfileSearchFixture fixture = new RankProfileSearchFixture(
                     application,
                     application.getQueryProfiles(),
                     "  rank-profile my_profile {\n" +
@@ -423,13 +433,15 @@ public class RankingExpressionWithTensorFlowTestCase {
                     "  }",
                     constant,
                     field);
+            fixture.compileRankProfile("my_profile");
+            return fixture;
         }
         catch (ParseException e) {
             throw new IllegalArgumentException(e);
         }
     }
 
-    private RankProfileSearchFixture fixtureWith(String rankProfile, StoringApplicationPackage application) {
+    private RankProfileSearchFixture fixtureWithUncompiled(String rankProfile, StoringApplicationPackage application) {
         try {
             return new RankProfileSearchFixture(application, application.getQueryProfiles(),
                                                 rankProfile, null, null);
@@ -463,6 +475,21 @@ public class RankingExpressionWithTensorFlowTestCase {
             return new StoringApplicationPackageFile(file, Path.fromString(root.toString()));
         }
 
+        @Override
+        public List<NamedReader> getFiles(Path path, String suffix) {
+            List<NamedReader> readers = new ArrayList<>();
+            for (File file : getFileReference(path).listFiles()) {
+                if ( ! file.getName().endsWith(suffix)) continue;
+                try {
+                    readers.add(new NamedReader(file.getName(), new FileReader(file)));
+                }
+                catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+            return readers;
+        }
+
     }
 
     static class StoringApplicationPackageFile extends ApplicationFile {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
index dba2bdbfbbf..0866d3192cf 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
@@ -25,6 +25,7 @@ public class RankingExpressionWithTensorTestCase {
                 "      }\n" +
                 "    }\n" +
                 "  }");
+        f.compileRankProfile("my_profile");
         f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile");
         f.assertRankProperty("{{x:1,y:2}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile");
         f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile");
@@ -47,6 +48,7 @@ public class RankingExpressionWithTensorTestCase {
                 "      }\n" +
                 "    }\n" +
                 "  }");
+        f.compileRankProfile("my_profile");
         f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile");
         f.assertRankProperty("{{x:1,y:2}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile");
         f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile");
@@ -65,6 +67,7 @@ public class RankingExpressionWithTensorTestCase {
                 "      }\n" +
                 "    }\n" +
                 "  }");
+        f.compileRankProfile("my_profile");
         f.assertSecondPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile");
         f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile");
         f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile");
@@ -85,6 +88,7 @@ public class RankingExpressionWithTensorTestCase {
                 "      expression: sum(my_tensor)\n" +
                 "    }\n" +
                 "  }");
+        f.compileRankProfile("my_profile");
         f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile");
         f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile");
         f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile");
@@ -106,6 +110,7 @@ public class RankingExpressionWithTensorTestCase {
                 "      }\n" +
                 "    }\n" +
                 "  }");
+        f.compileRankProfile("my_profile");
         f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile");
         f.assertMacro("reduce(constant(my_tensor), sum)", "my_macro", "my_profile");
         f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile");
@@ -127,6 +132,7 @@ public class RankingExpressionWithTensorTestCase {
                 "      my_number_2: 5.0\n" +
                 "    }\n" +
                 "  }");
+        f.compileRankProfile("my_profile");
         f.assertFirstPhaseExpression("3.0 + reduce(constant(my_tensor), sum) + 5.0", "my_profile");
         f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile");
         f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile");
@@ -139,7 +145,7 @@ public class RankingExpressionWithTensorTestCase {
     public void requireThatInvalidTensorTypeSpecThrowsException() throws ParseException {
         exception.expect(IllegalArgumentException.class);
         exception.expectMessage("For constant tensor 'my_tensor' in rank profile 'my_profile': Illegal tensor type spec: Failed parsing element 'x' in type spec 'tensor(x)'");
-        new RankProfileSearchFixture(
+        RankProfileSearchFixture f = new RankProfileSearchFixture(
                 "  rank-profile my_profile {\n" +
                 "    constants {\n" +
                 "      my_tensor {\n" +
@@ -148,6 +154,7 @@ public class RankingExpressionWithTensorTestCase {
                 "      }\n" +
                 "    }\n" +
                 "  }");
+        f.compileRankProfile("my_profile");
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXgboostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXgboostTestCase.java
index b65cb0b3d5f..f98783ad671 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXgboostTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXgboostTestCase.java
@@ -36,7 +36,7 @@ public class RankingExpressionWithXgboostTestCase {
                                                  String field,
                                                  RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage application) {
         try {
-            return new RankProfileSearchFixture(
+            RankProfileSearchFixture fixture = new RankProfileSearchFixture(
                     application,
                     application.getQueryProfiles(),
                     "  rank-profile my_profile {\n" +
@@ -46,6 +46,8 @@ public class RankingExpressionWithXgboostTestCase {
                             "  }",
                     constant,
                     field);
+            fixture.compileRankProfile("my_profile");
+            return fixture;
         } catch (ParseException e) {
             throw new IllegalArgumentException(e);
         }