Merge pull request #6642 from vespa-engine/bratseth/generate-rank-profiles-for-all-models-part-2-6

Import all signaturees

8 files changed, 23 insertions, 43 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConvertedModel.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConvertedModel.java
index a59de09af65..99e3ae8abdc 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConvertedModel.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConvertedModel.java
@@ -79,13 +79,12 @@ public class ConvertedModel {
      */
     public ConvertedModel(Path modelPath,
                           RankProfileTransformContext context,
-                          ImportedModels importedModels,
-                          FeatureArguments arguments) { // TODO: Remove
+                          ImportedModels importedModels) {
         this.modelPath = modelPath;
         this.modelName = toModelName(modelPath);
         ModelStore store = new ModelStore(context.rankProfile().getSearch().sourceApplication(), modelPath);
         if ( store.hasSourceModel())
-            expressions = convertModel(store, context.rankProfile(), context.queryProfiles(), importedModels, arguments);
+            expressions = convertModel(store, context.rankProfile(), context.queryProfiles(), importedModels);
         else
             expressions = transformFromStoredModel(store, context.rankProfile());
     }
@@ -93,10 +92,9 @@ public class ConvertedModel {
     private Map<String, RankingExpression> convertModel(ModelStore store,
                                                         RankProfile profile,
                                                         QueryProfileRegistry queryProfiles,
-                                                        ImportedModels importedModels,
-                                                        FeatureArguments arguments) {
+                                                        ImportedModels importedModels) {
         ImportedModel model = importedModels.imported(store.modelFiles.modelName(), store.sourceModelDir());
-        return transformFromImportedModel(model, store, profile, queryProfiles, arguments);
+        return transformFromImportedModel(model, store, profile, queryProfiles);
     }
 
     /** Returns the expression matching the given arguments */
@@ -136,8 +134,7 @@ public class ConvertedModel {
     private Map<String, RankingExpression> transformFromImportedModel(ImportedModel model,
                                                                       ModelStore store,
                                                                       RankProfile profile,
-                                                                      QueryProfileRegistry queryProfiles,
-                                                                      FeatureArguments arguments) {
+                                                                      QueryProfileRegistry queryProfiles) {
         // Add constants
         Set<String> constantsReplacedByMacros = new HashSet<>();
         model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v));
@@ -150,10 +147,7 @@ public class ConvertedModel {
         // Add expressions
         Map<String, RankingExpression> expressions = new HashMap<>();
         for (Map.Entry<String, ImportedModel.Signature> signatureEntry : model.signatures().entrySet()) {
-            if ( ! matches(signatureEntry.getValue(), arguments, Optional.empty())) continue;
-
             for (Map.Entry<String, String> outputEntry : signatureEntry.getValue().outputs().entrySet()) {
-                if ( ! matches(signatureEntry.getValue(), arguments, Optional.of(outputEntry.getKey()))) continue;
                 addExpression(model.expressions().get(outputEntry.getValue()),
                               modelName + "." + signatureEntry.getKey() + "." + outputEntry.getKey(),
                               constantsReplacedByMacros,
@@ -196,13 +190,6 @@ public class ConvertedModel {
         return expressions;
     }
 
-    private boolean matches(ImportedModel.Signature signature, FeatureArguments arguments, Optional<String> output) {
-        if ( ! modelName.equals(arguments.modelName)) return false;
-        if ( arguments.signature.isPresent() && ! signature.name().equals(arguments.signature().get())) return false;
-        if (output.isPresent() && arguments.output().isPresent() && ! output.get().matches(arguments.output().get())) return false;
-        return true;
-    }
-
     private void addExpression(RankingExpression expression,
                                String expressionName,
                                Set<String> constantsReplacedByMacros,
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java
index 8005608f070..51c44f4066e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java
@@ -45,7 +45,7 @@ public class OnnxFeatureConverter extends ExpressionTransformer<RankProfileTrans
         try {
             Path modelPath = Path.fromString(ConvertedModel.FeatureArguments.asString(feature.getArguments().expressions().get(0)));
             // TODO: Increase scope of this instance to a rank profile:
-            ConvertedModel convertedModel = new ConvertedModel(modelPath, context, importedModels, new ConvertedModel.FeatureArguments(feature.getArguments()));
+            ConvertedModel convertedModel = new ConvertedModel(modelPath, context, importedModels);
             return convertedModel.expression(asFeatureArguments(feature.getArguments()));
         }
         catch (IllegalArgumentException | UncheckedIOException e) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java
index 8e22146b381..1343af1b6ec 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java
@@ -38,7 +38,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil
         try {
             Path modelPath = Path.fromString(ConvertedModel.FeatureArguments.asString(feature.getArguments().expressions().get(0)));
             // TODO: Increase scope of this instance to a rank profile:
-            ConvertedModel convertedModel = new ConvertedModel(modelPath, context, importedModels, new ConvertedModel.FeatureArguments(feature.getArguments()));
+            ConvertedModel convertedModel = new ConvertedModel(modelPath, context, importedModels);
             return convertedModel.expression(asFeatureArguments(feature.getArguments()));
         }
         catch (IllegalArgumentException | UncheckedIOException e) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
index a7465fa9695..80629172fe3 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java
@@ -165,8 +165,7 @@ public class RankingExpressionWithOnnxTestCase {
         catch (IllegalArgumentException expected) {
             assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " +
                             "onnx('mnist_softmax.onnx','y'): " +
-                         "No expressions available in model 'mnist_softmax.onnx'",
-//                            "No expressions named 'y' in model 'mnist_softmax.onnx'. Available expressions: mnist_softmax.onnx.default.add",
+                            "No expressions named 'y' in model 'mnist_softmax.onnx'. Available expressions: mnist_softmax.onnx.default.add",
                     Exceptions.toMessageString(expected));
         }
     }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
index 9a96555bb78..d9bf23ec8fb 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
@@ -200,9 +200,8 @@ public class RankingExpressionWithTensorFlowTestCase {
         catch (IllegalArgumentException expected) {
             assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " +
                          "tensorflow('mnist_softmax/saved','serving_defaultz'): " +
-                         "No expressions available in model 'mnist_softmax_saved'",
-//                         "No expressions named 'serving_defaultz' in model 'mnist_softmax/saved'. "+
-//                         "Available expressions: mnist_softmax_saved.serving_default.y",
+                         "No expressions named 'serving_defaultz' in model 'mnist_softmax/saved'. "+
+                         "Available expressions: mnist_softmax_saved.serving_default.y",
                          Exceptions.toMessageString(expected));
         }
     }
@@ -218,9 +217,8 @@ public class RankingExpressionWithTensorFlowTestCase {
         catch (IllegalArgumentException expected) {
             assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " +
                          "tensorflow('mnist_softmax/saved','serving_default','x'): " +
-                         "No expressions available in model 'mnist_softmax_saved'",
-//                         "No expression 'mnist_softmax_saved.serving_default.x' in model 'mnist_softmax/saved'. " +
-//                         "Available expressions: mnist_softmax_saved.serving_default.y",
+                         "No expression 'mnist_softmax_saved.serving_default.x' in model 'mnist_softmax/saved'. " +
+                         "Available expressions: mnist_softmax_saved.serving_default.y",
                          Exceptions.toMessageString(expected));
         }
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java
index 7c962e5b6be..0397678fd41 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java
@@ -46,4 +46,5 @@ public class HttpGetConfigHandler extends HttpHandler {
         if (config == null) HttpConfigRequest.throwModelNotReady();
         return config;
     }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java
index 1978c2b241a..3c6a20ff58f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java
+++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java
@@ -24,9 +24,10 @@ import edu.umd.cs.findbugs.annotations.NonNull;
 /**
  * A transposed view for cloud.config.model.
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public final class ServiceModel {
+
     private static final String CLUSTERCONTROLLER_TYPENAME = "container-clustercontroller";
 
     private static final String CONTENT_CLUSTER_TYPENAME = "content";
@@ -61,9 +62,8 @@ public final class ServiceModel {
                 List<String> identifiers = s.getIdentifiers();
                 for (String identifier : identifiers) {
                     if (seenIdentifiers.containsKey(identifier)) {
-                        throw new RuntimeException(
-                                "Congrats, you have a publishable result. We have a very unexpected hash collision" + " between "
-                                        + seenIdentifiers.get(identifier) + " and " + s + ".");
+                        throw new RuntimeException("Hash collision" + " between " +
+                                                   seenIdentifiers.get(identifier) + " and " + s + ".");
                     }
                     seenIdentifiers.put(identifier, s);
                 }
@@ -74,9 +74,8 @@ public final class ServiceModel {
         servicesMap = servicesBuilder.build();
     }
 
-    private static void addService(Table<String, String, List<Service>> services,
-            String hostName,
-            com.yahoo.vespa.serviceview.bindings.Service s) {
+    private static void addService(Table<String, String, List<Service>> services, String hostName,
+                                   com.yahoo.vespa.serviceview.bindings.Service s) {
         boolean hasStateApi = false;
         int statePort = 0;
         List<Integer> ports = new ArrayList<>(s.ports.size());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 26df4efa3c1..3391eee4d62 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -172,18 +172,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request);
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job"))
-            return JobControllerApiHandlerHelper.jobTypeResponse(jobTypes(path), latestRuns(path), request.getUri());
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}"))
-            return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}"))
-            return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path));
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(jobTypes(path), latestRuns(path), request.getUri());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"))
-            return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
         return ErrorResponse.notFoundError("Nothing at " + path);
     }