Merge pull request #17951 from vespa-engine/balder/add-rankexpressionfile

Make DistributableResource to consolidate OnnxModel, RankingConstant …

5 files changed, 154 insertions, 111 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
new file mode 100644
index 00000000000..77ce2dd41b5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java
@@ -0,0 +1,72 @@
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.utils.FileSender;
+
+import java.util.Collection;
+import java.util.Objects;
+
+public class DistributableResource {
+    public enum PathType { FILE, URI };
+
+    /** The search definition-unique name of this constant */
+    private final String name;
+    private String path = null;
+    private String fileReference = "";
+    private PathType pathType = PathType.FILE;
+
+    public PathType getPathType() {
+        return pathType;
+    }
+
+    public DistributableResource(String name) {
+        this(name, null);
+    }
+    public DistributableResource(String name, String path) {
+        this.name = name;
+        this.path = path;
+    }
+
+    public void setFileName(String fileName) {
+        Objects.requireNonNull(fileName, "Filename cannot be null");
+        this.path = fileName;
+        this.pathType = PathType.FILE;
+    }
+
+    public void setUri(String uri) {
+        Objects.requireNonNull(uri, "uri cannot be null");
+        this.path = uri;
+        this.pathType = PathType.URI;
+    }
+
+    protected void setFileReference(String fileReference) { this.fileReference = fileReference; }
+    /** Initiate sending of this constant to some services over file distribution */
+    public void sendTo(Collection<? extends AbstractService> services) {
+        FileReference reference = (pathType == PathType.FILE)
+                ? FileSender.sendFileToServices(path, services)
+                : FileSender.sendUriToServices(path, services);
+        this.fileReference = reference.value();
+    }
+
+    public String getName() { return name; }
+    public String getFileName() { return path; }
+    public Path getFilePath() { return Path.fromString(path); }
+    public String getUri() { return path; }
+    public String getFileReference() { return fileReference; }
+
+    public void validate() {
+        if (path == null || path.isEmpty())
+            throw new IllegalArgumentException("Distributable resource must have a file or uri.");
+    }
+
+    public String toString() {
+        StringBuilder b = new StringBuilder();
+        b.append("resource '").append(name)
+                .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path)
+                .append("' with ref '").append(fileReference)
+                .append("'");
+        return b.toString();
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java
index 64338e24a8d..d20eecb82c5 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java
@@ -1,14 +1,9 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition;
 
-import com.yahoo.config.FileReference;
-import com.yahoo.path.Path;
 import com.yahoo.tensor.TensorType;
-import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.ml.OnnxModelInfo;
-import com.yahoo.vespa.model.utils.FileSender;
 
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -19,42 +14,26 @@ import java.util.Objects;
  *
  * @author lesters
  */
-public class OnnxModel {
+public class OnnxModel extends DistributableResource {
 
-    public enum PathType {FILE, URI};
-
-    private final String name;
-    private PathType pathType = PathType.FILE;
-    private String path = null;
-    private String fileReference = "";
     private OnnxModelInfo modelInfo = null;
     private Map<String, String> inputMap = new HashMap<>();
     private Map<String, String> outputMap = new HashMap<>();
 
     public OnnxModel(String name) {
-        this.name = name;
+        super(name);
     }
 
     public OnnxModel(String name, String fileName) {
-        this(name);
-        this.path = fileName;
+        super(name, fileName);
         validate();
     }
 
-    public void setFileName(String fileName) {
-        Objects.requireNonNull(fileName, "Filename cannot be null");
-        this.path = fileName;
-        this.pathType = PathType.FILE;
-    }
-
+    @Override
     public void setUri(String uri) {
         throw new IllegalArgumentException("URI for ONNX models are not currently supported");
     }
 
-    public PathType getPathType() {
-        return pathType;
-    }
-
     public void addInputNameMapping(String onnxName, String vespaName) {
         addInputNameMapping(onnxName, vespaName, true);
     }
@@ -84,20 +63,6 @@ public class OnnxModel {
         this.modelInfo = modelInfo;
     }
 
-    /** Initiate sending of this constant to some services over file distribution */
-    public void sendTo(Collection<? extends AbstractService> services) {
-        FileReference reference = (pathType == OnnxModel.PathType.FILE)
-                                  ? FileSender.sendFileToServices(path, services)
-                                  : FileSender.sendUriToServices(path, services);
-        this.fileReference = reference.value();
-    }
-
-    public String getName() { return name; }
-    public String getFileName() { return path; }
-    public Path   getFilePath() { return Path.fromString(path); }
-    public String getUri() { return path; }
-    public String getFileReference() { return fileReference; }
-
     public Map<String, String> getInputMap() { return Collections.unmodifiableMap(inputMap); }
     public Map<String, String> getOutputMap() { return Collections.unmodifiableMap(outputMap); }
 
@@ -108,19 +73,4 @@ public class OnnxModel {
     TensorType getTensorType(String onnxName, Map<String, TensorType> inputTypes) {
         return modelInfo != null ? modelInfo.getTensorType(onnxName, inputTypes) : TensorType.empty;
     }
-
-    public void validate() {
-        if (path == null || path.isEmpty())
-            throw new IllegalArgumentException("ONNX models must have a file or uri.");
-    }
-
-    public String toString() {
-        StringBuilder b = new StringBuilder();
-        b.append("onnx-model '").append(name)
-                .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path)
-                .append("' with ref '").append(fileReference)
-                .append("'");
-        return b.toString();
-    }
-
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java
new file mode 100644
index 00000000000..56385efeb0b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java
@@ -0,0 +1,35 @@
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.utils.FileSender;
+
+import java.util.Collection;
+
+public class RankExpressionFile extends DistributableResource {
+
+    public RankExpressionFile(String name, String path) {
+        super(name, path);
+        validate();
+    }
+
+    @Override
+    public void sendTo(Collection<? extends AbstractService> services) {
+        /*
+         *  TODO This is a very dirty hack due to using both SEARCH_DEFINITIONS_DIR and SCHEMA_DIR
+         *  and doing so inconsistently, combined with using both fields from application package on disk and in zookeeper.
+         *  The mess is spread out nicely, but ZookeeperClient, and writeSearchDefinitions and ZkApplicationPackage and FilesApplicationPackage
+         *  should be consolidated
+        */
+        try {
+            setFileReference(FileSender.sendFileToServices(ApplicationPackage.SCHEMAS_DIR + "/" + getFileName(), services).value());
+        } catch (IllegalArgumentException e1) {
+            try {
+                setFileReference(FileSender.sendFileToServices(ApplicationPackage.SEARCH_DEFINITIONS_DIR + "/" + getFileName(), services).value());
+            } catch (IllegalArgumentException e2) {
+                throw new IllegalArgumentException("Failed to find expression file '" + getFileName() + "' in '"
+                        + ApplicationPackage.SEARCH_DEFINITIONS_DIR + "' or '" + ApplicationPackage.SCHEMAS_DIR + "'.", e2);
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java
new file mode 100644
index 00000000000..ebc91e0693f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java
@@ -0,0 +1,35 @@
+package com.yahoo.searchdefinition;
+
+import com.yahoo.vespa.model.AbstractService;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RankExpressionFiles {
+    private final Map<String, RankExpressionFile> expressions = new HashMap<>();
+
+    public void add(RankExpressionFile expression) {
+        expression.validate();
+        String name = expression.getName();
+        if (expressions.containsKey(name))
+            throw new IllegalArgumentException("Rank expression file '" + name + "' defined twice");
+        expressions.put(name, expression);
+    }
+
+    /** Returns the ranking constant with the given name, or null if not present */
+    public RankExpressionFile get(String name) {
+        return expressions.get(name);
+    }
+
+    /** Returns a read-only map of the ranking constants in this indexed by name */
+    public Map<String, RankExpressionFile> asMap() {
+        return Collections.unmodifiableMap(expressions);
+    }
+
+    /** Initiate sending of these constants to some services over file distribution */
+    public void sendTo(Collection<? extends AbstractService> services) {
+        expressions.values().forEach(constant -> constant.sendTo(services));
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
index b41cf582204..8e376f13615 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
@@ -1,13 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition;
 
-import com.yahoo.config.FileReference;
 import com.yahoo.tensor.TensorType;
-import com.yahoo.vespa.model.AbstractService;
-import com.yahoo.vespa.model.utils.FileSender;
-
-import java.util.Collection;
-import java.util.Objects;
 
 /**
  * A global ranking constant distributed using file distribution.
@@ -17,81 +11,38 @@ import java.util.Objects;
  * @author arnej
  * @author bratseth
  */
-public class RankingConstant {
-
-    public enum PathType {FILE, URI};
-
-    /** The search definition-unique name of this constant */
-    private final String name;
+public class RankingConstant extends DistributableResource {
     private TensorType tensorType = null;
-    private String path = null;
-    private String fileReference = "";
-
-    public PathType getPathType() {
-        return pathType;
-    }
-
-    private PathType pathType = PathType.FILE;
 
     public RankingConstant(String name) {
-        this.name = name;
+        super(name);
     }
 
     public RankingConstant(String name, TensorType type, String fileName) {
-        this(name);
+        super(name, fileName);
         this.tensorType = type;
-        this.path = fileName;
         validate();
     }
 
-    public void setFileName(String fileName) {
-        Objects.requireNonNull(fileName, "Filename cannot be null");
-        this.path = fileName;
-        this.pathType = PathType.FILE;
-    }
-
-    public void setUri(String uri) {
-        Objects.requireNonNull(uri, "uri cannot be null");
-        this.path = uri;
-        this.pathType = PathType.URI;
-    }
-
     public void setType(TensorType type) {
         this.tensorType = type;
     }
 
-    /** Initiate sending of this constant to some services over file distribution */
-    public void sendTo(Collection<? extends AbstractService> services) {
-        FileReference reference = (pathType == RankingConstant.PathType.FILE)
-                                  ? FileSender.sendFileToServices(path, services)
-                                  : FileSender.sendUriToServices(path, services);
-        this.fileReference = reference.value();
-    }
-
-    public String getName() { return name; }
-    public String getFileName() { return path; }
-    public String getUri() { return path; }
-    public String getFileReference() { return fileReference; }
     public TensorType getTensorType() { return tensorType; }
     public String getType() { return tensorType.toString(); }
 
     public void validate() {
-        if (path == null || path.isEmpty())
-            throw new IllegalArgumentException("Ranking constants must have a file or uri.");
+        super.validate();
         if (tensorType == null)
-            throw new IllegalArgumentException("Ranking constant '" + name + "' must have a type.");
+            throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type.");
         if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty()))
-            throw new IllegalArgumentException("Illegal type in field " + name + " type " + tensorType +
+            throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType +
                                                ": Dense tensor dimensions must have a size");
     }
 
     public String toString() {
-        StringBuilder b = new StringBuilder();
-        b.append("constant '").append(name)
-         .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path)
-         .append("' with ref '").append(fileReference)
-         .append("' of type '").append(tensorType)
-         .append("'");
+        StringBuilder b = new StringBuilder(super.toString())
+                .append("' of type '").append(tensorType).append("'");
         return b.toString();
     }