Send model paths

4 files changed, 113 insertions, 44 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/DistributableResource.java b/config-model/src/main/java/com/yahoo/schema/DistributableResource.java
index 7a8a3963ba4..e7bdb68a03d 100644
--- a/config-model/src/main/java/com/yahoo/schema/DistributableResource.java
+++ b/config-model/src/main/java/com/yahoo/schema/DistributableResource.java
@@ -67,14 +67,9 @@ public class DistributableResource implements Comparable <DistributableResource>
 
     public void register(FileRegistry fileRegistry) {
         switch (pathType) {
-            case FILE:
-                fileReference = fileRegistry.addFile(path);
-                break;
-            case URI:
-                fileReference = fileRegistry.addUri(path);
-                break;
-            default:
-                throw new IllegalArgumentException("Unknown path type " + pathType);
+            case FILE -> fileReference = fileRegistry.addFile(path);
+            case URI -> fileReference = fileRegistry.addUri(path);
+            default -> throw new IllegalArgumentException("Unknown path type " + pathType);
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java
index 1631bd228df..ee1771cfbfc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java
@@ -3,6 +3,9 @@ package com.yahoo.vespa.model.builder.xml.dom;
 
 import com.yahoo.collections.Tuple2;
 import com.yahoo.config.ConfigurationRuntimeException;
+import com.yahoo.config.FileReference;
+import com.yahoo.config.ModelReference;
+import com.yahoo.config.UrlReference;
 import com.yahoo.text.XML;
 import com.yahoo.vespa.config.ConfigDefinition;
 import com.yahoo.vespa.config.ConfigDefinitionKey;
@@ -11,6 +14,7 @@ import com.yahoo.vespa.config.util.ConfigUtils;
 import com.yahoo.yolean.Exceptions;
 import org.w3c.dom.Element;
 import java.util.List;
+import java.util.Optional;
 import java.util.regex.Pattern;
 
 /**
@@ -133,19 +137,19 @@ public class DomConfigPayloadBuilder {
         }
         else if (element.hasAttribute("model-id") || element.hasAttribute("url") || element.hasAttribute("path")) {
             // special syntax for "model" fields
-            String modelString = modelElement("model-id", element);
-            modelString += " " + modelElement("url", element);
-            modelString += " " + modelElement("path", element);
-            payloadBuilder.setField(name, modelString);
+            var model = ModelReference.unresolved(modelElement("model-id", element),
+                                                  modelElement("url", element).map(UrlReference::new),
+                                                  modelElement("path", element).map(FileReference::new));
+            payloadBuilder.setField(name, model.toString());
         }
         else { // leaf value: <myValueName>value</myValue>
             payloadBuilder.setField(name, value);
         }
     }
 
-    private String modelElement(String attributeName, Element element) {
-        String value = XML.attribute(attributeName, element).orElse("\"\"").trim();
-        if (value.contains(" "))
+    private Optional<String> modelElement(String attributeName, Element element) {
+        Optional<String> value = XML.attribute(attributeName, element);
+        if (value.isPresent() && value.get().contains(" "))
             throw new IllegalArgumentException("The value of " + attributeName + " on " + element.getTagName() +
                                                "cannot contain space");
         return value;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
index b9a81592ae4..39759743723 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.model.utils;
 
 import com.yahoo.config.FileReference;
+import com.yahoo.config.ModelReference;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
@@ -17,6 +18,7 @@ import java.io.Serializable;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.logging.Level;
 
 /**
@@ -40,8 +42,7 @@ public class FileSender implements Serializable {
      * Sends all user configured files for a producer to all given services.
      */
     public <PRODUCER extends AbstractConfigProducer<?>> void sendUserConfiguredFiles(PRODUCER producer) {
-        if (services.isEmpty())
-            return;
+        if (services.isEmpty()) return;
 
         UserConfigRepo userConfigs = producer.getUserConfigs();
         Map<Path, FileReference> sentFiles = new HashMap<>();
@@ -64,22 +65,22 @@ public class FileSender implements Serializable {
             return;
         }
         // Inspect fields at this level
-        sendEntries(builder, sentFiles, configDefinition.getFileDefs());
-        sendEntries(builder, sentFiles, configDefinition.getPathDefs());
+        sendEntries(builder, sentFiles, configDefinition.getFileDefs(), false);
+        sendEntries(builder, sentFiles, configDefinition.getPathDefs(), false);
+        sendEntries(builder, sentFiles, configDefinition.getModelDefs(), true);
 
         // Inspect arrays
         for (Map.Entry<String, ConfigDefinition.ArrayDef> entry : configDefinition.getArrayDefs().entrySet()) {
-            if (isFileOrPathArray(entry)) {
-                ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey());
-                sendFileEntries(array.getElements(), sentFiles);
-            }
+            if ( ! isAnyFileType(entry.getValue().getTypeSpec().getType())) continue;
+            ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey());
+            sendFileEntries(array.getElements(), sentFiles, "model".equals(entry.getValue().getTypeSpec().getType()));
         }
-        // Maps
+
+        // Inspect maps
         for (Map.Entry<String, ConfigDefinition.LeafMapDef> entry : configDefinition.getLeafMapDefs().entrySet()) {
-            if (isFileOrPathMap(entry)) {
-                ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey());
-                sendFileEntries(map.getElements(), sentFiles);
-            }
+            if ( ! isAnyFileType(entry.getValue().getTypeSpec().getType())) continue;
+            ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey());
+            sendFileEntries(map.getElements(), sentFiles, "model".equals(entry.getValue().getTypeSpec().getType()));
         }
 
         // Inspect inner fields
@@ -98,45 +99,56 @@ public class FileSender implements Serializable {
                 sendUserConfiguredFiles(element, sentFiles, key);
             }
         }
-
-    }
-
-    private static boolean isFileOrPathMap(Map.Entry<String, ConfigDefinition.LeafMapDef> entry) {
-        String mapType = entry.getValue().getTypeSpec().getType();
-        return ("file".equals(mapType) || "path".equals(mapType));
     }
 
-    private static boolean isFileOrPathArray(Map.Entry<String, ConfigDefinition.ArrayDef> entry) {
-        String arrayType = entry.getValue().getTypeSpec().getType();
-        return ("file".equals(arrayType) || "path".equals(arrayType));
+    private static boolean isAnyFileType(String type) {
+        return "file".equals(type) || "path".equals(type) || "model".equals(type);
     }
 
     private void sendEntries(ConfigPayloadBuilder builder,
                              Map<Path, FileReference> sentFiles,
-                             Map<String, ? extends DefaultValued<String>> entries) {
+                             Map<String, ?> entries,
+                             boolean isModelType) {
         for (String name : entries.keySet()) {
             ConfigPayloadBuilder fileEntry = builder.getObject(name);
             if (fileEntry.getValue() == null)
                 throw new IllegalArgumentException("Unable to send file for field '" + name +
                                                    "': Invalid config value " + fileEntry.getValue());
-            sendFileEntry(fileEntry, sentFiles);
+            sendFileEntry(fileEntry, sentFiles, isModelType);
         }
     }
 
-    private void sendFileEntries(Collection<ConfigPayloadBuilder> builders, Map<Path, FileReference> sentFiles) {
+    private void sendFileEntries(Collection<ConfigPayloadBuilder> builders, Map<Path, FileReference> sentFiles, boolean isModelType) {
         for (ConfigPayloadBuilder builder : builders) {
-            sendFileEntry(builder, sentFiles);
+            sendFileEntry(builder, sentFiles, isModelType);
         }
     }
 
-    private void sendFileEntry(ConfigPayloadBuilder builder, Map<Path, FileReference> sentFiles) {
-        Path path = Path.fromString(builder.getValue());
+    private void sendFileEntry(ConfigPayloadBuilder builder, Map<Path, FileReference> sentFiles, boolean isModelType) {
+        Path path;
+        if (isModelType) {
+            var modelReference = ModelReference.valueOf(builder.getValue());
+            if (modelReference.path().isEmpty()) return;
+            path = Path.fromString(modelReference.path().get().value());
+        }
+        else {
+            path = Path.fromString(builder.getValue());
+        }
+
         FileReference reference = sentFiles.get(path);
         if (reference == null) {
             reference = fileRegistry.addFile(path.getRelative());
             sentFiles.put(path, reference);
         }
-        builder.setValue(reference.value());
+
+        if (isModelType) {
+            var model = ModelReference.valueOf(builder.getValue());
+            var modelWithReference = ModelReference.unresolved(model.modelId(), model.url(), Optional.of(reference));
+            builder.setValue(modelWithReference.toString());
+        }
+        else {
+            builder.setValue(reference.value());
+        }
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java
index c7358ff1d7e..e113e53f541 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.model.utils;
 
 import com.yahoo.config.FileNode;
 import com.yahoo.config.FileReference;
+import com.yahoo.config.ModelReference;
+import com.yahoo.config.UrlReference;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
@@ -22,6 +24,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -106,6 +109,42 @@ public class FileSenderTest {
     }
 
     @Test
+    void require_that_simple_model_field_with_just_path_is_modified() {
+        var originalValue = ModelReference.unresolved(new FileReference("myModel.onnx"));
+        def.addModelDef("modelVal");
+        builder.setField("modelVal", originalValue.toString());
+        assertFileSent("myModel.onnx", originalValue);
+    }
+
+    @Test
+    void require_that_simple_model_field_with_path_and_url_is_modified() {
+        var originalValue = ModelReference.unresolved(Optional.empty(),
+                                                      Optional.of(new UrlReference("myUrl")),
+                                                      Optional.of(new FileReference("myModel.onnx")));
+        def.addModelDef("modelVal");
+        builder.setField("modelVal", originalValue.toString());
+        assertFileSent("myModel.onnx", originalValue);
+    }
+
+    @Test
+    void require_that_simple_model_field_with_just_url_is_not_modified() {
+        var originalValue = ModelReference.unresolved(new UrlReference("myUrl"));
+        def.addModelDef("modelVal");
+        builder.setField("modelVal",originalValue.toString());
+        fileSender().sendUserConfiguredFiles(producer);
+        assertEquals(originalValue, ModelReference.valueOf(builder.getObject("modelVal").getValue()));
+    }
+
+    private void assertFileSent(String path, ModelReference originalValue) {
+        fileRegistry.pathToRef.put(path, new FileNode("myModelHash").value());
+        fileSender().sendUserConfiguredFiles(producer);
+        var expected = ModelReference.unresolved(originalValue.modelId(),
+                                                 originalValue.url(),
+                                                 Optional.of(new FileReference("myModelHash")));
+        assertEquals(expected, ModelReference.valueOf(builder.getObject("modelVal").getValue()));
+    }
+
+    @Test
     void require_that_fields_in_inner_arrays_are_modified() {
         def.innerArrayDef("inner").addFileDef("fileVal");
         def.innerArrayDef("inner").addStringDef("stringVal");
@@ -119,6 +158,25 @@ public class FileSenderTest {
     }
 
     @Test
+    void require_that_paths_and_model_fields_are_modified() {
+        def.arrayDef("fileArray").setTypeSpec(new ConfigDefinition.TypeSpec("fileArray", "file", null, null, null, null));
+        def.arrayDef("pathArray").setTypeSpec(new ConfigDefinition.TypeSpec("pathArray", "path", null, null, null, null));
+        def.arrayDef("stringArray").setTypeSpec(new ConfigDefinition.TypeSpec("stringArray", "string", null, null, null, null));
+        builder.getArray("fileArray").append("foo.txt");
+        builder.getArray("fileArray").append("bar.txt");
+        builder.getArray("pathArray").append("path.txt");
+        builder.getArray("stringArray").append("foo.txt");
+        fileRegistry.pathToRef.put("foo.txt", new FileNode("foohash").value());
+        fileRegistry.pathToRef.put("bar.txt", new FileNode("barhash").value());
+        fileRegistry.pathToRef.put("path.txt", new FileNode("pathhash").value());
+        fileSender().sendUserConfiguredFiles(producer);
+        assertEquals("foohash", builder.getArray("fileArray").get(0).getValue());
+        assertEquals("barhash", builder.getArray("fileArray").get(1).getValue());
+        assertEquals("pathhash", builder.getArray("pathArray").get(0).getValue());
+        assertEquals("foo.txt", builder.getArray("stringArray").get(0).getValue());
+    }
+
+    @Test
     void require_that_arrays_are_modified() {
         def.arrayDef("fileArray").setTypeSpec(new ConfigDefinition.TypeSpec("fileArray", "file", null, null, null, null));
         def.arrayDef("pathArray").setTypeSpec(new ConfigDefinition.TypeSpec("pathArray", "path", null, null, null, null));