Merge pull request #4785 from vespa-engine/balder/add-uri-support-to-tensor-constants-3

Balder/add uri support to tensor constants 3

18 files changed, 290 insertions, 19 deletions

diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
index d635fe90ded..515477641a8 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java
@@ -35,4 +35,9 @@ public class MockFileRegistry implements FileRegistry {
         return result;
     }
 
+    @Override
+    public FileReference addUri(String uri) {
+        throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName());
+    }
+
 }
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java
index 0b0b799f47f..ed85b987a3d 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java
@@ -74,6 +74,11 @@ public class PreGeneratedFileRegistry implements FileRegistry {
     }
 
     @Override
+    public FileReference addUri(String uri) {
+        return new FileReference(path2Hash.get(uri));
+    }
+
+    @Override
     public String fileSourceHost() {
         return fileSourceHost;
     }
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java
index 887da2d51c8..15ae4294762 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java
@@ -2,16 +2,17 @@
 package com.yahoo.config.application.api;
 
 import java.util.List;
-import java.util.Set;
 
 import com.yahoo.config.FileReference;
 
+
 /**
  * @author tonytv
  */
 public interface FileRegistry {
 
     FileReference addFile(String relativePath);
+    FileReference addUri(String uri);
 
     /**
      * Returns the name of the host which is the source of the files
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
index c65e0fad1c7..a2bdc6834c9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
@@ -12,12 +12,20 @@ import java.util.Objects;
  */
 public class RankingConstant {
 
+    public enum PathType {FILE, URI};
+
     /** The search definition-unique name of this constant */
     private final String name;
     private TensorType tensorType = null;
-    private String fileName = null;
+    private String path = null;
     private String fileReference = "";
 
+    public PathType getPathType() {
+        return pathType;
+    }
+
+    private PathType pathType = PathType.FILE;
+
     public RankingConstant(String name) {
         this.name = name;
     }
@@ -25,13 +33,20 @@ public class RankingConstant {
     public RankingConstant(String name, TensorType type, String fileName) {
         this(name);
         this.tensorType = type;
-        this.fileName = fileName;
+        this.path = fileName;
         validate();
     }
 
     public void setFileName(String fileName) {
         Objects.requireNonNull(fileName, "Filename cannot be null");
-        this.fileName = fileName;
+        this.path = fileName;
+        this.pathType = PathType.FILE;
+    }
+
+    public void setUri(String uri) {
+        Objects.requireNonNull(uri, "uri cannot be null");
+        this.path = uri;
+        this.pathType = PathType.URI;
     }
 
     /**
@@ -43,14 +58,15 @@ public class RankingConstant {
     public void setType(TensorType tensorType) { this.tensorType = tensorType; }
 
     public String getName() { return name; }
-    public String getFileName() { return fileName; }
+    public String getFileName() { return path; }
+    public String getUri() { return path; }
     public String getFileReference() { return fileReference; }
     public TensorType getTensorType() { return tensorType; }
     public String getType() { return tensorType.toString(); }
 
     public void validate() {
-        if (fileName == null || fileName.isEmpty())
-            throw new IllegalArgumentException("Ranking constants must have a file.");
+        if (path == null || path.isEmpty())
+            throw new IllegalArgumentException("Ranking constants must have a file or uri.");
         if (tensorType == null)
             throw new IllegalArgumentException("Ranking constant '" + name + "' must have a type.");
     }
@@ -58,7 +74,7 @@ public class RankingConstant {
     public String toString() {
         StringBuilder b = new StringBuilder();
         b.append("constant '").append(name)
-         .append("' from file '").append(fileName)
+         .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path)
          .append("' with ref '").append(fileReference)
          .append("' of type '").append(tensorType)
          .append("'");
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
index 3b75be5167d..65f7bbedc68 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -508,6 +508,13 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
             throw new RuntimeException("File does not exist: '" + relativePath + "'.");
         }
     }
+    public FileReference sendUri(String uri) {
+        try {
+            return getRoot().getFileDistributor().sendUriToHost(uri, getHost());
+        } catch (PathDoesNotExistException e) {
+            throw new RuntimeException("Uri does not exist: '" + uri + "'.");
+        }
+    }
 
     /**
      *
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
index 520ff231921..d022b2cf8ab 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
@@ -63,15 +63,19 @@ public class RankingConstantsValidator extends Validator {
     }
 
     private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage application) throws FileNotFoundException {
-        String constantFile = rankingConstant.getFileName();
-        if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) &&
-            constantFile.startsWith(FilesApplicationPackage.preprocessed))
-            constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length());
+        // TODO: Handle validation of URI soon too.
+        if (rankingConstant.getPathType() == RankingConstant.PathType.FILE) {
+            String constantFile = rankingConstant.getFileName();
+            if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) &&
+                    constantFile.startsWith(FilesApplicationPackage.preprocessed)) {
+                constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length());
+            }
 
-        ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile));
-        new ConstantTensorJsonValidator().validate(constantFile,
-                                                   rankingConstant.getTensorType(),
-                                                   tensorApplicationFile.createReader());
+            ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile));
+            new ConstantTensorJsonValidator().validate(constantFile,
+                    rankingConstant.getTensorType(),
+                    tensorApplicationFile.createReader());
+        }
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
index 213451da55e..e8d6a330358 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -38,11 +38,31 @@ public class FileDistributor {
         return reference;
     }
 
+    /**
+     * Adds the given file to the associated application packages' registry of file and marks the file
+     * for distribution to the given hosts.
+     * <b>Note: This class receives ownership of the given collection.</b>
+     *
+     * @return the reference to the file, created by the application package
+     */
+    public FileReference sendUriToHosts(String uri, Collection<Host> hosts) {
+        FileReference reference = fileRegistry.addUri(uri);
+        if (reference != null) {
+            addToFilesToDistribute(reference, hosts);
+        }
+
+        return reference;
+    }
+
     /** Same as sendFileToHost(relativePath,Collections.singletonList(host) */
     public FileReference sendFileToHost(String relativePath, Host host) {
         return sendFileToHosts(relativePath, Arrays.asList(host));
     }
 
+    public FileReference sendUriToHost(String uri, Host host) {
+        return sendUriToHosts(uri, Arrays.asList(host));
+    }
+
     private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) {
         Set<Host> oldHosts = getHosts(reference);
         oldHosts.addAll(hosts);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
index 58fc76f1508..9550cd82b22 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
@@ -38,7 +38,9 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
     public void prepareToDistributeFiles(List<SearchNode> backends) {
         for (SearchDefinitionSpec sds : localSDS) {
             for (RankingConstant constant : sds.getSearchDefinition().getSearch().getRankingConstants().values()) {
-                FileReference reference = FileSender.sendFileToServices(constant.getFileName(), backends);
+                FileReference reference = (constant.getPathType() == RankingConstant.PathType.FILE)
+                        ? FileSender.sendFileToServices(constant.getFileName(), backends)
+                        : FileSender.sendUriToServices(constant.getUri(), backends);
                 constant.setFileReference(reference.value());
             }
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
index 413363d7b0d..8995fcbca99 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
@@ -20,6 +20,8 @@ import java.util.*;
  */
 public class FileSender implements Serializable {
 
+    public enum FileType {FILE, URI};
+
     /**
      * Send the given file to all given services.
      *
@@ -34,6 +36,7 @@ public class FileSender implements Serializable {
             throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
                                             "using 'count' instead of <node> tags.");
         }
+
         FileReference fileref = null;
         for (AbstractService service : services) {
             // The same reference will be returned from each call.
@@ -42,6 +45,20 @@ public class FileSender implements Serializable {
         return fileref;
     }
 
+    public static FileReference sendUriToServices(String uri, Collection<? extends AbstractService> services) {
+        if (services.isEmpty()) {
+            throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
+                    "using 'count' instead of <node> tags.");
+        }
+
+        FileReference fileref = null;
+        for (AbstractService service : services) {
+            // The same reference will be returned from each call.
+            fileref = service.sendUri(uri);
+        }
+        return fileref;
+    }
+
     /**
      * Sends all user configured files for a producer to all given services.
      */
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 916a905dfcb..bf6376983a4 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -339,6 +339,7 @@ TOKEN :
 | < RANKSCOREDROPLIMIT: "rank-score-drop-limit" >
 | < CONSTANTS: "constants" >
 | < FILE: "file" >
+| < URI: "uri" >
 | < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* >
 | < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" >
 | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* >
@@ -347,6 +348,8 @@ TOKEN :
 | < LONG: ("-")? (["0"-"9"])+"L" >
 | < STRING: (["a"-"z","A"-"Z","_","0"-"9","."])+ >
 | < FILE_PATH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-", "/", "."])+ >
+| < HTTP: ["h","H"] ["t","T"] ["t","T"] ["p","P"] >
+| < URI_PATH: <HTTP> <COLON> ("//")? (["a"-"z","A"-"Z","0"-"9","_","-", "/", ".",":"])+ >
 | < LESSTHAN: "<" >
 | < GREATERTHAN: ">" >
 | < VARIABLE: "$" <IDENTIFIER> >
@@ -1805,11 +1808,12 @@ void rankingConstant(Search search) :
  */
 Object rankingConstantItem(RankingConstant constant) :
 {
-    String fileName = null;
+    String path = null;
     TensorType type = null;
 }
 {
-    ( (<FILE> <COLON> fileName = filePath() { } (<NL>)*) { constant.setFileName(fileName); }
+    ( (<FILE> <COLON> path = filePath() { } (<NL>)*) { constant.setFileName(path); }
+      | (<URI> <COLON> path = uriPath() { } (<NL>)*) { constant.setUri(path); }
       | type = tensorTypeWithPrefix(rankingConstantErrorMessage(constant.getName())) (<NL>)* { constant.setType(type); }
     )
     {
@@ -1828,6 +1832,12 @@ String filePath() : { }
     { return token.image; }
 }
 
+String uriPath() : { }
+{
+    ( <URI_PATH> )
+    { return token.image; }
+}
+
 /**
  * Consumes a rank-profile block of a search element.
  *
@@ -2550,6 +2560,7 @@ String identifier() : { }
       | <TRUE>
       | <TYPE>
       | <UCA>
+      | <URI>
       | <UPPERBOUND>
       | <USEDOCUMENT>
       | <VARIABLE>
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java
index 2880af9e74f..9bad5373191 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition;
 
+import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -48,6 +49,7 @@ public class RankingConstantTest {
         assertEquals(TENSOR_NAME, constant.getName());
         assertEquals(TENSOR_FILE, constant.getFileName());
         assertEquals(TENSOR_TYPE, constant.getType());
+        assertEquals(RankingConstant.PathType.FILE, constant.getPathType());
 
         assertFalse(constantIterator.hasNext());
     }
@@ -103,4 +105,80 @@ public class RankingConstantTest {
         assertEquals("simplename", constant.getFileName());
     }
 
+    @Test
+    public void constant_uri_is_allowed() throws Exception {
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
+        searchBuilder.importString(joinLines(
+                "search test {",
+                "  document test { }",
+                "  constant foo {",
+                "    type: tensor(x{})",
+                "    uri: http://somwhere.far.away/in/another-galaxy",
+                "  }",
+                "}"
+        ));
+        searchBuilder.build();
+        Search search = searchBuilder.getSearch();
+        RankingConstant constant = search.getRankingConstants().values().iterator().next();
+        assertEquals(RankingConstant.PathType.URI, constant.getPathType());
+        assertEquals("http://somwhere.far.away/in/another-galaxy", constant.getUri());
+    }
+    @Test
+    public void constant_uri_with_port_is_allowed() throws Exception {
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
+        searchBuilder.importString(joinLines(
+                "search test {",
+                "  document test { }",
+                "  constant foo {",
+                "    type: tensor(x{})",
+                "    uri: http://somwhere.far.away:4080/in/another-galaxy",
+                "  }",
+                "}"
+        ));
+        searchBuilder.build();
+        Search search = searchBuilder.getSearch();
+        RankingConstant constant = search.getRankingConstants().values().iterator().next();
+        assertEquals(RankingConstant.PathType.URI, constant.getPathType());
+        assertEquals("http://somwhere.far.away:4080/in/another-galaxy", constant.getUri());
+    }
+    @Test
+    public void constant_uri_no_dual_slashes_is_allowed() throws Exception {
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
+        searchBuilder.importString(joinLines(
+                "search test {",
+                "  document test { }",
+                "  constant foo {",
+                "    type: tensor(x{})",
+                "    uri: http:somwhere.far.away/in/another-galaxy",
+                "  }",
+                "}"
+        ));
+        searchBuilder.build();
+        Search search = searchBuilder.getSearch();
+        RankingConstant constant = search.getRankingConstants().values().iterator().next();
+        assertEquals(RankingConstant.PathType.URI, constant.getPathType());
+        assertEquals("http:somwhere.far.away/in/another-galaxy", constant.getUri());
+    }
+    @Test
+    public void constant_uri_only_supports_http() throws Exception {
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
+        thrown.expect(ParseException.class);
+        thrown.expectMessage("Encountered \" <IDENTIFIER> \"ftp \"\" at line 5, column 10.\n" +
+                "Was expecting:\n" +
+                "    <URI_PATH> ...");
+        searchBuilder.importString(joinLines(
+                "search test {",
+                "  document test { }",
+                "  constant foo {",
+                "    type: tensor(x{})",
+                "    uri: ftp:somwhere.far.away/in/another-galaxy",
+                "  }",
+                "}"
+        ));
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
index 61c376a7256..39bf2ee02fd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.yahoo.config.FileReference;
 
 public interface AddFileInterface {
+    FileReference addUri(String uri, String relativePath);
+    FileReference addUri(String uri, String relativePath, FileReference reference);
     FileReference addFile(String relativePath);
     FileReference addFile(String relativePath, FileReference reference);
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
index 82535143c89..0e7aa4a4fd2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
@@ -3,6 +3,12 @@ package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.Files;
 
 public class ApplicationFileManager implements AddFileInterface {
 
@@ -24,4 +30,41 @@ public class ApplicationFileManager implements AddFileInterface {
         return master.addFile(new File(applicationDir, relativePath));
     }
 
+    @Override
+    public FileReference addUri(String uri, String relativePath) {
+        download(uri, relativePath);
+        return addFile(relativePath);
+    }
+
+    @Override
+    public FileReference addUri(String uri, String relativePath, FileReference reference) {
+        download(uri, relativePath);
+        return addFile(relativePath, reference);
+    }
+
+    void download(String uri, String relativePath) {
+        File file = new File(applicationDir, relativePath);
+        FileOutputStream fos = null;
+        ReadableByteChannel rbc = null;
+        try {
+            Files.createDirectories(file.toPath().getParent());
+            URL website = new URL(uri);
+            rbc = Channels.newChannel(website.openStream());
+            fos = new FileOutputStream(file.getAbsolutePath());
+            fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Failed creating directory " + file.getParent(), e);
+        } finally {
+            try {
+                if (fos != null) {
+                    fos.close();
+                }
+                if (rbc != null) {
+                    rbc.close();
+                }
+            } catch (IOException e) {
+                throw new IllegalArgumentException("Failed closing down after downloading " + uri + " to " + file.getAbsolutePath());
+            }
+        }
+    }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java
index 8f2cb194bbd..1fe72e27461 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java
@@ -21,6 +21,12 @@ public class CombinedLegacyRegistry implements FileRegistry {
     }
 
     @Override
+    public FileReference addUri(String uri) {
+        FileReference reference = future.addUri(uri);
+        return reference;
+    }
+
+    @Override
     public String fileSourceHost() {
         return future.fileSourceHost();
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
index 1a76454fbed..b0a802d831d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
@@ -4,7 +4,10 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.net.HostName;
+import com.yahoo.text.Utf8;
+import net.jpountz.xxhash.XXHashFactory;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -34,6 +37,17 @@ public class FileDBRegistry implements FileRegistry {
         });
     }
 
+    public synchronized FileReference addUri(String uri, FileReference reference) {
+        String relativePath = uriToRelativeFile(uri);
+        Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri));
+        return cachedReference.orElseGet(() -> {
+            FileReference newRef = manager.addUri(uri, relativePath, reference);
+            entries.add(new Entry(uri, newRef));
+            fileReferenceCache.put(uri, newRef);
+            return newRef;
+        });
+    }
+
     @Override
     public synchronized FileReference addFile(String relativePath) {
         Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath));
@@ -46,6 +60,18 @@ public class FileDBRegistry implements FileRegistry {
     }
 
     @Override
+    public synchronized FileReference addUri(String uri) {
+        String relativePath = uriToRelativeFile(uri);
+        Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri));
+        return cachedReference.orElseGet(() -> {
+            FileReference newRef = manager.addUri(uri, relativePath);
+            entries.add(new Entry(uri, newRef));
+            fileReferenceCache.put(uri, newRef);
+            return newRef;
+        });
+    }
+
+    @Override
     public String fileSourceHost() {
         return HostName.getLocalhost();
     }
@@ -55,4 +81,16 @@ public class FileDBRegistry implements FileRegistry {
         return entries;
     }
 
+    private static String uriToRelativeFile(String uri) {
+        String relative = "uri/" + String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0));
+        if (uri.endsWith(".json")) {
+            relative += ".json";
+        } else if (uri.endsWith(".json.lz4")) {
+            relative += ".json.lz4";
+        } else if (uri.endsWith(".lz4")) {
+            relative += ".lz4";
+        }
+        return relative;
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
index 6c2da338ef0..117bf3e236b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
@@ -26,6 +26,17 @@ public class FileDistributionProvider {
         ManagerWrapper(FileDistributionManager manager) {
             this.manager = manager;
         }
+
+        @Override
+        public FileReference addUri(String uri, String relativePath) {
+            throw new IllegalStateException("addUri is not possible with legacy filedistribution.");
+        }
+
+        @Override
+        public FileReference addUri(String uri, String relativePath, FileReference reference) {
+            throw new IllegalStateException("addUri is not possible with legacy filedistribution.");
+        }
+
         @Override
         public FileReference addFile(String relativePath) {
             return new FileReference(manager.addFile(relativePath));
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index ea8405f6b65..731e343532a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -104,6 +104,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
                         log.log(LogLevel.WARNING, "Unexpected error when building model ", e);
                         throw new InternalServerException(applicationId + ": Error loading model", e);
                     } else {
+                        log.log(LogLevel.WARNING, "Input error when building model ", e);
                         throw new IllegalArgumentException(applicationId + ": Error loading model", e);
                     }
                 } else {
diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala
index 69443c73b3a..6507b4c72f0 100644
--- a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala
+++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala
@@ -58,6 +58,10 @@ class LocalFileDb(appPath: Path) extends FileAcquirer with FileRegistry {
   override def export(): util.List[Entry] = {
     new java.util.ArrayList(fileReferenceToFile.keys.map{ (ref: FileReference) => new Entry(fileReferenceToFile.get(ref).get.getPath, ref)}.asJavaCollection)
   }
+
+  override def addUri(uri: String): FileReference = {
+    throw new RuntimeException("addUri(uri: String) is not implemented here.");
+  }
 }
 
 object LocalFileDb {