Merge pull request #4785 from vespa-engine/balder/add-uri-support-to-tensor-constants-3

Balder/add uri support to tensor constants 3

6 files changed, 101 insertions, 0 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
index 61c376a7256..39bf2ee02fd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.yahoo.config.FileReference;
 
 public interface AddFileInterface {
+    FileReference addUri(String uri, String relativePath);
+    FileReference addUri(String uri, String relativePath, FileReference reference);
     FileReference addFile(String relativePath);
     FileReference addFile(String relativePath, FileReference reference);
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
index 82535143c89..0e7aa4a4fd2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
@@ -3,6 +3,12 @@ package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.Files;
 
 public class ApplicationFileManager implements AddFileInterface {
 
@@ -24,4 +30,41 @@ public class ApplicationFileManager implements AddFileInterface {
         return master.addFile(new File(applicationDir, relativePath));
     }
 
+    @Override
+    public FileReference addUri(String uri, String relativePath) {
+        download(uri, relativePath);
+        return addFile(relativePath);
+    }
+
+    @Override
+    public FileReference addUri(String uri, String relativePath, FileReference reference) {
+        download(uri, relativePath);
+        return addFile(relativePath, reference);
+    }
+
+    void download(String uri, String relativePath) {
+        File file = new File(applicationDir, relativePath);
+        FileOutputStream fos = null;
+        ReadableByteChannel rbc = null;
+        try {
+            Files.createDirectories(file.toPath().getParent());
+            URL website = new URL(uri);
+            rbc = Channels.newChannel(website.openStream());
+            fos = new FileOutputStream(file.getAbsolutePath());
+            fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Failed creating directory " + file.getParent(), e);
+        } finally {
+            try {
+                if (fos != null) {
+                    fos.close();
+                }
+                if (rbc != null) {
+                    rbc.close();
+                }
+            } catch (IOException e) {
+                throw new IllegalArgumentException("Failed closing down after downloading " + uri + " to " + file.getAbsolutePath());
+            }
+        }
+    }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java
index 8f2cb194bbd..1fe72e27461 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java
@@ -21,6 +21,12 @@ public class CombinedLegacyRegistry implements FileRegistry {
     }
 
     @Override
+    public FileReference addUri(String uri) {
+        FileReference reference = future.addUri(uri);
+        return reference;
+    }
+
+    @Override
     public String fileSourceHost() {
         return future.fileSourceHost();
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
index 1a76454fbed..b0a802d831d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
@@ -4,7 +4,10 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.net.HostName;
+import com.yahoo.text.Utf8;
+import net.jpountz.xxhash.XXHashFactory;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -34,6 +37,17 @@ public class FileDBRegistry implements FileRegistry {
         });
     }
 
+    public synchronized FileReference addUri(String uri, FileReference reference) {
+        String relativePath = uriToRelativeFile(uri);
+        Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri));
+        return cachedReference.orElseGet(() -> {
+            FileReference newRef = manager.addUri(uri, relativePath, reference);
+            entries.add(new Entry(uri, newRef));
+            fileReferenceCache.put(uri, newRef);
+            return newRef;
+        });
+    }
+
     @Override
     public synchronized FileReference addFile(String relativePath) {
         Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath));
@@ -46,6 +60,18 @@ public class FileDBRegistry implements FileRegistry {
     }
 
     @Override
+    public synchronized FileReference addUri(String uri) {
+        String relativePath = uriToRelativeFile(uri);
+        Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri));
+        return cachedReference.orElseGet(() -> {
+            FileReference newRef = manager.addUri(uri, relativePath);
+            entries.add(new Entry(uri, newRef));
+            fileReferenceCache.put(uri, newRef);
+            return newRef;
+        });
+    }
+
+    @Override
     public String fileSourceHost() {
         return HostName.getLocalhost();
     }
@@ -55,4 +81,16 @@ public class FileDBRegistry implements FileRegistry {
         return entries;
     }
 
+    private static String uriToRelativeFile(String uri) {
+        String relative = "uri/" + String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0));
+        if (uri.endsWith(".json")) {
+            relative += ".json";
+        } else if (uri.endsWith(".json.lz4")) {
+            relative += ".json.lz4";
+        } else if (uri.endsWith(".lz4")) {
+            relative += ".lz4";
+        }
+        return relative;
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
index 6c2da338ef0..117bf3e236b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
@@ -26,6 +26,17 @@ public class FileDistributionProvider {
         ManagerWrapper(FileDistributionManager manager) {
             this.manager = manager;
         }
+
+        @Override
+        public FileReference addUri(String uri, String relativePath) {
+            throw new IllegalStateException("addUri is not possible with legacy filedistribution.");
+        }
+
+        @Override
+        public FileReference addUri(String uri, String relativePath, FileReference reference) {
+            throw new IllegalStateException("addUri is not possible with legacy filedistribution.");
+        }
+
         @Override
         public FileReference addFile(String relativePath) {
             return new FileReference(manager.addFile(relativePath));
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index ea8405f6b65..731e343532a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -104,6 +104,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
                         log.log(LogLevel.WARNING, "Unexpected error when building model ", e);
                         throw new InternalServerException(applicationId + ": Error loading model", e);
                     } else {
+                        log.log(LogLevel.WARNING, "Input error when building model ", e);
                         throw new IllegalArgumentException(applicationId + ": Error loading model", e);
                     }
                 } else {