- Add testing of FileDBRegistry.

- Remove PreGeneratedRegistry.
- Allow files to not exist after bootstrap.

7 files changed, 137 insertions, 32 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
index 99632ec323e..bb99771b475 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
@@ -8,13 +8,13 @@ import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.application.api.UnparsedConfigDefinition;
-import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.serialization.AllocatedHostsSerializer;
 import com.yahoo.io.reader.NamedReader;
 import com.yahoo.path.Path;
 import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.ConfigDefinitionKey;
+import com.yahoo.vespa.config.server.filedistribution.FileDBRegistry;
 import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.yolean.Exceptions;
@@ -228,7 +228,7 @@ public class ZooKeeperClient {
     }
 
     private void write(Version vespaVersion, FileRegistry fileRegistry) {
-        String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry);
+        String exportedRegistry = FileDBRegistry.exportRegistry(fileRegistry);
         curator.set(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).append(vespaVersion.toFullString()),
                     Utf8.toBytes(exportedRegistry));
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
index 163c19abe75..1218ef6599d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -10,6 +11,6 @@ import java.nio.ByteBuffer;
  */
 public interface AddFileInterface {
     FileReference addUri(String uri, String relativePath);
-    FileReference addFile(String relativePath);
+    FileReference addFile(String relativePath) throws IOException;
     FileReference addBlob(ByteBuffer blob, String relativePath);
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
index a1907c01085..064cb6423da 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java
@@ -26,20 +26,28 @@ public class ApplicationFileManager implements AddFileInterface {
     }
 
     @Override
-    public FileReference addFile(String relativePath) {
+    public FileReference addFile(String relativePath) throws IOException {
         return fileDirectory.addFile(new File(applicationDir, relativePath));
     }
 
     @Override
     public FileReference addUri(String uri, String relativePath) {
         download(uri, relativePath);
-        return addFile(relativePath);
+        try {
+            return addFile(relativePath);
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
     }
 
     @Override
     public FileReference addBlob(ByteBuffer blob, String relativePath) {
         writeBlob(blob, relativePath);
-        return addFile(relativePath);
+        try {
+            return addFile(relativePath);
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
     }
 
     private void writeBlob(ByteBuffer blob, String relativePath) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
index 5ec83529601..0d0f6d09582 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.filedistribution;
 
+import com.google.common.collect.ImmutableMap;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.net.HostName;
@@ -8,6 +9,7 @@ import com.yahoo.text.Utf8;
 import net.jpountz.xxhash.XXHashFactory;
 
 import java.io.BufferedReader;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Reader;
 import java.nio.ByteBuffer;
@@ -23,15 +25,17 @@ import java.util.regex.Pattern;
  */
 public class FileDBRegistry implements FileRegistry {
 
+    private final boolean silenceNonExistingFiles;
     private final AddFileInterface manager;
     private final Map<String, FileReference> fileReferenceCache = new HashMap<>();
     private static final String entryDelimiter = "\t";
     private static final Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL);
 
     public FileDBRegistry(AddFileInterface manager) {
+        silenceNonExistingFiles = false;
         this.manager = manager;
     }
-    public static FileRegistry create(AddFileInterface manager, Reader persistedState) {
+    public static FileDBRegistry create(AddFileInterface manager, Reader persistedState) {
         try (BufferedReader reader = new BufferedReader(persistedState)) {
             String ignoredFileSourceHost = reader.readLine();
             if (ignoredFileSourceHost == null)
@@ -58,20 +62,30 @@ public class FileDBRegistry implements FileRegistry {
         return refs;
     }
     private FileDBRegistry(AddFileInterface manager, Map<String, FileReference> knownReferences) {
+        silenceNonExistingFiles = true;
         this.manager = manager;
-        for (Map.Entry<String, FileReference> e : knownReferences.entrySet()) {
-            fileReferenceCache.put(e.getKey(), e.getValue());
-        }
+        fileReferenceCache.putAll(knownReferences);
     }
 
     @Override
     public synchronized FileReference addFile(String relativePath) {
         Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath));
         return cachedReference.orElseGet(() -> {
-            FileReference newRef = manager.addFile(relativePath);
-            fileReferenceCache.put(relativePath, newRef);
-            return newRef;
-        });
+            try {
+                FileReference newRef = manager.addFile(relativePath);
+                fileReferenceCache.put(relativePath, newRef);
+                return newRef;
+            } catch (FileNotFoundException e) {
+                if (silenceNonExistingFiles) {
+                    return new FileReference("non-existing-file");
+                } else {
+                    throw new IllegalArgumentException(e);
+                }
+            } catch (IOException e) {
+                throw new IllegalArgumentException(e);
+            }
+            }
+        );
     }
 
     @Override
@@ -88,7 +102,7 @@ public class FileDBRegistry implements FileRegistry {
     @Override
     public FileReference addBlob(ByteBuffer blob) {
         String blobName = FileRegistry.blobName(blob);
-        String relativePath = blobToRelativeFile(blob, blobName);
+        String relativePath = blobToRelativeFile(blobName);
         synchronized (this) {
             Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName));
             return cachedReference.orElseGet(() -> {
@@ -108,6 +122,22 @@ public class FileDBRegistry implements FileRegistry {
         return entries;
     }
 
+    synchronized Map<String, FileReference> getMap() {
+        return ImmutableMap.copyOf(fileReferenceCache);
+    }
+
+    public static String exportRegistry(FileRegistry registry) {
+        List<Entry> entries = registry.export();
+        StringBuilder builder = new StringBuilder();
+
+        builder.append(HostName.getLocalhost()).append('\n');
+        for (FileRegistry.Entry entry : entries) {
+            builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()).append('\n');
+        }
+
+        return builder.toString();
+    }
+
     private static String uriToRelativeFile(String uri) {
         String relative = "uri/" + XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0);
         if (uri.endsWith(".json")) {
@@ -120,9 +150,8 @@ public class FileDBRegistry implements FileRegistry {
         return relative;
     }
 
-    private static String blobToRelativeFile(ByteBuffer blob, String blobName) {
-        String relative = "blob/" + blobName;
-        return relative;
+    private static String blobToRelativeFile(String blobName) {
+        return "blob/" + blobName;
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
index f7ba895666b..1fe8c741a17 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
@@ -97,15 +97,11 @@ public class FileDirectory  {
         return hasher.hash(ByteBuffer.wrap(wholeFile), hasher.hash(ByteBuffer.wrap(Utf8.toBytes(file.getName())), 0));
     }
 
-    public FileReference addFile(File source) {
-        try {
-            Long hash = computeHash(source);
-            verifyExistingFile(source, hash);
-            FileReference fileReference = fileReferenceFromHash(hash);
-            return addFile(source, fileReference);
-        } catch (IOException e) {
-            throw new IllegalArgumentException(e);
-        }
+    public FileReference addFile(File source) throws IOException {
+        Long hash = computeHash(source);
+        verifyExistingFile(source, hash);
+        FileReference fileReference = fileReferenceFromHash(hash);
+        return addFile(source, fileReference);
     }
 
     // If there exists a directory for a file reference, but content does not have correct hash or the file we want to add
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java
new file mode 100644
index 00000000000..10ac54dc757
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java
@@ -0,0 +1,66 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.filedistribution;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.config.application.api.FileRegistry;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author Tony Vaagenes
+ */
+public class FileDBRegistryTestCase {
+
+    private static final String BLOB = "Some blob";
+    private static final String APP = "src/test/apps/zkapp";
+    private static final String FOO_FILE = "files/foo.json";
+    private static final String NO_FOO_FILE = "files/no_foo.json";
+    private static final String BLOB_NAME = "c5674b55c15c9c95";
+    private static final FileReference BLOB_REF = new FileReference("b3424b78130fc005");
+    private static final FileReference FOO_REF = new FileReference("b5ce94ca1feae86c");
+    @Test
+    public void importAndExport() throws IOException {
+        TemporaryFolder tmpDir = new TemporaryFolder();
+        tmpDir.create();
+        AddFileInterface fileManager = new ApplicationFileManager(new File(APP), new FileDirectory(tmpDir.newFolder()));
+        FileRegistry fileRegistry = new FileDBRegistry(fileManager);
+        assertEquals(FOO_REF, fileRegistry.addFile(FOO_FILE));
+        try {
+            fileRegistry.addFile(NO_FOO_FILE);
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("src/test/apps/zkapp/files/no_foo.json (No such file or directory)", e.getCause().getMessage());
+        }
+        assertEquals(BLOB_REF, fileRegistry.addBlob(ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8))));
+        String serializedRegistry = FileDBRegistry.exportRegistry(fileRegistry);
+
+        FileDBRegistry importedRegistry = FileDBRegistry.create(fileManager, new StringReader(serializedRegistry));
+
+        assertEquals(Set.of(BLOB_NAME, FOO_FILE), importedRegistry.getMap().keySet());
+        assertEquals(BLOB_REF, importedRegistry.getMap().get(BLOB_NAME));
+        assertEquals(FOO_REF, importedRegistry.getMap().get(FOO_FILE));
+
+        assertEquals(2, importedRegistry.export().size());
+
+        checkConsistentEntry(fileRegistry.export().get(0), importedRegistry);
+        checkConsistentEntry(fileRegistry.export().get(1), importedRegistry);
+
+        assertEquals(new FileReference("non-existing-file"), importedRegistry.addFile(NO_FOO_FILE));
+        assertEquals(2, importedRegistry.export().size());
+        tmpDir.delete();
+    }
+
+    void checkConsistentEntry(FileRegistry.Entry entry, FileRegistry registry) {
+        assertEquals(entry.reference, registry.addFile(entry.relativePath));
+    }
+}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
index b11308214a3..8b3227e3a69 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java
@@ -3,10 +3,10 @@ package com.yahoo.vespa.config.server.filedistribution;
 
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.FileRegistry;
-import com.yahoo.net.HostName;
 import net.jpountz.xxhash.XXHashFactory;
 
 import java.io.File;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -30,11 +30,16 @@ public class MockFileRegistry implements FileRegistry {
     public FileReference addFile(String relativePath) {
         if (relativePath.isEmpty())
             relativePath = "./";
-        addFileInterface.addFile(relativePath);
 
-        FileReference fileReference = new FileReference(relativePath);
-        entries.add(new Entry(relativePath, fileReference));
-        return fileReference;
+        try {
+            addFileInterface.addFile(relativePath);
+
+            FileReference fileReference = new FileReference(relativePath);
+            entries.add(new Entry(relativePath, fileReference));
+            return fileReference;
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
     }
 
     public List<Entry> export() { return entries; }