Merge pull request #4980 from vespa-engine/hmusum/walk-file-tree-to-create-hash-when-filereference-is-a-directory

Walk file tree to create hash when file reference is a directory

2 files changed, 40 insertions, 10 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
index 9249d419379..432cbc4d997 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
@@ -77,8 +77,24 @@ public class FileDirectory  {
     }
 
     private Long computeReference(File file) throws IOException {
-        byte [] wholeFile = IOUtils.readFileBytes(file);
         XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
+        if (file.isDirectory()) {
+            return Files.walk(file.toPath(), 100).map(path -> {
+                try {
+                    log.log(LogLevel.DEBUG, "Calculating hash for '" + path + "'");
+                    return hash(file, hasher);
+                } catch (IOException e) {
+                    log.log(LogLevel.WARNING, "Failed getting hash from '" + path + "'");
+                    return 0;
+                }
+            }).mapToLong(Number::longValue).sum();
+        } else {
+            return hash(file, hasher);
+        }
+    }
+
+    private long hash(File file, XXHash64 hasher) throws IOException {
+        byte[] wholeFile = file.isDirectory() ?  new byte[0] : IOUtils.readFileBytes(file);
         return hasher.hash(ByteBuffer.wrap(wholeFile), hasher.hash(ByteBuffer.wrap(Utf8.toBytes(file.getName())), 0));
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java
index ad807f9527f..9b1d018f892 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java
@@ -12,6 +12,8 @@ import org.junit.rules.TemporaryFolder;
 import java.io.File;
 import java.io.IOException;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 public class FileDirectoryTest {
@@ -32,7 +34,9 @@ public class FileDirectoryTest {
         FileReference bar = createFile("bar");
 
         assertTrue(fileDirectory.getFile(foo).exists());
+        assertEquals("ea315b7acac56246", foo.value());
         assertTrue(fileDirectory.getFile(bar).exists());
+        assertEquals("2b8e97f15c854e1d", bar.value());
     }
 
 
@@ -40,11 +44,24 @@ public class FileDirectoryTest {
     public void requireThatFileReferenceWithSubDirectoriesWorks() throws IOException {
         FileDirectory fileDirectory = new FileDirectory(temporaryFolder.getRoot());
 
-        FileReference foo = createFileInSubDir("subdir", "foo");
-        FileReference bar = createFileInSubDir("subdir", "bar");
-
-        assertTrue(fileDirectory.getFile(foo).exists());
-        assertTrue(fileDirectory.getFile(bar).exists());
+        String subdirName = "subdir";
+        File subDirectory = new File(temporaryFolder.getRoot(), subdirName);
+        createFileInSubDir(subDirectory, "foo");
+        FileReference fileReference = fileDirectory.addFile(subDirectory);
+        File dir = fileDirectory.getFile(fileReference);
+        assertTrue(dir.exists());
+        assertTrue(new File(dir, "foo").exists());
+        assertFalse(new File(dir, "doesnotexist").exists());
+        assertEquals("1315a322fc323608", fileReference.value());
+
+
+        // Add a file, should be available and file reference should have another value
+        createFileInSubDir(subDirectory, "bar");
+        fileReference = fileDirectory.addFile(subDirectory);
+        dir = fileDirectory.getFile(fileReference);
+        assertTrue(new File(dir, "foo").exists());
+        assertTrue(new File(dir, "bar").exists());
+        assertEquals("9ca074b47a4b510c", fileReference.value());
     }
 
     // Content in created file is equal to the filename string
@@ -54,16 +71,13 @@ public class FileDirectoryTest {
         return fileDirectory.addFile(file);
     }
 
-    private FileReference createFileInSubDir(String subdirName, String filename) throws IOException {
-        File subDirectory = new File(temporaryFolder.getRoot(), subdirName);
+    private void createFileInSubDir(File subDirectory, String filename) throws IOException {
         if (!subDirectory.exists())
             subDirectory.mkdirs();
         File file = new File(subDirectory, filename);
         IOUtils.writeFile(file, filename, false);
-        return fileDirectory.addFile(file);
     }
 
-
 }