diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-08-24 13:52:03 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-08-24 13:52:03 +0200 |
commit | 6708917a1699529e85be872b8c853da9a5298837 (patch) | |
tree | 8427fd4a407a8e09badcbec345ccc914ab07554a | |
parent | d27622ab42bcedb731289e26fcb4800f488dcda6 (diff) |
- Add testing of FileDBRegistry.
- Remove PreGeneratedRegistry.
- Allow files to not exist after bootstrap.
10 files changed, 139 insertions, 186 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java deleted file mode 100644 index 5a7f2db1968..00000000000 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.model.application.provider; - -import com.yahoo.config.FileReference; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.net.HostName; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.Reader; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; - -/** - * Registry of files added earlier (i.e. during deployment) - * - * @author Tony Vaagenes - */ -public class PreGeneratedFileRegistry implements FileRegistry { - - private final String fileSourceHost; - private final Map<String, FileReference> path2Hash; - - private static final String entryDelimiter = "\t"; - private static final Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL); - - public static Map<String, FileReference> decode(BufferedReader reader) { - Map<String, FileReference> refs = new HashMap<>(); - try { - String line; - while ((line = reader.readLine()) != null) { - String[] parts = entryDelimiterPattern.split(line); - if (parts.length < 2) - throw new IllegalArgumentException("Cannot split '" + line + "' into two parts"); - refs.put(parts[0], new FileReference(parts[1])); - } - } catch (IOException e) { - throw new RuntimeException("Error while reading pre-generated file registry", e); - } - return refs; - } - private PreGeneratedFileRegistry(Reader readerArg) { - try (BufferedReader reader = new BufferedReader(readerArg)) { - fileSourceHost = reader.readLine(); - if (fileSourceHost == null) - throw new RuntimeException("Error while reading pre-generated file registry"); - - path2Hash = decode(reader); - } catch (IOException e) { - throw new RuntimeException("Error while reading pre-generated file registry", e); - } - } - - public static String exportRegistry(FileRegistry registry) { - List<Entry> entries = registry.export(); - StringBuilder builder = new StringBuilder(); - - builder.append(HostName.getLocalhost()).append('\n'); - for (FileRegistry.Entry entry : entries) { - builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()).append('\n'); - } - - return builder.toString(); - } - - public static PreGeneratedFileRegistry importRegistry(Reader reader) { - return new PreGeneratedFileRegistry(reader); - } - - public FileReference addFile(String relativePath) { - FileReference reference = path2Hash.get(relativePath); - if (reference == null) { - throw new IllegalArgumentException("File '" + relativePath + "' not found"); - } - return reference; - } - - @Override - public FileReference addUri(String uri) { - FileReference reference = path2Hash.get(uri); - if (reference == null) { - throw new IllegalArgumentException("Uri '" + uri + "' not found"); - } - return reference; - } - @Override - public FileReference addBlob(ByteBuffer blob) { - String blobName = FileRegistry.blobName(blob); - FileReference reference = path2Hash.get(blobName); - if (reference == null) { - throw new IllegalArgumentException("Blob '" + blobName + "(" + blob.remaining()+ ")' not found"); - } - return reference; - } - - public Set<String> getPaths() { - return path2Hash.keySet(); - } - - @Override - public List<Entry> export() { - List<Entry> entries = new ArrayList<>(); - for (Map.Entry<String, FileReference> entry : path2Hash.entrySet()) { - entries.add(new Entry(entry.getKey(), entry.getValue())); - } - return entries; - } -} diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java deleted file mode 100644 index 532ebb1ab81..00000000000 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.model.application.provider; - -import com.yahoo.config.application.api.FileRegistry; -import org.junit.Test; - -import java.io.StringReader; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -/** - * @author Tony Vaagenes - */ -public class PreGeneratedFileRegistryTestCase { - - private static final String BLOB = "Some blob"; - @Test - public void importAndExport() { - FileRegistry fileRegistry = new MockFileRegistry(); - fileRegistry.addFile("1234"); - fileRegistry.addBlob(ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8))); - String serializedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); - - PreGeneratedFileRegistry importedRegistry = PreGeneratedFileRegistry.importRegistry(new StringReader(serializedRegistry)); - - assertEquals(Set.of("1234", "c5674b55c15c9c95.blob"), importedRegistry.getPaths()); - - assertEquals(2, importedRegistry.getPaths().size()); - - checkConsistentEntry(fileRegistry.export().get(0), importedRegistry); - checkConsistentEntry(fileRegistry.export().get(1), importedRegistry); - } - - void checkConsistentEntry(FileRegistry.Entry entry, FileRegistry registry) { - assertEquals(entry.reference, registry.addFile(entry.relativePath)); - } -} diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index 7e4a6aec53a..a0b1402f559 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -39,7 +39,9 @@ public interface FileRegistry { } static String blobName(ByteBuffer blob) { + blob.mark(); long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0); + blob.reset(); return Long.toHexString(blobHash); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 99632ec323e..bb99771b475 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -8,13 +8,13 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; -import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.server.filedistribution.FileDBRegistry; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.Curator; import com.yahoo.yolean.Exceptions; @@ -228,7 +228,7 @@ public class ZooKeeperClient { } private void write(Version vespaVersion, FileRegistry fileRegistry) { - String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); + String exportedRegistry = FileDBRegistry.exportRegistry(fileRegistry); curator.set(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).append(vespaVersion.toFullString()), Utf8.toBytes(exportedRegistry)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java index 163c19abe75..1218ef6599d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; +import java.io.IOException; import java.nio.ByteBuffer; /** @@ -10,6 +11,6 @@ import java.nio.ByteBuffer; */ public interface AddFileInterface { FileReference addUri(String uri, String relativePath); - FileReference addFile(String relativePath); + FileReference addFile(String relativePath) throws IOException; FileReference addBlob(ByteBuffer blob, String relativePath); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index a1907c01085..064cb6423da 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -26,20 +26,28 @@ public class ApplicationFileManager implements AddFileInterface { } @Override - public FileReference addFile(String relativePath) { + public FileReference addFile(String relativePath) throws IOException { return fileDirectory.addFile(new File(applicationDir, relativePath)); } @Override public FileReference addUri(String uri, String relativePath) { download(uri, relativePath); - return addFile(relativePath); + try { + return addFile(relativePath); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } } @Override public FileReference addBlob(ByteBuffer blob, String relativePath) { writeBlob(blob, relativePath); - return addFile(relativePath); + try { + return addFile(relativePath); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } } private void writeBlob(ByteBuffer blob, String relativePath) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index 5ec83529601..0d0f6d09582 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; +import com.google.common.collect.ImmutableMap; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.net.HostName; @@ -8,6 +9,7 @@ import com.yahoo.text.Utf8; import net.jpountz.xxhash.XXHashFactory; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; import java.nio.ByteBuffer; @@ -23,15 +25,17 @@ import java.util.regex.Pattern; */ public class FileDBRegistry implements FileRegistry { + private final boolean silenceNonExistingFiles; private final AddFileInterface manager; private final Map<String, FileReference> fileReferenceCache = new HashMap<>(); private static final String entryDelimiter = "\t"; private static final Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL); public FileDBRegistry(AddFileInterface manager) { + silenceNonExistingFiles = false; this.manager = manager; } - public static FileRegistry create(AddFileInterface manager, Reader persistedState) { + public static FileDBRegistry create(AddFileInterface manager, Reader persistedState) { try (BufferedReader reader = new BufferedReader(persistedState)) { String ignoredFileSourceHost = reader.readLine(); if (ignoredFileSourceHost == null) @@ -58,20 +62,30 @@ public class FileDBRegistry implements FileRegistry { return refs; } private FileDBRegistry(AddFileInterface manager, Map<String, FileReference> knownReferences) { + silenceNonExistingFiles = true; this.manager = manager; - for (Map.Entry<String, FileReference> e : knownReferences.entrySet()) { - fileReferenceCache.put(e.getKey(), e.getValue()); - } + fileReferenceCache.putAll(knownReferences); } @Override public synchronized FileReference addFile(String relativePath) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath)); return cachedReference.orElseGet(() -> { - FileReference newRef = manager.addFile(relativePath); - fileReferenceCache.put(relativePath, newRef); - return newRef; - }); + try { + FileReference newRef = manager.addFile(relativePath); + fileReferenceCache.put(relativePath, newRef); + return newRef; + } catch (FileNotFoundException e) { + if (silenceNonExistingFiles) { + return new FileReference("non-existing-file"); + } else { + throw new IllegalArgumentException(e); + } + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + ); } @Override @@ -88,7 +102,7 @@ public class FileDBRegistry implements FileRegistry { @Override public FileReference addBlob(ByteBuffer blob) { String blobName = FileRegistry.blobName(blob); - String relativePath = blobToRelativeFile(blob, blobName); + String relativePath = blobToRelativeFile(blobName); synchronized (this) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName)); return cachedReference.orElseGet(() -> { @@ -108,6 +122,22 @@ public class FileDBRegistry implements FileRegistry { return entries; } + synchronized Map<String, FileReference> getMap() { + return ImmutableMap.copyOf(fileReferenceCache); + } + + public static String exportRegistry(FileRegistry registry) { + List<Entry> entries = registry.export(); + StringBuilder builder = new StringBuilder(); + + builder.append(HostName.getLocalhost()).append('\n'); + for (FileRegistry.Entry entry : entries) { + builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()).append('\n'); + } + + return builder.toString(); + } + private static String uriToRelativeFile(String uri) { String relative = "uri/" + XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0); if (uri.endsWith(".json")) { @@ -120,9 +150,8 @@ public class FileDBRegistry implements FileRegistry { return relative; } - private static String blobToRelativeFile(ByteBuffer blob, String blobName) { - String relative = "blob/" + blobName; - return relative; + private static String blobToRelativeFile(String blobName) { + return "blob/" + blobName; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index f7ba895666b..1fe8c741a17 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -97,15 +97,11 @@ public class FileDirectory { return hasher.hash(ByteBuffer.wrap(wholeFile), hasher.hash(ByteBuffer.wrap(Utf8.toBytes(file.getName())), 0)); } - public FileReference addFile(File source) { - try { - Long hash = computeHash(source); - verifyExistingFile(source, hash); - FileReference fileReference = fileReferenceFromHash(hash); - return addFile(source, fileReference); - } catch (IOException e) { - throw new IllegalArgumentException(e); - } + public FileReference addFile(File source) throws IOException { + Long hash = computeHash(source); + verifyExistingFile(source, hash); + FileReference fileReference = fileReferenceFromHash(hash); + return addFile(source, fileReference); } // If there exists a directory for a file reference, but content does not have correct hash or the file we want to add diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java new file mode 100644 index 00000000000..10ac54dc757 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author Tony Vaagenes + */ +public class FileDBRegistryTestCase { + + private static final String BLOB = "Some blob"; + private static final String APP = "src/test/apps/zkapp"; + private static final String FOO_FILE = "files/foo.json"; + private static final String NO_FOO_FILE = "files/no_foo.json"; + private static final String BLOB_NAME = "c5674b55c15c9c95"; + private static final FileReference BLOB_REF = new FileReference("b3424b78130fc005"); + private static final FileReference FOO_REF = new FileReference("b5ce94ca1feae86c"); + @Test + public void importAndExport() throws IOException { + TemporaryFolder tmpDir = new TemporaryFolder(); + tmpDir.create(); + AddFileInterface fileManager = new ApplicationFileManager(new File(APP), new FileDirectory(tmpDir.newFolder())); + FileRegistry fileRegistry = new FileDBRegistry(fileManager); + assertEquals(FOO_REF, fileRegistry.addFile(FOO_FILE)); + try { + fileRegistry.addFile(NO_FOO_FILE); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("src/test/apps/zkapp/files/no_foo.json (No such file or directory)", e.getCause().getMessage()); + } + assertEquals(BLOB_REF, fileRegistry.addBlob(ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8)))); + String serializedRegistry = FileDBRegistry.exportRegistry(fileRegistry); + + FileDBRegistry importedRegistry = FileDBRegistry.create(fileManager, new StringReader(serializedRegistry)); + + assertEquals(Set.of(BLOB_NAME, FOO_FILE), importedRegistry.getMap().keySet()); + assertEquals(BLOB_REF, importedRegistry.getMap().get(BLOB_NAME)); + assertEquals(FOO_REF, importedRegistry.getMap().get(FOO_FILE)); + + assertEquals(2, importedRegistry.export().size()); + + checkConsistentEntry(fileRegistry.export().get(0), importedRegistry); + checkConsistentEntry(fileRegistry.export().get(1), importedRegistry); + + assertEquals(new FileReference("non-existing-file"), importedRegistry.addFile(NO_FOO_FILE)); + assertEquals(2, importedRegistry.export().size()); + tmpDir.delete(); + } + + void checkConsistentEntry(FileRegistry.Entry entry, FileRegistry registry) { + assertEquals(entry.reference, registry.addFile(entry.relativePath)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java index b11308214a3..8b3227e3a69 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.net.HostName; import net.jpountz.xxhash.XXHashFactory; import java.io.File; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.ArrayList; @@ -30,11 +30,16 @@ public class MockFileRegistry implements FileRegistry { public FileReference addFile(String relativePath) { if (relativePath.isEmpty()) relativePath = "./"; - addFileInterface.addFile(relativePath); - FileReference fileReference = new FileReference(relativePath); - entries.add(new Entry(relativePath, fileReference)); - return fileReference; + try { + addFileInterface.addFile(relativePath); + + FileReference fileReference = new FileReference(relativePath); + entries.add(new Entry(relativePath, fileReference)); + return fileReference; + } catch (IOException e) { + throw new IllegalArgumentException(e); + } } public List<Entry> export() { return entries; } |