diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-12-15 14:22:21 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2018-01-25 13:33:55 +0000 |
commit | d2834dc7a53fe629425b53ad95f6d503ca901f31 (patch) | |
tree | 58ff101ceb86b87bc9e7dec68d7339d5a39facb2 | |
parent | f03d9cab494447f079a342124fbd0c0b3541f439 (diff) |
Add option to also send an uri.
10 files changed, 141 insertions, 0 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java index d635fe90ded..515477641a8 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -35,4 +35,9 @@ public class MockFileRegistry implements FileRegistry { return result; } + @Override + public FileReference addUri(String uri) { + throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName()); + } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java index 0b0b799f47f..2b77db119c8 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java @@ -74,6 +74,11 @@ public class PreGeneratedFileRegistry implements FileRegistry { } @Override + public FileReference addUri(String uri) { + throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName()); + } + + @Override public String fileSourceHost() { return fileSourceHost; } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index 887da2d51c8..cb4917fe919 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -12,6 +12,7 @@ import com.yahoo.config.FileReference; public interface FileRegistry { FileReference addFile(String relativePath); + FileReference addUri(String uri); /** * Returns the name of the host which is the source of the files diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 3b75be5167d..65f7bbedc68 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -508,6 +508,13 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon throw new RuntimeException("File does not exist: '" + relativePath + "'."); } } + public FileReference sendUri(String uri) { + try { + return getRoot().getFileDistributor().sendUriToHost(uri, getHost()); + } catch (PathDoesNotExistException e) { + throw new RuntimeException("Uri does not exist: '" + uri + "'."); + } + } /** * diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index 213451da55e..01944aee243 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -38,11 +38,29 @@ public class FileDistributor { return reference; } + /** + * Adds the given file to the associated application packages' registry of file and marks the file + * for distribution to the given hosts. + * <b>Note: This class receives ownership of the given collection.</b> + * + * @return the reference to the file, created by the application package + */ + public FileReference sendUriToHosts(String uri, Collection<Host> hosts) { + FileReference reference = fileRegistry.addUri(uri); + addToFilesToDistribute(reference, hosts); + + return reference; + } + /** Same as sendFileToHost(relativePath,Collections.singletonList(host) */ public FileReference sendFileToHost(String relativePath, Host host) { return sendFileToHosts(relativePath, Arrays.asList(host)); } + public FileReference sendUriToHost(String uri, Host host) { + return sendUriToHosts(uri, Arrays.asList(host)); + } + private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) { Set<Host> oldHosts = getHosts(reference); oldHosts.addAll(hosts); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 413363d7b0d..8995fcbca99 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -20,6 +20,8 @@ import java.util.*; */ public class FileSender implements Serializable { + public enum FileType {FILE, URI}; + /** * Send the given file to all given services. * @@ -34,6 +36,7 @@ public class FileSender implements Serializable { throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + "using 'count' instead of <node> tags."); } + FileReference fileref = null; for (AbstractService service : services) { // The same reference will be returned from each call. @@ -42,6 +45,20 @@ public class FileSender implements Serializable { return fileref; } + public static FileReference sendUriToServices(String uri, Collection<? extends AbstractService> services) { + if (services.isEmpty()) { + throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + + "using 'count' instead of <node> tags."); + } + + FileReference fileref = null; + for (AbstractService service : services) { + // The same reference will be returned from each call. + fileref = service.sendUri(uri); + } + return fileref; + } + /** * Sends all user configured files for a producer to all given services. */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java index 61c376a7256..39bf2ee02fd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; public interface AddFileInterface { + FileReference addUri(String uri, String relativePath); + FileReference addUri(String uri, String relativePath, FileReference reference); FileReference addFile(String relativePath); FileReference addFile(String relativePath, FileReference reference); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index 82535143c89..0e7aa4a4fd2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -3,6 +3,12 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; public class ApplicationFileManager implements AddFileInterface { @@ -24,4 +30,41 @@ public class ApplicationFileManager implements AddFileInterface { return master.addFile(new File(applicationDir, relativePath)); } + @Override + public FileReference addUri(String uri, String relativePath) { + download(uri, relativePath); + return addFile(relativePath); + } + + @Override + public FileReference addUri(String uri, String relativePath, FileReference reference) { + download(uri, relativePath); + return addFile(relativePath, reference); + } + + void download(String uri, String relativePath) { + File file = new File(applicationDir, relativePath); + FileOutputStream fos = null; + ReadableByteChannel rbc = null; + try { + Files.createDirectories(file.toPath().getParent()); + URL website = new URL(uri); + rbc = Channels.newChannel(website.openStream()); + fos = new FileOutputStream(file.getAbsolutePath()); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + } catch (IOException e) { + throw new IllegalArgumentException("Failed creating directory " + file.getParent(), e); + } finally { + try { + if (fos != null) { + fos.close(); + } + if (rbc != null) { + rbc.close(); + } + } catch (IOException e) { + throw new IllegalArgumentException("Failed closing down after downloading " + uri + " to " + file.getAbsolutePath()); + } + } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java index 8f2cb194bbd..7714b34e00e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java @@ -21,6 +21,12 @@ public class CombinedLegacyRegistry implements FileRegistry { } @Override + public FileReference addUri(String uri) { + FileReference reference = legacy.addUri(uri); + return future.addUri(uri, reference); + } + + @Override public String fileSourceHost() { return future.fileSourceHost(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index 1a76454fbed..653b2566096 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -4,7 +4,10 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.net.HostName; +import com.yahoo.text.Utf8; +import net.jpountz.xxhash.XXHashFactory; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -34,6 +37,17 @@ public class FileDBRegistry implements FileRegistry { }); } + public synchronized FileReference addUri(String uri, FileReference reference) { + String relativePath = uriToRelativeFile(uri); + Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri)); + return cachedReference.orElseGet(() -> { + FileReference newRef = manager.addUri(uri, relativePath, reference); + entries.add(new Entry(uri, newRef)); + fileReferenceCache.put(uri, newRef); + return newRef; + }); + } + @Override public synchronized FileReference addFile(String relativePath) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath)); @@ -46,6 +60,18 @@ public class FileDBRegistry implements FileRegistry { } @Override + public synchronized FileReference addUri(String uri) { + String relativePath = uriToRelativeFile(uri); + Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri)); + return cachedReference.orElseGet(() -> { + FileReference newRef = manager.addUri(uri, relativePath); + entries.add(new Entry(uri, newRef)); + fileReferenceCache.put(uri, newRef); + return newRef; + }); + } + + @Override public String fileSourceHost() { return HostName.getLocalhost(); } @@ -55,4 +81,15 @@ public class FileDBRegistry implements FileRegistry { return entries; } + private static String uriToRelativeFile(String uri) { + String relative = "uri/" + String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); + if (uri.endsWith(".json")) { + relative += ".json"; + } else if (uri.endsWith(".json.lz4")) { + relative += ".json.lz4"; + } else if (uri.endsWith(".lz4")) { + relative += ".lz4"; + } + return relative; + } } |