diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /filedistributionmanager |
Publish
Diffstat (limited to 'filedistributionmanager')
6 files changed, 194 insertions, 0 deletions
diff --git a/filedistributionmanager/.gitignore b/filedistributionmanager/.gitignore new file mode 100644 index 00000000000..39c0275a1b2 --- /dev/null +++ b/filedistributionmanager/.gitignore @@ -0,0 +1,2 @@ +target +/pom.xml.build diff --git a/filedistributionmanager/OWNERS b/filedistributionmanager/OWNERS new file mode 100644 index 00000000000..31af040f698 --- /dev/null +++ b/filedistributionmanager/OWNERS @@ -0,0 +1 @@ +bratseth diff --git a/filedistributionmanager/pom.xml b/filedistributionmanager/pom.xml new file mode 100644 index 00000000000..e58112568b0 --- /dev/null +++ b/filedistributionmanager/pom.xml @@ -0,0 +1,20 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>filedistributionmanager</artifactId> + <version>6-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> diff --git a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java new file mode 100644 index 00000000000..63e93f1a358 --- /dev/null +++ b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java @@ -0,0 +1,150 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.locks.Lock; + +/** + * @author tonytv + */ +public class FileDistributionManager { + + private final static boolean available; + + private long nativeFileDistributionManager; + + private final File applicationDirectory; + private final String appId; + private final Lock lock; + + + private native static void setup(); + + private native void init(byte[] fileDbDirectory, byte[] zkServers); + + private native String addFileImpl(byte[] absolutePath); + + private native void setDeployedFilesImpl(byte[] host, byte[] appId, byte[][] fileReferences); + + private native void limitSendingOfDeployedFilesToImpl(byte[][] hostNames, byte[] appId); + private native void limitFilesToImpl(byte[][] fileReferences); + private native void removeDeploymentsThatHaveDifferentApplicationIdImpl(byte[][] asByteArrays, byte[] bytes); + + private native byte[] getProgressImpl(byte[] fileReference, + byte[][] hostNames); + + private byte[][] getAsByteArrays(Collection<String> strings) { + byte[][] byteArrays = new byte[strings.size()][]; + int i = 0; + for (String string : strings) { + byteArrays[i++] = string.getBytes(); + } + return byteArrays; + } + + @Override + protected void finalize() throws Throwable { + shutdown(); + super.finalize(); + } + + static { + available = loadLibrary("filedistributionmanager"); + if (available) + setup(); + } + + private static boolean loadLibrary(String name) { + try { + System.loadLibrary(name); + return true; + } + catch (UnsatisfiedLinkError e) { + return false; + } + } + + /** Returns whether this functionality is available in this runtime */ + public static boolean isAvailable() { return available; } + + private byte[] absolutePath(File file) { + return file.getAbsolutePath().getBytes(); + } + + private void ensureDirExists(File dir) { + if (!dir.exists()) { + throw new PathDoesNotExistException(dir.getPath()); + } + } + + public FileDistributionManager(File fileDbDirectory, File applicationDirectory, String zkServers, String appId, Lock lock) { + ensureDirExists(applicationDirectory); + ensureDirExists(fileDbDirectory); + + this.applicationDirectory = applicationDirectory; + this.appId = appId; + this.lock = lock; + + init(fileDbDirectory.getPath().getBytes(), zkServers.getBytes()); + } + + public String addFile(String relativePath) { + File path = new File(applicationDirectory, relativePath); + if (!path.exists()) + throw new PathDoesNotExistException(path.getPath()); + + try (LockGuard guard = new LockGuard(lock)) { + return addFileImpl(absolutePath(path)); + } + } + + public void setDeployedFiles(String hostName, Collection<String> fileReferences) { + try (LockGuard guard = new LockGuard(lock)) { + setDeployedFilesImpl(hostName.getBytes(), appId.getBytes(), getAsByteArrays(fileReferences)); + } + } + + public void reloadDeployFileDistributor() { + try (LockGuard guard = new LockGuard(lock)) { + Runtime.getRuntime().exec("pkill -SIGUSR1 -x filedistributor"); + } catch (IOException e) { + throw new RuntimeException("Failed to reinitialize the filedistributor", e); + } + } + + public void limitSendingOfDeployedFilesTo(Collection<String> hostNames) { + try (LockGuard guard = new LockGuard(lock)) { + limitSendingOfDeployedFilesToImpl(getAsByteArrays(hostNames), appId.getBytes()); + } + } + + public byte[] getProgress(String fileReference, + List<String> hostNamesSortedAscending) { + return getProgressImpl(fileReference.getBytes(), getAsByteArrays(hostNamesSortedAscending)); + } + + + public native void shutdown(); + + public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { + try (LockGuard guard = new LockGuard(lock)) { + removeDeploymentsThatHaveDifferentApplicationIdImpl(getAsByteArrays(targetHostnames), appId.getBytes()); + } + } + + private static class LockGuard implements AutoCloseable { + private final Lock lock; + public LockGuard(Lock lock) { + this.lock = lock; + lock.lock(); + } + + @Override + public void close() { + lock.unlock(); + } + } +} diff --git a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/PathDoesNotExistException.java b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/PathDoesNotExistException.java new file mode 100755 index 00000000000..bfff2723013 --- /dev/null +++ b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/PathDoesNotExistException.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution; + +/** + * @author gjoranv + */ +public class PathDoesNotExistException extends RuntimeException { + public final String path; + + PathDoesNotExistException(String path) { + super("Path '" + path + "' does not exist."); + this.path = path; + } +} diff --git a/filedistributionmanager/src/main/resources/filedistributionmanager_java.mak b/filedistributionmanager/src/main/resources/filedistributionmanager_java.mak new file mode 100644 index 00000000000..dbb9d9b52cd --- /dev/null +++ b/filedistributionmanager/src/main/resources/filedistributionmanager_java.mak @@ -0,0 +1,7 @@ +ifndef MODULEDEF_FILEDISTRIBUTIONMANAGER_JAVA +MODULEDEF_FILEDISTRIBUTIONMANAGER_JAVA=FILEDISTRIBUTIONMANAGER_JAVA +MODULE_DEFINES_FILEDISTRIBUTIONMANAGER_JAVA= +MODULE_INCLUDES_FILEDISTRIBUTIONMANAGER_JAVA= +MODULE_MAKEMAKE_FILEDISTRIBUTIONMANAGER_JAVA= +MODULE_LIST += FILEDISTRIBUTIONMANAGER_JAVA +endif |