diff options
-rwxr-xr-x | bootstrap.sh | 3 | ||||
-rw-r--r-- | config-model-fat/pom.xml | 5 | ||||
-rw-r--r-- | config-model/pom.xml | 5 | ||||
-rw-r--r-- | configserver/pom.xml | 5 | ||||
-rw-r--r-- | filedistribution_test/.gitignore | 8 | ||||
-rw-r--r-- | filedistribution_test/OWNERS | 1 | ||||
-rw-r--r-- | filedistribution_test/README | 1 | ||||
-rw-r--r-- | filedistribution_test/pom.xml | 43 | ||||
-rw-r--r-- | filedistribution_test/src/test/java/FileDistributionManagerTestCase.java | 120 | ||||
-rw-r--r-- | filedistributionmanager/.gitignore | 2 | ||||
-rw-r--r-- | filedistributionmanager/OWNERS | 1 | ||||
-rw-r--r-- | filedistributionmanager/README | 1 | ||||
-rw-r--r-- | filedistributionmanager/pom.xml | 20 | ||||
-rw-r--r-- | filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java | 139 | ||||
-rwxr-xr-x | filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/PathDoesNotExistException.java | 14 | ||||
-rw-r--r-- | filedistributionmanager/src/main/resources/filedistributionmanager_java.mak | 8 | ||||
-rw-r--r-- | pom.xml | 2 |
17 files changed, 1 insertions, 377 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index 29e18226ca7..35c54f1166f 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -67,10 +67,9 @@ case "$MODE" in ;; full) echo "Building full set of dependencies." - mvn_install -am -pl filedistributionmanager,jrt,linguistics,messagebus + mvn_install -am -pl jrt,linguistics,messagebus ;; default) echo "Building default set of dependencies." - mvn_install -am -pl filedistributionmanager ;; esac diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 8688f12c199..6700e7c44d0 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -250,11 +250,6 @@ <artifactId>indexinglanguage</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>filedistributionmanager</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>searchsummary</artifactId> diff --git a/config-model/pom.xml b/config-model/pom.xml index c159d00eb24..53ec473745b 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -272,11 +272,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>filedistributionmanager</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>searchsummary</artifactId> <version>${project.version}</version> </dependency> diff --git a/configserver/pom.xml b/configserver/pom.xml index ccb523f36b3..97149d00746 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -132,11 +132,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>filedistributionmanager</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>statistics</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/filedistribution_test/.gitignore b/filedistribution_test/.gitignore deleted file mode 100644 index 9f355e75159..00000000000 --- a/filedistribution_test/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -/boost -/filedistribution -/libtorrent -/zookeeper_c_client_dev -pom.xml.build -/target -/appdir -/dbdir diff --git a/filedistribution_test/OWNERS b/filedistribution_test/OWNERS deleted file mode 100644 index 31af040f698..00000000000 --- a/filedistribution_test/OWNERS +++ /dev/null @@ -1 +0,0 @@ -bratseth diff --git a/filedistribution_test/README b/filedistribution_test/README deleted file mode 100644 index 1b39ca981ca..00000000000 --- a/filedistribution_test/README +++ /dev/null @@ -1 +0,0 @@ -Tests of filedistribution.
\ No newline at end of file diff --git a/filedistribution_test/pom.xml b/filedistribution_test/pom.xml deleted file mode 100644 index 5cc113c6453..00000000000 --- a/filedistribution_test/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>filedistribution_test</artifactId> - <version>6-SNAPSHOT</version> - <dependencies> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>filedistributionmanager</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <argLine>-Djava.library.path=${libraryPath}</argLine> - <environmentVariables> - <LD_LIBRARY_PATH>${libraryPath}</LD_LIBRARY_PATH> - </environmentVariables> - <skipTests>false</skipTests> - </configuration> - </plugin> - </plugins> - </build> - <properties> - <libraryPath>boost/lib64:libtorrent/lib:zookeeper_c_client_dev/lib64:filedistribution/lib</libraryPath> - </properties> -</project> diff --git a/filedistribution_test/src/test/java/FileDistributionManagerTestCase.java b/filedistribution_test/src/test/java/FileDistributionManagerTestCase.java deleted file mode 100644 index 19b0b0380b7..00000000000 --- a/filedistribution_test/src/test/java/FileDistributionManagerTestCase.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.yahoo.vespa.filedistribution.FileDistributionManager; - -import static org.junit.Assume.assumeTrue; -import org.junit.*; -import org.junit.rules.TemporaryFolder; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -import java.io.File; -import java.io.IOException; -import java.io.FileWriter; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; - - -/** - * @author tonytv - */ -public class FileDistributionManagerTestCase { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - File testDir; - File dbDir = new File(testDir, "dbdir"); - File appDir = new File(testDir, "appdir"); - File components = new File(appDir, "components"); - FileDistributionManager manager; - MockLock lock; - - private String addFile(String name, String contents) throws IOException { - File file = new File(components, name); - FileWriter writer = new FileWriter(file, false); - writer.write(contents); - writer.close(); - - return manager.addFile("components/" + name); - } - - @Before - public void before() throws IOException { - assumeTrue(FileDistributionManager.isAvailable()); - - System.out.println(System.getProperty("java.library.path")); - testDir = folder.newFolder("filedistributionmanagertest" + System.currentTimeMillis()); - dbDir.mkdir(); - appDir.mkdir(); - components.mkdir(); - lock = new MockLock(); - - manager = new FileDistributionManager(dbDir, appDir, "mockfiledistributionmodel.testing", "foo", lock); - } - - @After - public void after() { - if (manager != null) - manager.shutdown(); - } - - @Test - public void addFiles() throws IOException { - final String commonContent = "content"; - final String name1 = "searcher1", name2 = "searcher2", name3 = "searcher3"; - - String hash1 = addFile(name1, commonContent); - String hash2 = addFile(name2, commonContent); - String hash3 = addFile(name3, "different content"); - assertThat(lock.numAcquire, is(3)); - assertThat(lock.numRelease, is(3)); - - assertNotSame(hash1, hash2); - assertNotSame(hash1, hash3); - assertNotSame(hash2, hash3); - - assertTrue(hash1.length() == 40); - - assertFileExists(name1, hash1); - assertFileExists(name2, hash2); - assertFileExists(name3, hash3); - } - - private void assertFileExists(String name, String hash) { - File destinationDir = new File(dbDir, hash + ".new"); - assertTrue(destinationDir.exists()); - - File file = new File(destinationDir, name); - assertTrue(file.exists()); - } - - private class MockLock implements Lock { - int numAcquire = 0; - int numRelease = 0; - - public void lock() { - numAcquire++; - - } - - public void lockInterruptibly() throws InterruptedException { - } - - public boolean tryLock() { - return false; - } - - public boolean tryLock(long time, TimeUnit unit) throws InterruptedException { - return false; - } - - public void unlock() { - numRelease++; - } - - public Condition newCondition() { - return null; - } - } -} diff --git a/filedistributionmanager/.gitignore b/filedistributionmanager/.gitignore deleted file mode 100644 index 39c0275a1b2..00000000000 --- a/filedistributionmanager/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -target -/pom.xml.build diff --git a/filedistributionmanager/OWNERS b/filedistributionmanager/OWNERS deleted file mode 100644 index 31af040f698..00000000000 --- a/filedistributionmanager/OWNERS +++ /dev/null @@ -1 +0,0 @@ -bratseth diff --git a/filedistributionmanager/README b/filedistributionmanager/README deleted file mode 100644 index 935a0160136..00000000000 --- a/filedistributionmanager/README +++ /dev/null @@ -1 +0,0 @@ -Java wrapper of the file distribution component.
\ No newline at end of file diff --git a/filedistributionmanager/pom.xml b/filedistributionmanager/pom.xml deleted file mode 100644 index 9f037814a12..00000000000 --- a/filedistributionmanager/pom.xml +++ /dev/null @@ -1,20 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>filedistributionmanager</artifactId> - <version>6-SNAPSHOT</version> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> -</project> diff --git a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java deleted file mode 100644 index f0c463abe70..00000000000 --- a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.filedistribution; - -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.concurrent.locks.Lock; - -/** - * @author tonytv - */ -public class FileDistributionManager { - - private final static boolean available; - - @SuppressWarnings({"UnusedDeclaration"}) // Needs to be here due to JNI - private long nativeFileDistributionManager; - - private final File applicationDirectory; - private final String appId; - private final Lock lock; - - - private native static void setup(); - - private native void init(byte[] fileDbDirectory, byte[] zkServers); - - private native String addFileImpl(byte[] absolutePath); - - private native void setDeployedFilesImpl(byte[] host, byte[] appId, byte[][] fileReferences); - - private native void removeDeploymentsThatHaveDifferentApplicationIdImpl(byte[][] asByteArrays, byte[] bytes); - - private byte[][] getAsByteArrays(Collection<String> strings) { - byte[][] byteArrays = new byte[strings.size()][]; - int i = 0; - for (String string : strings) { - byteArrays[i++] = string.getBytes(); - } - return byteArrays; - } - - @Override - protected void finalize() throws Throwable { - shutdown(); - super.finalize(); - } - - static { - available = loadLibrary("filedistributionmanager"); - if (available) - setup(); - } - - private static boolean loadLibrary(String name) { - try { - System.loadLibrary(name); - return true; - } - catch (UnsatisfiedLinkError e) { - return false; - } - } - - /** Returns whether this functionality is available in this runtime */ - public static boolean isAvailable() { return available; } - - private byte[] absolutePath(File file) { - return file.getAbsolutePath().getBytes(); - } - - private void ensureDirExists(File dir) { - if (!dir.exists()) { - throw new PathDoesNotExistException(dir.getPath()); - } - } - - public FileDistributionManager(File fileDbDirectory, File applicationDirectory, String zkServers, String appId, Lock lock) { - ensureDirExists(applicationDirectory); - ensureDirExists(fileDbDirectory); - - this.applicationDirectory = applicationDirectory; - this.appId = appId; - this.lock = lock; - - init(fileDbDirectory.getPath().getBytes(), zkServers.getBytes()); - } - - public String addFile(String relativePath) { - File path = new File(applicationDirectory, relativePath); - if (!path.exists()) - throw new PathDoesNotExistException(path.getPath()); - - try (LockGuard guard = new LockGuard(lock)) { - return addFileImpl(absolutePath(path)); - } - } - - public void setDeployedFiles(String hostName, Collection<String> fileReferences) { - try (LockGuard guard = new LockGuard(lock)) { - setDeployedFilesImpl(hostName.getBytes(), appId.getBytes(), getAsByteArrays(fileReferences)); - } - } - - public void reloadDeployFileDistributor() { - try (LockGuard guard = new LockGuard(lock)) { - /* - * Try sending signal to vespa-filedistributor and - * filedistributor processes. Note 15 char limit for process - * name on Linux, see pkill manual page for details. - */ - Runtime.getRuntime().exec("pkill -SIGUSR1 ^(vespa-filedistr|filedistributor)$"); - } catch (IOException e) { - throw new RuntimeException("Failed to reinitialize the filedistributor", e); - } - } - - - public native void shutdown(); - - public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { - try (LockGuard guard = new LockGuard(lock)) { - removeDeploymentsThatHaveDifferentApplicationIdImpl(getAsByteArrays(targetHostnames), appId.getBytes()); - } - } - - private static class LockGuard implements AutoCloseable { - private final Lock lock; - public LockGuard(Lock lock) { - this.lock = lock; - lock.lock(); - } - - @Override - public void close() { - lock.unlock(); - } - } -} diff --git a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/PathDoesNotExistException.java b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/PathDoesNotExistException.java deleted file mode 100755 index 5dfdc0f9d7d..00000000000 --- a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/PathDoesNotExistException.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.filedistribution; - -/** - * @author gjoranv - */ -public class PathDoesNotExistException extends RuntimeException { - public final String path; - - PathDoesNotExistException(String path) { - super("Path '" + path + "' does not exist."); - this.path = path; - } -} diff --git a/filedistributionmanager/src/main/resources/filedistributionmanager_java.mak b/filedistributionmanager/src/main/resources/filedistributionmanager_java.mak deleted file mode 100644 index ea4da1df3d0..00000000000 --- a/filedistributionmanager/src/main/resources/filedistributionmanager_java.mak +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -ifndef MODULEDEF_FILEDISTRIBUTIONMANAGER_JAVA -MODULEDEF_FILEDISTRIBUTIONMANAGER_JAVA=FILEDISTRIBUTIONMANAGER_JAVA -MODULE_DEFINES_FILEDISTRIBUTIONMANAGER_JAVA= -MODULE_INCLUDES_FILEDISTRIBUTIONMANAGER_JAVA= -MODULE_MAKEMAKE_FILEDISTRIBUTIONMANAGER_JAVA= -MODULE_LIST += FILEDISTRIBUTIONMANAGER_JAVA -endif @@ -73,9 +73,7 @@ <module>document</module> <module>documentgen-test</module> <module>fileacquirer</module> - <module>filedistributionmanager</module> <module>filedistribution</module> - <module>filedistribution_test</module> <module>fsa</module> <module>indexinglanguage</module> <module>jaxrs_client_utils</module> |