Merge pull request #30708 from vespa-engine/arnej/add-container-llama

add container-llama bundle

10 files changed, 164 insertions, 0 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f077dd87b9d..ab79c607cab 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -120,6 +120,7 @@ add_subdirectory(container-core)
 add_subdirectory(container-disc)
 add_subdirectory(container-messagebus)
 add_subdirectory(container-onnxruntime)
+add_subdirectory(container-llama)
 add_subdirectory(container-search)
 add_subdirectory(container-search-and-docproc)
 add_subdirectory(container-spifly)
diff --git a/container-llama/CMakeLists.txt b/container-llama/CMakeLists.txt
new file mode 100644
index 00000000000..ba41a4ef835
--- /dev/null
+++ b/container-llama/CMakeLists.txt
@@ -0,0 +1,2 @@
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+install_jar(container-llama.jar)
diff --git a/container-llama/README.md b/container-llama/README.md
new file mode 100644
index 00000000000..3f172726124
--- /dev/null
+++ b/container-llama/README.md
@@ -0,0 +1,4 @@
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+# container-llama
+
+llama repackaged as a bundle
diff --git a/container-llama/pom.xml b/container-llama/pom.xml
new file mode 100644
index 00000000000..1eb057abcaf
--- /dev/null
+++ b/container-llama/pom.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0"?>
+<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>container-llama</artifactId>
+  <packaging>container-plugin</packaging>
+  <version>8-SNAPSHOT</version>
+  <parent>
+    <groupId>com.yahoo.vespa</groupId>
+    <artifactId>parent</artifactId>
+    <version>8-SNAPSHOT</version>
+    <relativePath>../parent/pom.xml</relativePath>
+  </parent>
+  <properties>
+    <maven.javadoc.skip>true</maven.javadoc.skip> <!-- Javadoc plugin fails because of no source code in module -->
+  </properties>
+  <dependencies>
+    <!-- provided -->
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>annotations</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <!-- Not directly used in this module, but needed to get Import-Packages for JDK packages it exports. -->
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>jdisc_core</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <!-- compile -->
+    <dependency>
+      <groupId>de.kherud</groupId>
+      <artifactId>llama</artifactId>
+      <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.jetbrains</groupId>
+          <artifactId>annotations</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>bundle-plugin</artifactId>
+        <extensions>true</extensions>
+        <configuration>
+	 <bundleType>CORE</bundleType>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+	 <execution>
+	   <phase>package</phase>
+	   <goals>
+	     <goal>shade</goal>
+	   </goals>
+	   <configuration>
+	     <minimizeJar>false</minimizeJar>
+	     <createDependencyReducedPom>false</createDependencyReducedPom>
+	     <filters>
+	       <filter>
+		<artifact>de.kherud:*</artifact>
+		<excludes>
+		  <exclude>de/kherud/llama/Linux-Android//**</exclude>
+		  <exclude>de/kherud/llama/Linux/**</exclude>
+		  <exclude>de/kherud/llama/Mac/**</exclude>
+		  <exclude>de/kherud/llama/Windows/**</exclude>
+		</excludes>
+	       </filter>
+	     </filters>
+	     <transformers>
+	       <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+		<manifestEntries>
+		  <Bundle-Activator>ai.vespa.llama.LlamaBundleActivator</Bundle-Activator>
+		</manifestEntries>
+	       </transformer>
+	     </transformers>
+	   </configuration>
+	 </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/container-llama/src/main/java/ai/vespa/llama/LlamaBundleActivator.java b/container-llama/src/main/java/ai/vespa/llama/LlamaBundleActivator.java
new file mode 100644
index 00000000000..11ba05e363d
--- /dev/null
+++ b/container-llama/src/main/java/ai/vespa/llama/LlamaBundleActivator.java
@@ -0,0 +1,51 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package ai.vespa.llama;
+
+import org.osgi.framework.BundleActivator;
+import org.osgi.framework.BundleContext;
+
+import java.util.logging.Logger;
+
+/**
+ * @author arnej
+ * Finds native libraries when the bundle is activated.
+ **/
+public class LlamaBundleActivator implements BundleActivator {
+
+    private static final String PATH_PROPNAME = "de.kherud.llama.lib.path";
+    private static final Logger log = Logger.getLogger(LlamaBundleActivator.class.getName());
+
+    @Override
+    public void start(BundleContext ctx) {
+        log.fine("start bundle");
+        if (checkFilenames(
+                    "/dev/nvidia0",
+                    "/opt/vespa-deps/lib64/cuda/libllama.so",
+                    "/opt/vespa-deps/lib64/cuda/libjllama.so")) {
+            System.setProperty(PATH_PROPNAME, "/opt/vespa-deps/lib64/cuda");
+        } else if (checkFilenames(
+                    "/opt/vespa-deps/lib64/libllama.so",
+                    "/opt/vespa-deps/lib64/libjllama.so")) {
+            System.setProperty(PATH_PROPNAME, "/opt/vespa-deps/lib64");
+        } else {
+            throw new IllegalArgumentException("Cannot find shared libraries");
+        }
+    }
+
+    @Override
+    public void stop(BundleContext ctx) {
+        log.fine("stop bundle");
+    }
+
+    private boolean checkFilenames(String... filenames) {
+        for (String fn : filenames) {
+            var f = new java.io.File(fn);
+            if (! f.canRead()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+}
diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml
index ac8c9a9a793..5813d2655d7 100644
--- a/dependency-versions/pom.xml
+++ b/dependency-versions/pom.xml
@@ -115,6 +115,7 @@
         <junit.vespa.version>5.10.2</junit.vespa.version>
         <junit.platform.vespa.version>1.10.2</junit.platform.vespa.version>
         <junit4.vespa.version>4.13.2</junit4.vespa.version>
+        <kherud.llama.vespa.version>2.3.5</kherud.llama.vespa.version>
         <luben.zstd.vespa.version>1.5.6-1</luben.zstd.vespa.version>
         <lucene.vespa.version>9.10.0</lucene.vespa.version>
         <maven-archiver.vespa.version>3.6.2</maven-archiver.vespa.version>
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 53e82e372af..4886bd43db8 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -629,6 +629,7 @@ fi
 %{_prefix}/lib/jars/config-provisioning-jar-with-dependencies.jar
 %{_prefix}/lib/jars/container-apache-http-client-bundle-jar-with-dependencies.jar
 %{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar
+%{_prefix}/lib/jars/container-llama.jar
 %{_prefix}/lib/jars/container-onnxruntime.jar
 %{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar
 %{_prefix}/lib/jars/container-spifly.jar
diff --git a/parent/pom.xml b/parent/pom.xml
index 864bff7fe07..b21e9b844f5 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -594,6 +594,11 @@
                 <version>${onnxruntime.vespa.version}</version>
             </dependency>
             <dependency>
+                <groupId>de.kherud</groupId>
+                <artifactId>llama</artifactId>
+                <version>${kherud.llama.vespa.version}</version>
+            </dependency>
+            <dependency>
                 <groupId>com.yahoo.athenz</groupId>
                 <artifactId>athenz-cert-refresher</artifactId>
                 <version>${athenz.vespa.version}</version>
diff --git a/pom.xml b/pom.xml
index 52e0ad68215..2ed4e763b95 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,6 +56,7 @@
         <module>container-dev</module>
         <module>container-disc</module>
         <module>container-documentapi</module>
+        <module>container-llama</module>
         <module>container-messagebus</module>
         <module>container-onnxruntime</module>
         <module>container-search-and-docproc</module>
diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
index aaa97604add..fd7baf675b6 100644
--- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
+++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
@@ -30,6 +30,7 @@ com.google.jimfs:jimfs:${jimfs.vespa.version}
 com.google.protobuf:protobuf-java:${protobuf.vespa.version}
 com.ibm.icu:icu4j:${icu4j.vespa.version}
 com.microsoft.onnxruntime:onnxruntime:${onnxruntime.vespa.version}
+de.kherud:llama:${kherud.llama.vespa.version}
 com.sun.activation:javax.activation:1.2.0
 com.sun.istack:istack-commons-runtime:4.1.2
 com.sun.xml.bind:jaxb-core:${jaxb-core.vespa.version}