diff options
20 files changed, 33 insertions, 29 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bdb3a23730..33cf9914a07 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,7 +96,6 @@ vespa_install_data(tsan-suppressions.txt etc/vespa) # Include vespa config definitions in every target include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) -add_subdirectory(airlift-zstd) add_subdirectory(application-model) add_subdirectory(client) add_subdirectory(cloud-tenant-cd) diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 08630c1eca5..ce673aea54e 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -110,7 +110,6 @@ <include>com.yahoo.vespa:vespalog:*:provided</include> <!-- Vespa test dependencies --> - <include>com.yahoo.vespa:airlift-zstd:*:test</include> <include>com.yahoo.vespa:application:*:test</include> <include>com.yahoo.vespa:cloud-tenant-cd:*:test</include> <include>com.yahoo.vespa:config-application-package:*:test</include> @@ -143,6 +142,7 @@ <include>com.microsoft.onnxruntime:onnxruntime:jar:${onnxruntime.vespa.version}:test</include> <include>com.thaiopensource:jing:20091111:test</include> <include>commons-codec:commons-codec:${commons-codec.vespa.version}:test</include> + <include>io.airlift:aircompressor:${aircompressor.vespa.version}:test</include> <include>io.airlift:airline:${airline.vespa.version}:test</include> <include>io.prometheus:simpleclient:${prometheus.client.vespa.version}:test</include> <include>io.prometheus:simpleclient_common:${prometheus.client.vespa.version}:test</include> diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 3e12a4fccb5..db97d5d2e2c 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -138,7 +138,6 @@ <i>com.yahoo.vespa:fat-model-dependencies:*:*</i> <!-- Vespa artifacts embedded --> - <i>com.yahoo.vespa:airlift-zstd:*:*</i> <i>com.yahoo.vespa:annotations:*:*</i> <i>com.yahoo.vespa:component:*:*</i> <i>com.yahoo.vespa:config:*:*</i> @@ -196,6 +195,7 @@ <i>com.sun.xml.bind:jaxb-core:*:*</i> <i>com.sun.xml.bind:jaxb-impl:*:*</i> <i>com.thaiopensource:jing:*:*</i> + <i>io.airlift:aircompressor:*:*</i> <i>io.prometheus:simpleclient:*:*</i> <i>io.prometheus:simpleclient_common:*:*</i> <i>io.prometheus:simpleclient_tracer_common:*:*</i> diff --git a/configserver/pom.xml b/configserver/pom.xml index 7cf1c64326a..83cd725f142 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -34,9 +34,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>airlift-zstd</artifactId> - <version>${project.version}</version> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -216,7 +215,7 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <allowEmbeddedArtifacts>com.yahoo.vespa:airlift-zstd</allowEmbeddedArtifacts> + <allowEmbeddedArtifacts>io.airlift:aircompressor</allowEmbeddedArtifacts> </configuration> </plugin> <plugin> diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 740332e766c..a56cd148b66 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -129,7 +129,6 @@ <include>com.yahoo.vespa:vespalog:*:provided</include> <!-- Vespa test dependencies --> - <include>com.yahoo.vespa:airlift-zstd:*:test</include> <include>com.yahoo.vespa:application:*:test</include> <include>com.yahoo.vespa:config-application-package:*:test</include> <include>com.yahoo.vespa:config-model-api:*:test</include> @@ -156,6 +155,7 @@ <include>com.microsoft.onnxruntime:onnxruntime:${onnxruntime.vespa.version}:test</include> <include>com.thaiopensource:jing:20091111:test</include> <include>commons-codec:commons-codec:${commons-codec.vespa.version}:test</include> + <include>io.airlift:aircompressor:${aircompressor.vespa.version}:test</include> <include>io.airlift:airline:${airline.vespa.version}:test</include> <include>io.prometheus:simpleclient:${prometheus.client.vespa.version}:test</include> <include>io.prometheus:simpleclient_common:${prometheus.client.vespa.version}:test</include> diff --git a/container-test/pom.xml b/container-test/pom.xml index baff67df3c0..3676b7a3280 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -49,9 +49,8 @@ for user projects must be added in compile scope here. These dependencies are explicitly excluded (or set to non-compile scope) in the container and/or container-dev modules. --> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>airlift-zstd</artifactId> - <version>${project.version}</version> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> </dependency> <dependency> <groupId>com.ibm.icu</groupId> diff --git a/container/pom.xml b/container/pom.xml index 284745c1cd1..140a204bc83 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -35,8 +35,8 @@ <artifactId>asm</artifactId> </exclusion> <exclusion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>airlift-zstd</artifactId> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> </exclusion> <exclusion> <groupId>net.java.dev.jna</groupId> diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index acf39dafe66..2719ac304ab 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -53,6 +53,7 @@ <!-- Dependencies used internally in Vespa, not visible for users, or only visible in test classpath --> + <aircompressor.vespa.version>0.25</aircompressor.vespa.version> <airline.vespa.version>0.9</airline.vespa.version> <antlr.vespa.version>3.5.3</antlr.vespa.version> <antlr4.vespa.version>4.13.1</antlr4.vespa.version> diff --git a/dist/vespa.spec b/dist/vespa.spec index cd1381fd48d..7cef6b4d045 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -652,7 +652,6 @@ fi %dir %{_prefix} %dir %{_prefix}/lib %dir %{_prefix}/lib/jars -%{_prefix}/lib/jars/airlift-zstd.jar %{_prefix}/lib/jars/application-model-jar-with-dependencies.jar %{_prefix}/lib/jars/bc*-jdk18on-*.jar %{_prefix}/lib/jars/config-bundle-jar-with-dependencies.jar diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index 7104567cb05..bcfeddacff3 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; -import ai.vespa.airlift.zstd.ZstdInputStream; +import io.airlift.compress.zstd.ZstdInputStream; import com.yahoo.compress.ZstdOutputStream; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java index 6eed21c8865..eeec7e1b59c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java @@ -14,7 +14,7 @@ import java.io.InputStream; public class ZstdCompressingInputStream extends InputStream { public static final int DEFAULT_INPUT_BUFFER_SIZE = 8 * 1024; - static final ZstdCompressor compressor = new ZstdCompressor(); + private final ZstdCompressor compressor = new ZstdCompressor(); private final InputStream is; private final byte[] inputBuffer; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java index 68a71a65a27..616100363e9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.sync; +import com.yahoo.compress.ZstdCompressor; import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; @@ -9,7 +10,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Random; -import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream.compressor; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,6 +35,7 @@ public class ZstdCompressingInputStreamTest { private static void assertCompression(byte[] data, int bufferSize) { byte[] compressedData = compress(data, bufferSize); byte[] decompressedData = new byte[data.length]; + var compressor = new ZstdCompressor(); compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); assertArrayEquals(data, decompressedData); diff --git a/parent/pom.xml b/parent/pom.xml index 4772650e48e..db0b215d51a 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -675,6 +675,11 @@ </dependency> <dependency> <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> + <version>${aircompressor.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.airlift</groupId> <artifactId>airline</artifactId> <version>${airline.vespa.version}</version> </dependency> @@ -24,7 +24,6 @@ </developers> <modules> - <module>airlift-zstd</module> <module>application</module> <module>application-model</module> <module>bundle-plugin-test</module> diff --git a/screwdriver.yaml b/screwdriver.yaml index b6e0fa5f35b..3733fa58476 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -547,8 +547,6 @@ jobs: while read f; do (echo -e "---\nrender_with_liquid: false\n---\n"; cat ${f})>${f}.new; mv ${f}.new ${f}; done - build-site: | bundle exec jekyll build - - ignore-files: | - rm -rf _site/airlift-zstd - check-links: | bundle exec htmlproofer \ --assume-extension --check-html --check-external-hash --no-enforce-http \ diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 1fc7fabc161..844c0adabc0 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -51,6 +51,7 @@ commons-collections:commons-collections:${commons-collections.vespa.version} commons-fileupload:commons-fileupload:1.5 commons-io:commons-io:${commons-io.vespa.version} commons-logging:commons-logging:${commons-logging.vespa.version} +io.airlift:aircompressor:${aircompressor.vespa.version} io.airlift:airline:${airline.vespa.version} io.dropwizard.metrics:metrics-core:${dropwizard.metrics.vespa.version} io.jsonwebtoken:jjwt-api:${java-jjwt.vespa.version} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/CipherUtils.java b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/CipherUtils.java index 36900f65e72..f2debef61e3 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/CipherUtils.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/CipherUtils.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.security.tool.crypto; -import ai.vespa.airlift.zstd.ZstdInputStream; +import io.airlift.compress.zstd.ZstdInputStream; import com.yahoo.compress.ZstdOutputStream; import com.yahoo.security.AeadCipher; diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 99924fe36c3..16ae251aa8b 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -32,9 +32,9 @@ <artifactId>jna</artifactId> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>airlift-zstd</artifactId> - <version>${project.version}</version> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> + <version>${aircompressor.vespa.version}</version> </dependency> <dependency> <groupId>net.openhft</groupId> diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index 1a9078e640e..1aca29cd21b 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -19,7 +19,6 @@ import java.util.Random; */ public class Compressor { - private final ZstdCompressor zstdCompressor = new ZstdCompressor(); private final CompressionType type; private final int level; private final double compressionThresholdFactor; @@ -101,6 +100,7 @@ public class Compressor { if (len < compressMinSizeBytes) { return compact(CompressionType.INCOMPRESSIBLE, data, offset, len); } + ZstdCompressor zstdCompressor = new ZstdCompressor(); byte[] compressed = zstdCompressor.compress(data, offset, len); return new Compression(CompressionType.ZSTD, len, compressed); default: @@ -151,6 +151,7 @@ public class Compressor { return uncompressedLZ4Data; case ZSTD: int compressedLength = expectedCompressedSize.orElseThrow(() -> new IllegalArgumentException("Zstd decompressor requires input size")); + ZstdCompressor zstdCompressor = new ZstdCompressor(); byte[] decompressedData = zstdCompressor.decompress(compressedData, compressedDataOffset, compressedLength); expectedCompressedSize.ifPresent(expectedSize -> { if (compressedData.length != expectedSize) { diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java index 5bec9aa4e8a..245a672de20 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java @@ -11,8 +11,8 @@ import java.util.Arrays; */ public class ZstdCompressor { - private static final ai.vespa.airlift.zstd.ZstdCompressor compressor = new ai.vespa.airlift.zstd.ZstdCompressor(); - private static final ai.vespa.airlift.zstd.ZstdDecompressor decompressor = new ai.vespa.airlift.zstd.ZstdDecompressor(); + private io.airlift.compress.zstd.ZstdCompressor compressor = new io.airlift.compress.zstd.ZstdCompressor(); + private io.airlift.compress.zstd.ZstdDecompressor decompressor = new io.airlift.compress.zstd.ZstdDecompressor(); public byte[] compress(byte[] input, int inputOffset, int inputLength) { int maxCompressedLength = getMaxCompressedLength(inputLength); @@ -41,12 +41,14 @@ public class ZstdCompressor { return decompressor.decompress(input, inputOffset, inputLength, output, outputOffset, maxOutputLength); } + private static final io.airlift.compress.Compressor threadUnsafe = new io.airlift.compress.zstd.ZstdCompressor(); + public static int getMaxCompressedLength(int uncompressedLength) { - return compressor.maxCompressedLength(uncompressedLength); + return threadUnsafe.maxCompressedLength(uncompressedLength); } public static int getDecompressedLength(byte[] input, int inputOffset, int inputLength) { - return (int) ai.vespa.airlift.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength); + return (int) io.airlift.compress.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength); } } |