diff options
author | Arne Juul <arnej@vespa.ai> | 2023-10-26 13:52:15 +0000 |
---|---|---|
committer | Arne Juul <arnej@vespa.ai> | 2023-10-26 13:52:15 +0000 |
commit | ec1c22c585fc18167e08b4f095431c98def5229b (patch) | |
tree | f81754ccb0b48852e76878d914d3a07ce99b1c7f | |
parent | bce3b8e926bf9da880172acbe1ba4b12d5e026d6 (diff) |
switch to io.airlift:aircompressor
19 files changed, 29 insertions, 30 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bdb3a23730..33cf9914a07 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,7 +96,6 @@ vespa_install_data(tsan-suppressions.txt etc/vespa) # Include vespa config definitions in every target include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) -add_subdirectory(airlift-zstd) add_subdirectory(application-model) add_subdirectory(client) add_subdirectory(cloud-tenant-cd) diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 08630c1eca5..82c811baa98 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -47,6 +47,7 @@ <include>com.fasterxml.jackson.core:jackson-databind:${jackson-databind.vespa.version}:provided</include> <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.vespa.version}:provided</include> <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.vespa.version}:provided</include> + <include>io.airlift:aircompressor:${aircompress.vespa.version}:provided</include> <!-- Guava with its internal dependencies --> <include>com.google.guava:guava:${guava.vespa.version}:provided</include> @@ -110,7 +111,6 @@ <include>com.yahoo.vespa:vespalog:*:provided</include> <!-- Vespa test dependencies --> - <include>com.yahoo.vespa:airlift-zstd:*:test</include> <include>com.yahoo.vespa:application:*:test</include> <include>com.yahoo.vespa:cloud-tenant-cd:*:test</include> <include>com.yahoo.vespa:config-application-package:*:test</include> diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 3e12a4fccb5..db97d5d2e2c 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -138,7 +138,6 @@ <i>com.yahoo.vespa:fat-model-dependencies:*:*</i> <!-- Vespa artifacts embedded --> - <i>com.yahoo.vespa:airlift-zstd:*:*</i> <i>com.yahoo.vespa:annotations:*:*</i> <i>com.yahoo.vespa:component:*:*</i> <i>com.yahoo.vespa:config:*:*</i> @@ -196,6 +195,7 @@ <i>com.sun.xml.bind:jaxb-core:*:*</i> <i>com.sun.xml.bind:jaxb-impl:*:*</i> <i>com.thaiopensource:jing:*:*</i> + <i>io.airlift:aircompressor:*:*</i> <i>io.prometheus:simpleclient:*:*</i> <i>io.prometheus:simpleclient_common:*:*</i> <i>io.prometheus:simpleclient_tracer_common:*:*</i> diff --git a/configserver/pom.xml b/configserver/pom.xml index 7cf1c64326a..83cd725f142 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -34,9 +34,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>airlift-zstd</artifactId> - <version>${project.version}</version> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -216,7 +215,7 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <allowEmbeddedArtifacts>com.yahoo.vespa:airlift-zstd</allowEmbeddedArtifacts> + <allowEmbeddedArtifacts>io.airlift:aircompressor</allowEmbeddedArtifacts> </configuration> </plugin> <plugin> diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 740332e766c..74805433669 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -66,6 +66,7 @@ <include>com.fasterxml.jackson.core:jackson-databind:${jackson-databind.vespa.version}:provided</include> <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.vespa.version}:provided</include> <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.vespa.version}:provided</include> + <include>io.airlift:aircompressor:${aircompress.vespa.version}:provided</include> <!-- Guava with its internal dependencies --> <include>com.google.guava:guava:${guava.vespa.version}:provided</include> @@ -129,7 +130,6 @@ <include>com.yahoo.vespa:vespalog:*:provided</include> <!-- Vespa test dependencies --> - <include>com.yahoo.vespa:airlift-zstd:*:test</include> <include>com.yahoo.vespa:application:*:test</include> <include>com.yahoo.vespa:config-application-package:*:test</include> <include>com.yahoo.vespa:config-model-api:*:test</include> diff --git a/container-test/pom.xml b/container-test/pom.xml index baff67df3c0..0936f0803c1 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -49,11 +49,6 @@ for user projects must be added in compile scope here. These dependencies are explicitly excluded (or set to non-compile scope) in the container and/or container-dev modules. --> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>airlift-zstd</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>com.ibm.icu</groupId> <artifactId>icu4j</artifactId> </dependency> diff --git a/container/pom.xml b/container/pom.xml index 284745c1cd1..7dde2c25d4b 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -35,8 +35,8 @@ <artifactId>asm</artifactId> </exclusion> <exclusion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>airlift-zstd</artifactId> + <groupId>io.airlift</groupId> + <artifactId>aircompress</artifactId> </exclusion> <exclusion> <groupId>net.java.dev.jna</groupId> diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index f404cc529de..3c53ddc5850 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -53,6 +53,7 @@ <!-- Dependencies used internally in Vespa, not visible for users, or only visible in test classpath --> + <aircompress.vespa.version>0.25</aircompress.vespa.version> <airline.vespa.version>0.9</airline.vespa.version> <antlr.vespa.version>3.5.3</antlr.vespa.version> <antlr4.vespa.version>4.13.1</antlr4.vespa.version> diff --git a/dist/vespa.spec b/dist/vespa.spec index cd1381fd48d..7cef6b4d045 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -652,7 +652,6 @@ fi %dir %{_prefix} %dir %{_prefix}/lib %dir %{_prefix}/lib/jars -%{_prefix}/lib/jars/airlift-zstd.jar %{_prefix}/lib/jars/application-model-jar-with-dependencies.jar %{_prefix}/lib/jars/bc*-jdk18on-*.jar %{_prefix}/lib/jars/config-bundle-jar-with-dependencies.jar diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index 7104567cb05..bcfeddacff3 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; -import ai.vespa.airlift.zstd.ZstdInputStream; +import io.airlift.compress.zstd.ZstdInputStream; import com.yahoo.compress.ZstdOutputStream; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java index 6eed21c8865..eeec7e1b59c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java @@ -14,7 +14,7 @@ import java.io.InputStream; public class ZstdCompressingInputStream extends InputStream { public static final int DEFAULT_INPUT_BUFFER_SIZE = 8 * 1024; - static final ZstdCompressor compressor = new ZstdCompressor(); + private final ZstdCompressor compressor = new ZstdCompressor(); private final InputStream is; private final byte[] inputBuffer; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java index 68a71a65a27..616100363e9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.sync; +import com.yahoo.compress.ZstdCompressor; import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; @@ -9,7 +10,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Random; -import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream.compressor; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,6 +35,7 @@ public class ZstdCompressingInputStreamTest { private static void assertCompression(byte[] data, int bufferSize) { byte[] compressedData = compress(data, bufferSize); byte[] decompressedData = new byte[data.length]; + var compressor = new ZstdCompressor(); compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); assertArrayEquals(data, decompressedData); diff --git a/parent/pom.xml b/parent/pom.xml index 3718f41dbed..8873949cfd0 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -675,6 +675,11 @@ </dependency> <dependency> <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> + <version>${aircompress.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.airlift</groupId> <artifactId>airline</artifactId> <version>${airline.vespa.version}</version> </dependency> @@ -24,7 +24,6 @@ </developers> <modules> - <module>airlift-zstd</module> <module>application</module> <module>application-model</module> <module>bundle-plugin-test</module> diff --git a/screwdriver.yaml b/screwdriver.yaml index ccab9bc4bd0..beed0f91e27 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -549,8 +549,6 @@ jobs: while read f; do (echo -e "---\nrender_with_liquid: false\n---\n"; cat ${f})>${f}.new; mv ${f}.new ${f}; done - build-site: | bundle exec jekyll build - - ignore-files: | - rm -rf _site/airlift-zstd - check-links: | bundle exec htmlproofer \ --assume-extension --check-html --check-external-hash --no-enforce-http \ diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 941f7d78b99..6e031c70c32 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -51,6 +51,7 @@ commons-collections:commons-collections:${commons-collections.vespa.version} commons-fileupload:commons-fileupload:1.5 commons-io:commons-io:${commons-io.vespa.version} commons-logging:commons-logging:${commons-logging.vespa.version} +io.airlift:aircompressor:${aircompress.vespa.version} io.airlift:airline:${airline.vespa.version} io.dropwizard.metrics:metrics-core:${dropwizard.metrics.vespa.version} io.jsonwebtoken:jjwt-api:${java-jjwt.vespa.version} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/CipherUtils.java b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/CipherUtils.java index 36900f65e72..f2debef61e3 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/CipherUtils.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/crypto/CipherUtils.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.security.tool.crypto; -import ai.vespa.airlift.zstd.ZstdInputStream; +import io.airlift.compress.zstd.ZstdInputStream; import com.yahoo.compress.ZstdOutputStream; import com.yahoo.security.AeadCipher; diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 99924fe36c3..a0dea8b29fc 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -32,9 +32,9 @@ <artifactId>jna</artifactId> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>airlift-zstd</artifactId> - <version>${project.version}</version> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> + <version>${aircompress.vespa.version}</version> </dependency> <dependency> <groupId>net.openhft</groupId> diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java index 5bec9aa4e8a..245a672de20 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java @@ -11,8 +11,8 @@ import java.util.Arrays; */ public class ZstdCompressor { - private static final ai.vespa.airlift.zstd.ZstdCompressor compressor = new ai.vespa.airlift.zstd.ZstdCompressor(); - private static final ai.vespa.airlift.zstd.ZstdDecompressor decompressor = new ai.vespa.airlift.zstd.ZstdDecompressor(); + private io.airlift.compress.zstd.ZstdCompressor compressor = new io.airlift.compress.zstd.ZstdCompressor(); + private io.airlift.compress.zstd.ZstdDecompressor decompressor = new io.airlift.compress.zstd.ZstdDecompressor(); public byte[] compress(byte[] input, int inputOffset, int inputLength) { int maxCompressedLength = getMaxCompressedLength(inputLength); @@ -41,12 +41,14 @@ public class ZstdCompressor { return decompressor.decompress(input, inputOffset, inputLength, output, outputOffset, maxOutputLength); } + private static final io.airlift.compress.Compressor threadUnsafe = new io.airlift.compress.zstd.ZstdCompressor(); + public static int getMaxCompressedLength(int uncompressedLength) { - return compressor.maxCompressedLength(uncompressedLength); + return threadUnsafe.maxCompressedLength(uncompressedLength); } public static int getDecompressedLength(byte[] input, int inputOffset, int inputLength) { - return (int) ai.vespa.airlift.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength); + return (int) io.airlift.compress.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength); } } |