diff options
10 files changed, 63 insertions, 33 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index 7fd36b0d142..1719ea72cb0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -38,7 +38,7 @@ public class DistributableResource { Objects.requireNonNull(blob, "Blob cannot be null"); this.name = name; this.blob = blob; - path = name; + path = name + ".lz4"; pathType = PathType.BLOB; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index bde8567c34e..52d443e6998 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.io.IOUtils; +import net.jpountz.lz4.LZ4FrameOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -70,7 +71,13 @@ public class ApplicationFileManager implements AddFileInterface { file = new File(path.toFile(), relativePath); Files.createDirectories(file.getParentFile().toPath()); fos = new FileOutputStream(file); - fos.write(blob.array(), blob.arrayOffset(), blob.remaining()); + if (relativePath.endsWith(".lz4")) { + LZ4FrameOutputStream lz4 = new LZ4FrameOutputStream(fos); + lz4.write(blob.array(), blob.arrayOffset(), blob.remaining()); + lz4.close(); + } else { + fos.write(blob.array(), blob.arrayOffset(), blob.remaining()); + } return file; } catch (IOException e) { throw new IllegalArgumentException("Failed creating temp file", e); diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index eccc236f0ca..fbfd34814ac 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -17,8 +17,10 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import net.jpountz.lz4.LZ4FrameInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -207,10 +209,16 @@ public class RankProfilesConfigImporter { return expressions; } + protected final String readExpressionFromFile(File file) throws IOException { + return (file.getName().endsWith(".lz4")) + ? Utf8.toString(IOUtils.readBytes(new LZ4FrameInputStream(new FileInputStream(file)), 65536)) + : Utf8.toString(IOUtils.readFileBytes(file)); + } + protected RankingExpression readExpressionFromFile(String name, FileReference fileReference) throws ParseException { try { File file = fileAcquirer.waitFor(fileReference, 7, TimeUnit.DAYS); - return new RankingExpression(name, Utf8.toString(IOUtils.readFileBytes(file))); + return new RankingExpression(name, readExpressionFromFile(file)); } catch (InterruptedException e) { throw new IllegalStateException("Gave up waiting for expression " + name); @@ -245,8 +253,8 @@ public class RankProfilesConfigImporter { private static final Pattern valuePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.value"); private static final Pattern typePattern = Pattern.compile("constant\\(([a-zA-Z0-9_.]+)\\)\\.type"); - private Map<String, TensorType> types = new HashMap<>(); - private Map<String, String> values = new HashMap<>(); + private final Map<String, TensorType> types = new HashMap<>(); + private final Map<String, String> values = new HashMap<>(); void addIfSmallConstantInfo(String key, String value) { tryValue(key, value); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java index 20993583d41..250ae9a4883 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java @@ -10,7 +10,6 @@ import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; -import com.yahoo.text.Utf8; import java.io.IOException; import java.util.Optional; @@ -44,7 +43,7 @@ public class RankProfilesConfigImporterWithMockedConstants extends RankProfilesC @Override protected RankingExpression readExpressionFromFile(String name, FileReference fileReference) throws ParseException { try { - return new RankingExpression(name, Utf8.toString(IOUtils.readFileBytes(constantsPath.append(fileReference.value()).toFile()))); + return new RankingExpression(name, readExpressionFromFile(constantsPath.append(fileReference.value()).toFile())); } catch (IOException e) { throw new IllegalArgumentException("Missing expression file '" + fileReference.value() + "' for expression '" + name + "'.", e); } diff --git a/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 b/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 Binary files differnew file mode 100644 index 00000000000..30f23b963db --- /dev/null +++ b/model-evaluation/src/test/resources/config/rankexpression/constants/overflow.firstphase.expr.lz4 diff --git a/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg b/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg index b82aca648a8..aab15fd400f 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg +++ b/model-evaluation/src/test/resources/config/rankexpression/rank-profiles.cfg @@ -84,8 +84,8 @@ rankprofile[3].fef.property[1].name "rankingExpression(firstphase).expressionNam rankprofile[3].fef.property[1].value "overflow.firstphase" rankprofile[3].fef.property[2].name "vespa.rank.secondphase" rankprofile[3].fef.property[2].value "rankingExpression(secondphase)" -rankprofile[3].fef.property[3].name "rankingExpression(secondphase).rankingScript" -rankprofile[3].fef.property[3].value "exp(0) + mysum(attribute(foo),\"attribute( bar )\",\"attribute( \\\"baz\\\" )\")" +rankprofile[3].fef.property[3].name "rankingExpression(secondphase).expressionName" +rankprofile[3].fef.property[3].value "overflow.secondphase" rankprofile[3].fef.property[4].name "vespa.hitcollector.heapsize" rankprofile[3].fef.property[4].value "101" rankprofile[3].fef.property[5].name "vespa.hitcollector.arraysize" diff --git a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg index f7106d02ad3..8cb02567538 100644 --- a/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg +++ b/model-evaluation/src/test/resources/config/rankexpression/ranking-expressions.cfg @@ -1,2 +1,4 @@ expression[0].name "overflow.firstphase" expression[0].fileref "overflow.firstphase.expr" +expression[1].name "overflow.secondphase" +expression[1].fileref "overflow.firstphase.expr.lz4" diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index ad51ac1ca38..98228292cc7 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -82,22 +82,25 @@ public class Compressor { * @throws IllegalArgumentException if the compression type is not supported */ public Compression compress(CompressionType requestedCompression, byte[] data, Optional<Integer> uncompressedSize) { + return compress(requestedCompression, data, 0, uncompressedSize.orElse(data.length)); + } + public Compression compress(CompressionType requestedCompression, byte[] data, int offset, int len) { switch (requestedCompression) { case NONE: - data = uncompressedSize.isPresent() ? Arrays.copyOf(data, uncompressedSize.get()) : data; + if ((offset != 0) || (len != data.length)) { + data = Arrays.copyOfRange(data, offset, offset + len); + } return new Compression(CompressionType.NONE, data.length, data); case LZ4: - int dataSize = uncompressedSize.isPresent() ? uncompressedSize.get() : data.length; - if (dataSize < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data); - byte[] compressedData = getCompressor().compress(data, 0, dataSize); - if (compressedData.length + 8 >= dataSize * compressionThresholdFactor) - return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data); - return new Compression(CompressionType.LZ4, dataSize, compressedData); + if (len < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, len, data); + byte[] compressedData = getCompressor().compress(data, offset, len); + if (compressedData.length + 8 >= len * compressionThresholdFactor) + return new Compression(CompressionType.INCOMPRESSIBLE, len, data); + return new Compression(CompressionType.LZ4, len, compressedData); case ZSTD: - int dataLength = uncompressedSize.orElse(data.length); - if (dataLength < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataLength, data); - byte[] compressed = zstdCompressor.compress(data, 0, dataLength); - return new Compression(CompressionType.ZSTD, dataLength, compressed); + if (len < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, len, data); + byte[] compressed = zstdCompressor.compress(data, offset, len); + return new Compression(CompressionType.ZSTD, len, compressed); default: throw new IllegalArgumentException(requestedCompression + " is not supported"); } diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java index fa32ec1ff9c..64140ab9706 100644 --- a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java +++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java @@ -2,8 +2,25 @@ package com.yahoo.io; -import java.io.*; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.io.Writer; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.util.List; @@ -19,7 +36,7 @@ import java.nio.ByteBuffer; */ public abstract class IOUtils { - static private final Charset utf8Charset = Charset.forName("utf-8"); + static private final Charset utf8Charset = StandardCharsets.UTF_8; /** Closes a writer, or does nothing if the writer is null */ public static void closeWriter(Writer writer) { @@ -341,7 +358,7 @@ public abstract class IOUtils { public static String readFile(File file) throws IOException { try { if (file == null) return null; - return new String(Files.readAllBytes(file.toPath()), "utf-8"); + return Files.readString(file.toPath(), utf8Charset); } catch (NoSuchFileException e) { throw new NoSuchFileException("Could not find file '" + file.getAbsolutePath() + "'"); @@ -370,9 +387,7 @@ public abstract class IOUtils { if (lengthL>Integer.MAX_VALUE) throw new IllegalArgumentException("File too big for byte array: "+file.getCanonicalPath()); - InputStream in = null; - try { - in = new FileInputStream(file); + try (InputStream in = new FileInputStream(file)) { int length = (int)lengthL; byte[] array = new byte[length]; int offset = 0; @@ -381,10 +396,6 @@ public abstract class IOUtils { offset += count; return array; } - finally { - if (in != null) - in.close(); - } } /** diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java index 26c76de4bc1..20ed0fc8018 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java @@ -33,16 +33,16 @@ final class BufferedOutput { public int position() { return pos; } - final void put(byte b) { + void put(byte b) { reserve(1); buf[pos++] = b; } - final void absolutePut(int position, byte b) { + void absolutePut(int position, byte b) { buf[position] = b; } - final void put(byte[] bytes) { + void put(byte[] bytes) { reserve(bytes.length); for (byte b : bytes) { buf[pos++] = b; |