From 5fce9fedea4b8259d8d1bc1d26d47cc3b837b252 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 7 Jun 2022 13:13:45 +0200 Subject: Further GC document level compression. Avoids a buffer copy that is no longer relevant. --- .../com/yahoo/document/BaseStructDataType.java | 3 -- .../document/DocumentTypeManagerConfigurer.java | 2 - .../serialization/VespaDocumentDeserializer6.java | 51 ++++------------------ .../serialization/VespaDocumentSerializer6.java | 2 +- 4 files changed, 9 insertions(+), 49 deletions(-) (limited to 'document/src/main/java') diff --git a/document/src/main/java/com/yahoo/document/BaseStructDataType.java b/document/src/main/java/com/yahoo/document/BaseStructDataType.java index a84a08c5677..adecae9d7eb 100755 --- a/document/src/main/java/com/yahoo/document/BaseStructDataType.java +++ b/document/src/main/java/com/yahoo/document/BaseStructDataType.java @@ -1,9 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.compress.CompressionType; -import com.yahoo.compress.Compressor; - import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 2f2f5647603..d6833a482f1 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.compress.CompressionType; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; @@ -434,7 +433,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } void createEmptyStructs() { - String docName = docTypeConfig.name(); for (var typeconf : docTypeConfig.structtype()) { if (isPositionStruct(typeconf)) { int geoVersion = usev8geopositions ? 8 : 7; diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java index 47ae31b53fe..2796609543f 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -2,8 +2,6 @@ package com.yahoo.document.serialization; import com.yahoo.collections.Tuple2; -import com.yahoo.compress.CompressionType; -import com.yahoo.compress.Compressor; import com.yahoo.document.annotation.AlternateSpanList; import com.yahoo.document.annotation.Annotation; import com.yahoo.document.annotation.AnnotationReference; @@ -69,7 +67,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import static com.yahoo.text.Utf8.calculateStringPositions; @@ -80,8 +77,7 @@ import static com.yahoo.text.Utf8.calculateStringPositions; */ public class VespaDocumentDeserializer6 extends BufferSerializer implements DocumentDeserializer { - private final Compressor compressor = new Compressor(); - private DocumentTypeManager manager; + private final DocumentTypeManager manager; private short version; private List spanNodes; private List annotations; @@ -263,21 +259,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu } int dataSize = getInt(null); - byte comprCode = getByte(null); - CompressionType compression = CompressionType.valueOf(comprCode); - - int uncompressedSize = 0; - if (compression != CompressionType.NONE && - compression != CompressionType.INCOMPRESSIBLE) - { - // uncompressedsize (full size of FIELDS only, after decompression) - long pSize = getInt2_4_8Bytes(null); - //TODO: Look into how to support data segments larger than INT_MAX bytes - if (pSize > Integer.MAX_VALUE) { - throw new DeserializationException("Uncompressed size of data block is too large."); - } - uncompressedSize = (int) pSize; - } + byte ignoredComprCode = getByte(null); int numberOfFields = getInt1_4Bytes(null); @@ -289,14 +271,12 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu // save a reference to the big buffer we're reading from: GrowableByteBuffer bigBuf = buf; - - byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); - + GrowableByteBuffer thisStructOnly = GrowableByteBuffer.wrap(getBuf().array(), position(), dataSize); // set position in original buffer to after data position(position() + dataSize); // for a while: deserialize from this buffer instead: - buf = GrowableByteBuffer.wrap(destination); + buf = thisStructOnly; s.clear(); StructDataType type = s.getDataType(); @@ -325,21 +305,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu } int dataSize = getInt(null); - byte comprCode = getByte(null); - CompressionType compression = CompressionType.valueOf(comprCode); - - int uncompressedSize = 0; - if (compression != CompressionType.NONE && - compression != CompressionType.INCOMPRESSIBLE) - { - // uncompressedsize (full size of FIELDS only, after decompression) - long pSize = getInt2_4_8Bytes(null); - //TODO: Look into how to support data segments larger than INT_MAX bytes - if (pSize > Integer.MAX_VALUE) { - throw new DeserializationException("Uncompressed size of data block is too large."); - } - uncompressedSize = (int) pSize; - } + byte unusedComprCode = getByte(null); int numberOfFields = getInt1_4Bytes(null); @@ -351,14 +317,13 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu // save a reference to the big buffer we're reading from: GrowableByteBuffer bigBuf = buf; - - byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); + GrowableByteBuffer thisStructOnly = GrowableByteBuffer.wrap(getBuf().array(), position(), dataSize); // set position in original buffer to after data position(position() + dataSize); // for a while: deserialize from this buffer instead: - buf = GrowableByteBuffer.wrap(destination); + buf = thisStructOnly; StructDataType priType = target.getDataType().contentStruct(); @@ -613,7 +578,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu DocumentType docType = manager.getDocumentType(new DataTypeName(docTypeName)); if (docType == null) { throw new DeserializationException("No known document type with name " + - new Utf8String(docTypeName).toString()); + new Utf8String(docTypeName)); } return docType; } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index ee36dcb1a9c..2de345b6e35 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -483,7 +483,7 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume write(tree.getRoot()); { //add all annotations to temporary list and sort it, to get predictable serialization - List tmpAnnotationList = new ArrayList(tree.numAnnotations()); + List tmpAnnotationList = new ArrayList<>(tree.numAnnotations()); for (Annotation annotation : tree) { tmpAnnotationList.add(annotation); } -- cgit v1.2.3