GC dompression support in java.

7 files changed, 59 insertions, 171 deletions

diff --git a/document/src/main/java/com/yahoo/document/BaseStructDataType.java b/document/src/main/java/com/yahoo/document/BaseStructDataType.java
index 97357f3fa7c..1bce5d716a4 100755
--- a/document/src/main/java/com/yahoo/document/BaseStructDataType.java
+++ b/document/src/main/java/com/yahoo/document/BaseStructDataType.java
@@ -18,8 +18,6 @@ public abstract class BaseStructDataType extends StructuredDataType {
     protected Map<Integer, Field> fieldIds = new LinkedHashMap<>();
     protected Map<String, Field> fields = new LinkedHashMap<>();
 
-    protected Compressor compressor = new Compressor(CompressionType.NONE);
-
     BaseStructDataType(String name) {
         super(name);
     }
@@ -101,27 +99,25 @@ public abstract class BaseStructDataType extends StructuredDataType {
         return fields.size();
     }
 
-    /** Returns the compressor to use to compress data of this type */
-    public Compressor getCompressor() { return compressor; }
+    /** Returns the compressor to use to compress data of this type
+     * @deprecated Will go away on Vespa 8
+     */
+    @Deprecated
+    public Compressor getCompressor() { return new Compressor(CompressionType.NONE); }
 
-    /** Returns a view of the configuration of the compressor used to compress this type */
+    /** Returns a view of the configuration of the compressor used to compress this type
+     * @deprecated Will go away on Vespa 8
+     */
+    @Deprecated
     public CompressionConfig getCompressionConfig() {
-        // CompressionConfig accepts a percentage (but exposes a factor) ...
-        float compressionThresholdPercentage = (float)compressor.compressionThresholdFactor() * 100;
-
-        return new CompressionConfig(compressor.type(),
-                                     compressor.level(),
-                                     compressionThresholdPercentage,
-                                     compressor.compressMinSizeBytes());
+        return new CompressionConfig();
     }
 
-    /** Set the config to the compressor used to compress data of this type */
-    public void setCompressionConfig(CompressionConfig config) {
-        CompressionType type = config.type;
-        compressor = new Compressor(type,
-                                    config.compressionLevel,
-                                    config.thresholdFactor(),
-                                    (int)config.minsize);
-    }
+    /**
+     *  Set the config to the compressor used to compress data of this type
+     * @deprecated Ignored and will go away on Vespa 8
+     */
+    @Deprecated
+    public void setCompressionConfig(CompressionConfig config) { }
 
 }
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index db0566aecc1..a0ac3cb6620 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -10,7 +10,6 @@ import java.util.logging.Level;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -24,17 +23,14 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
 
     private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName());
 
-    private DocumentTypeManager managerToConfigure;
+    private final DocumentTypeManager managerToConfigure;
 
     public DocumentTypeManagerConfigurer(DocumentTypeManager manager) {
         this.managerToConfigure = manager;
     }
 
-    private static CompressionConfig makeCompressionConfig(DocumentmanagerConfig.Datatype.Structtype cfg) {
-        return new CompressionConfig(toCompressorType(cfg.compresstype()), cfg.compresslevel(),
-                                     cfg.compressthreshold(), cfg.compressminsize());
-    }
-
+    /** Deprecated and will go away on Vespa 8 */
+    @Deprecated
     public static CompressionType toCompressorType(DocumentmanagerConfig.Datatype.Structtype.Compresstype.Enum value) {
         switch (value) {
             case NONE: return CompressionType.NONE;
@@ -43,7 +39,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         }
         throw new IllegalArgumentException("Compression type " + value + " is not supported");
     }
-
     /**
      * <p>Makes the DocumentTypeManager subscribe on its config.</p>
      *
@@ -73,8 +68,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         setupAnnotationRefTypes(config, manager);
 
         log.log(Level.FINE, "Configuring document manager with " + config.datatype().size() + " data types.");
-        ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>();
-        failed.addAll(config.datatype());
+        ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>(config.datatype());
         while (!failed.isEmpty()) {
             ArrayList<DocumentmanagerConfig.Datatype> tmp = failed;
             failed = new ArrayList<>();
@@ -82,7 +76,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                 DocumentmanagerConfig.Datatype thisDataType = tmp.get(i);
                 int id = thisDataType.id();
                 try {
-                    registerTypeIdMapping(config, manager, thisDataType, id);
+                    registerTypeIdMapping(manager, thisDataType, id);
                 } catch (IllegalArgumentException e) {
                     failed.add(thisDataType);
                 }
@@ -95,24 +89,24 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         manager.replaceTemporaryTypes();
     }
 
-    private static void registerTypeIdMapping(DocumentmanagerConfig config, DocumentTypeManager manager, DocumentmanagerConfig.Datatype thisDataType, int id) {
-        for (Object o : thisDataType.arraytype()) {
-            registerArrayType(manager, id, (DocumentmanagerConfig.Datatype.Arraytype) o);
+    private static void registerTypeIdMapping(DocumentTypeManager manager, DocumentmanagerConfig.Datatype thisDataType, int id) {
+        for (var o : thisDataType.arraytype()) {
+            registerArrayType(manager, id, o);
         }
-        for (Object o : thisDataType.maptype()) {
-            registerMapType(manager, id, (DocumentmanagerConfig.Datatype.Maptype) o);
+        for (var o : thisDataType.maptype()) {
+            registerMapType(manager, id, o);
         }
-        for (Object o : thisDataType.weightedsettype()) {
-            registerWeightedSetType(manager, id, (DocumentmanagerConfig.Datatype.Weightedsettype) o);
+        for (var o : thisDataType.weightedsettype()) {
+            registerWeightedSetType(manager, id, o);
         }
-        for (Object o : thisDataType.structtype()) {
-            registerStructType(config, manager, id, (DocumentmanagerConfig.Datatype.Structtype) o);
+        for (var o : thisDataType.structtype()) {
+            registerStructType(manager, id, o);
         }
-        for (Object o : thisDataType.documenttype()) {
-            registerDocumentType(manager, (DocumentmanagerConfig.Datatype.Documenttype) o);
+        for (var o : thisDataType.documenttype()) {
+            registerDocumentType(manager, o);
         }
-        for (Object o : thisDataType.referencetype()) {
-            registerReferenceType(manager, id, (DocumentmanagerConfig.Datatype.Referencetype) o);
+        for (var o : thisDataType.referencetype()) {
+            registerReferenceType(manager, id, o);
         }
     }
 
@@ -139,20 +133,17 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         manager.register(type);
     }
 
-    @SuppressWarnings("deprecation")
     private static void registerDocumentType(DocumentTypeManager manager, DocumentmanagerConfig.Datatype.Documenttype doc) {
         StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), "");
         var importedFields = doc.importedfield().stream()
                 .map(f -> f.name())
                 .collect(Collectors.toUnmodifiableSet());
         DocumentType type = new DocumentType(doc.name(), header, importedFields);
-        for (Object j : doc.inherits()) {
-            DocumentmanagerConfig.Datatype.Documenttype.Inherits parent =
-                    (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j;
+        for (var parent : doc.inherits()) {
             DataTypeName name = new DataTypeName(parent.name());
             DocumentType parentType = manager.getDocumentType(name);
             if (parentType == null) {
-                throw new IllegalArgumentException("Could not find document type '" + name.toString() + "'.");
+                throw new IllegalArgumentException("Could not find document type '" + name + "'.");
             }
             type.inherit(parentType);
         }
@@ -164,18 +155,11 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         manager.register(type);
     }
 
-    private static void registerStructType(DocumentmanagerConfig config, DocumentTypeManager manager, int id,
+    private static void registerStructType(DocumentTypeManager manager, int id,
                                            DocumentmanagerConfig.Datatype.Structtype struct) {
         StructDataType type = new StructDataType(id, struct.name());
 
-        if (config.enablecompression()) {
-            CompressionConfig comp = makeCompressionConfig(struct);
-            type.setCompressionConfig(comp);
-        }
-
-        for (Object j : struct.field()) {
-            DocumentmanagerConfig.Datatype.Structtype.Field field =
-                    (DocumentmanagerConfig.Datatype.Structtype.Field) j;
+        for (var field : struct.field()) {
             DataType fieldType = (field.datatype() == id)
                                ? manager.getDataTypeAndReturnTemporary(field.datatype(), field.detailedtype())
                                : manager.getDataType(field.datatype(), field.detailedtype());
@@ -231,8 +215,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         for (int i = 0; i < config.datatype().size(); i++) {
             DocumentmanagerConfig.Datatype thisDataType = config.datatype(i);
             int id = thisDataType.id();
-            for (Object o : thisDataType.annotationreftype()) {
-                DocumentmanagerConfig.Datatype.Annotationreftype annRefType = (DocumentmanagerConfig.Datatype.Annotationreftype) o;
+            for (var annRefType : thisDataType.annotationreftype()) {
                 AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annRefType.annotation());
                 if (annotationType == null) {
                     throw new IllegalArgumentException("Found reference to " + annRefType.annotation() + ", which does not exist!");
@@ -275,11 +258,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         for (int i = 0; i < config.datatype().size(); i++) {
             DocumentmanagerConfig.Datatype thisDataType = config.datatype(i);
             int id = thisDataType.id();
-            for (Object o : thisDataType.structtype()) {
-                DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o;
+            for (var struct : thisDataType.structtype()) {
                 StructDataType thisStruct = (StructDataType) manager.getDataType(id, "");
 
-                for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) {
+                for (var parent : struct.inherits()) {
                     StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name());
                     thisStruct.inherit(parentStruct);
                 }
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java
index fd13885ac36..b47dbe69199 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java
@@ -65,25 +65,20 @@ public class Struct extends StructuredFieldValue {
         return this.version;
     }
 
+    /** @deprecated Will go away on Vespa 8 */
+    @Deprecated
     public com.yahoo.compress.CompressionType getCompressionType() {
-        if (getDataType().getCompressionConfig() == null) {
-            return com.yahoo.compress.CompressionType.NONE;
-        }
-        return getDataType().getCompressionConfig().type;
+        return com.yahoo.compress.CompressionType.NONE;
     }
 
-    public int getCompressionLevel() {
-        if ( getDataType().getCompressionConfig() == null) {
-            return 9;
-        }
-        return getDataType().getCompressionConfig().compressionLevel;
-    }
+    /** @deprecated Will go away on Vespa 8 */
+    @Deprecated
+    public int getCompressionLevel() { return 9; }
 
+    /** @deprecated Will go away on Vespa 8 */
+    @Deprecated
     public float getCompressionThreshold() {
-        if (getDataType().getCompressionConfig() == null) {
-            return .95f;
-        }
-        return getDataType().getCompressionConfig().threshold;
+        return .95f;
     }
 
     @Override
diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java
index cbcd0c64bd0..09e41a0e8bf 100644
--- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java
+++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java
@@ -1,7 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.serialization;
 
-import com.yahoo.compress.Compressor;
+import com.yahoo.compress.CompressionType;
 
 import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.CollectionDataType;
@@ -53,7 +53,6 @@ import com.yahoo.document.update.TensorModifyUpdate;
 import com.yahoo.document.update.TensorRemoveUpdate;
 import com.yahoo.document.update.ValueUpdate;
 import com.yahoo.io.GrowableByteBuffer;
-import com.yahoo.tensor.serialization.TypedBinaryFormat;
 import com.yahoo.vespa.objects.BufferSerializer;
 import com.yahoo.vespa.objects.FieldBase;
 
@@ -344,19 +343,10 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume
         buffer.flip();
         buf = bigBuffer;
 
-        int uncompressedSize = buffer.remaining();
-        Compressor.Compression compression =
-            s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining());
-
         // Actual serialization starts here.
         int lenPos = buf.position();
         putInt(null, 0); // Move back to this after compression is done.
-        buf.put(compression.type().getCode());
-
-        if (compression.data() != null && compression.type().isCompressed()) {
-            buf.putInt2_4_8Bytes(uncompressedSize);
-        }
-
+        buf.put(CompressionType.NONE.getCode());
         buf.putInt1_4Bytes(s.getFieldCount());
 
         for (int i = 0; i < s.getFieldCount(); ++i) {
@@ -365,11 +355,7 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume
         }
 
         int pos = buf.position();
-        if (compression.data() != null && compression.type().isCompressed()) {
-            put(null, compression.data());
-        } else {
-            put(null, buffer.getByteBuffer());
-        }
+        put(null, buffer.getByteBuffer());
         int dataLength = buf.position() - pos;
 
         int posNow = buf.position();
diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
index 5f283df9614..db877bb9712 100644
--- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
@@ -112,21 +112,11 @@ public class DocumentSerializationTestCase extends AbstractTypesTest {
             weightedSet.put(new StringFieldValue("Weighted 1"), 199);
             doc.setFieldValue("wsfield", weightedSet);
 
-            CompressionConfig noncomp = new CompressionConfig();
-            CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
             {
-                doc.getDataType().contentStruct().setCompressionConfig(noncomp);
                 FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false);
                 doc.serialize(fout);
                 fout.close();
             }
-            {
-                doc.getDataType().contentStruct().setCompressionConfig(lz4comp);
-                FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false);
-                doc.serialize(fout);
-                doc.getDataType().contentStruct().setCompressionConfig(noncomp);
-                fout.close();
-            }
         }
 
         class TestDoc {
diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java
index 5a694d21dd8..144c7d62894 100644
--- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java
@@ -428,7 +428,7 @@ public class DocumentTestCase extends DocumentTestCaseBase {
         }
     }
 
-    class ModifyIteratorHandler extends FieldPathIteratorHandler {
+    static class ModifyIteratorHandler extends FieldPathIteratorHandler {
 
         public ModificationStatus doModify(FieldValue fv) {
             if (fv instanceof StringFieldValue) {
@@ -444,7 +444,7 @@ public class DocumentTestCase extends DocumentTestCaseBase {
         }
     }
 
-    class AddIteratorHandler extends FieldPathIteratorHandler {
+    static class AddIteratorHandler extends FieldPathIteratorHandler {
 
         @SuppressWarnings("unchecked")
         public ModificationStatus doModify(FieldValue fv) {
@@ -658,7 +658,7 @@ public class DocumentTestCase extends DocumentTestCaseBase {
         validateCppDoc(doc);
     }
 
-    public void validateCppDoc(Document doc) throws IOException {
+    public void validateCppDoc(Document doc) {
         validateCppDocNotMap(doc);
         MapFieldValue map = (MapFieldValue)doc.getFieldValue("mapfield");
         assertEquals(map.get(new StringFieldValue("foo1")), new StringFieldValue("bar1"));
@@ -666,7 +666,7 @@ public class DocumentTestCase extends DocumentTestCaseBase {
     }
 
     @SuppressWarnings("unchecked")
-    public void validateCppDocNotMap(Document doc) throws IOException {
+    public void validateCppDocNotMap(Document doc) {
         // in practice to validate v6 serialization
         assertEquals("id:ns:serializetest::http://test.doc.id/", doc.getId().toString());
         assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
@@ -694,7 +694,6 @@ public class DocumentTestCase extends DocumentTestCaseBase {
     }
 
     @Test
-    @SuppressWarnings("deprecation")
     public void testGenerateSerializedFile() throws IOException {
 
         docMan = setUpCppDocType();
@@ -744,18 +743,6 @@ public class DocumentTestCase extends DocumentTestCaseBase {
         FileOutputStream fos = new FileOutputStream("src/tests/data/serializejava.dat");
         fos.write(buf.array(), 0, size);
         fos.close();
-
-        CompressionConfig noncomp = new CompressionConfig();
-        CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
-
-        doc.getDataType().contentStruct().setCompressionConfig(lz4comp);
-        buf = new GrowableByteBuffer(size, 2.0f);
-
-        doc.serialize(buf);
-        doc.getDataType().contentStruct().setCompressionConfig(noncomp);
-        fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat");
-        fos.write(buf.array(), 0, buf.position());
-        fos.close();
     }
 
     @Test
@@ -801,53 +788,6 @@ public class DocumentTestCase extends DocumentTestCaseBase {
     }
 
     @Test
-    @SuppressWarnings("deprecation")
-    public void testSerializeDeserializeCompressed() {
-        setUpSertestDocType();
-        Document doc = getSertestDocument();
-
-        CompressionConfig noncomp = new CompressionConfig();
-        CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
-
-        doc.getDataType().contentStruct().setCompressionConfig(lz4comp);
-
-        GrowableByteBuffer data = new GrowableByteBuffer();
-        doc.serialize(data);
-        int size = doc.getSerializedSize();
-        doc.getDataType().contentStruct().setCompressionConfig(noncomp);
-
-        assertEquals(size, data.position());
-
-        data.flip();
-
-        try {
-            FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
-            fos.write(data.array(), 0, data.remaining());
-            fos.close();
-        } catch (Exception e) {
-        }
-
-        Document doc2 = docMan.createDocument(data);
-
-        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
-        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
-        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
-        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
-        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
-        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
-        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
-        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
-        byte[] docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
-        byte[] doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
-        assertEquals(docRawBytes.length, doc2RawBytes.length);
-        for (int i = 0; i < docRawBytes.length; i++) {
-            assertEquals(docRawBytes[i], doc2RawBytes[i]);
-        }
-        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
-        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
-    }
-
-    @Test
     public void testDeserialize() {
         setUpSertestDocType();
 
@@ -937,7 +877,7 @@ public class DocumentTestCase extends DocumentTestCaseBase {
     }
 
     @Test
-    public void testCompressionConfigured() {
+    public void testCompressionConfiguredIsIgnored() {
 
         int size_uncompressed;
         {
@@ -966,7 +906,7 @@ public class DocumentTestCase extends DocumentTestCaseBase {
         doc.serialize(data);
         int size_compressed = data.position();
 
-        assertTrue(size_compressed + " < " + size_uncompressed, size_compressed < size_uncompressed);
+        assertEquals(size_compressed, size_uncompressed);
     }
 
     @Test
diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java
index 379089f1c79..ff66da9a3d7 100644
--- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java
+++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java
@@ -68,7 +68,6 @@ public class VespaDocumentSerializerTestCase {
 
         CompressionFixture() {
             docType = new DocumentType("map_of_structs");
-            docType.contentStruct().setCompressionConfig(new CompressionConfig(CompressionType.LZ4));
 
             nestedType = new StructDataType("nested_type");
             nestedType.addField(new Field("str", DataType.STRING));