diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-11-10 20:23:53 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-11-10 20:25:24 +0100 |
commit | c272d764d842f04d864bcca5b57468ec7e8aad92 (patch) | |
tree | 5626b49fc39c09c479bd4867eaa1994e7b2bc194 /document | |
parent | 566108512ea68568354c6798180ff1a750c6b6aa (diff) |
GC dompression support in java.
Diffstat (limited to 'document')
7 files changed, 59 insertions, 171 deletions
diff --git a/document/src/main/java/com/yahoo/document/BaseStructDataType.java b/document/src/main/java/com/yahoo/document/BaseStructDataType.java index 97357f3fa7c..1bce5d716a4 100755 --- a/document/src/main/java/com/yahoo/document/BaseStructDataType.java +++ b/document/src/main/java/com/yahoo/document/BaseStructDataType.java @@ -18,8 +18,6 @@ public abstract class BaseStructDataType extends StructuredDataType { protected Map<Integer, Field> fieldIds = new LinkedHashMap<>(); protected Map<String, Field> fields = new LinkedHashMap<>(); - protected Compressor compressor = new Compressor(CompressionType.NONE); - BaseStructDataType(String name) { super(name); } @@ -101,27 +99,25 @@ public abstract class BaseStructDataType extends StructuredDataType { return fields.size(); } - /** Returns the compressor to use to compress data of this type */ - public Compressor getCompressor() { return compressor; } + /** Returns the compressor to use to compress data of this type + * @deprecated Will go away on Vespa 8 + */ + @Deprecated + public Compressor getCompressor() { return new Compressor(CompressionType.NONE); } - /** Returns a view of the configuration of the compressor used to compress this type */ + /** Returns a view of the configuration of the compressor used to compress this type + * @deprecated Will go away on Vespa 8 + */ + @Deprecated public CompressionConfig getCompressionConfig() { - // CompressionConfig accepts a percentage (but exposes a factor) ... - float compressionThresholdPercentage = (float)compressor.compressionThresholdFactor() * 100; - - return new CompressionConfig(compressor.type(), - compressor.level(), - compressionThresholdPercentage, - compressor.compressMinSizeBytes()); + return new CompressionConfig(); } - /** Set the config to the compressor used to compress data of this type */ - public void setCompressionConfig(CompressionConfig config) { - CompressionType type = config.type; - compressor = new Compressor(type, - config.compressionLevel, - config.thresholdFactor(), - (int)config.minsize); - } + /** + * Set the config to the compressor used to compress data of this type + * @deprecated Ignored and will go away on Vespa 8 + */ + @Deprecated + public void setCompressionConfig(CompressionConfig config) { } } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index db0566aecc1..a0ac3cb6620 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -10,7 +10,6 @@ import java.util.logging.Level; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -24,17 +23,14 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName()); - private DocumentTypeManager managerToConfigure; + private final DocumentTypeManager managerToConfigure; public DocumentTypeManagerConfigurer(DocumentTypeManager manager) { this.managerToConfigure = manager; } - private static CompressionConfig makeCompressionConfig(DocumentmanagerConfig.Datatype.Structtype cfg) { - return new CompressionConfig(toCompressorType(cfg.compresstype()), cfg.compresslevel(), - cfg.compressthreshold(), cfg.compressminsize()); - } - + /** Deprecated and will go away on Vespa 8 */ + @Deprecated public static CompressionType toCompressorType(DocumentmanagerConfig.Datatype.Structtype.Compresstype.Enum value) { switch (value) { case NONE: return CompressionType.NONE; @@ -43,7 +39,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } throw new IllegalArgumentException("Compression type " + value + " is not supported"); } - /** * <p>Makes the DocumentTypeManager subscribe on its config.</p> * @@ -73,8 +68,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub setupAnnotationRefTypes(config, manager); log.log(Level.FINE, "Configuring document manager with " + config.datatype().size() + " data types."); - ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>(); - failed.addAll(config.datatype()); + ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>(config.datatype()); while (!failed.isEmpty()) { ArrayList<DocumentmanagerConfig.Datatype> tmp = failed; failed = new ArrayList<>(); @@ -82,7 +76,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub DocumentmanagerConfig.Datatype thisDataType = tmp.get(i); int id = thisDataType.id(); try { - registerTypeIdMapping(config, manager, thisDataType, id); + registerTypeIdMapping(manager, thisDataType, id); } catch (IllegalArgumentException e) { failed.add(thisDataType); } @@ -95,24 +89,24 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub manager.replaceTemporaryTypes(); } - private static void registerTypeIdMapping(DocumentmanagerConfig config, DocumentTypeManager manager, DocumentmanagerConfig.Datatype thisDataType, int id) { - for (Object o : thisDataType.arraytype()) { - registerArrayType(manager, id, (DocumentmanagerConfig.Datatype.Arraytype) o); + private static void registerTypeIdMapping(DocumentTypeManager manager, DocumentmanagerConfig.Datatype thisDataType, int id) { + for (var o : thisDataType.arraytype()) { + registerArrayType(manager, id, o); } - for (Object o : thisDataType.maptype()) { - registerMapType(manager, id, (DocumentmanagerConfig.Datatype.Maptype) o); + for (var o : thisDataType.maptype()) { + registerMapType(manager, id, o); } - for (Object o : thisDataType.weightedsettype()) { - registerWeightedSetType(manager, id, (DocumentmanagerConfig.Datatype.Weightedsettype) o); + for (var o : thisDataType.weightedsettype()) { + registerWeightedSetType(manager, id, o); } - for (Object o : thisDataType.structtype()) { - registerStructType(config, manager, id, (DocumentmanagerConfig.Datatype.Structtype) o); + for (var o : thisDataType.structtype()) { + registerStructType(manager, id, o); } - for (Object o : thisDataType.documenttype()) { - registerDocumentType(manager, (DocumentmanagerConfig.Datatype.Documenttype) o); + for (var o : thisDataType.documenttype()) { + registerDocumentType(manager, o); } - for (Object o : thisDataType.referencetype()) { - registerReferenceType(manager, id, (DocumentmanagerConfig.Datatype.Referencetype) o); + for (var o : thisDataType.referencetype()) { + registerReferenceType(manager, id, o); } } @@ -139,20 +133,17 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub manager.register(type); } - @SuppressWarnings("deprecation") private static void registerDocumentType(DocumentTypeManager manager, DocumentmanagerConfig.Datatype.Documenttype doc) { StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), ""); var importedFields = doc.importedfield().stream() .map(f -> f.name()) .collect(Collectors.toUnmodifiableSet()); DocumentType type = new DocumentType(doc.name(), header, importedFields); - for (Object j : doc.inherits()) { - DocumentmanagerConfig.Datatype.Documenttype.Inherits parent = - (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j; + for (var parent : doc.inherits()) { DataTypeName name = new DataTypeName(parent.name()); DocumentType parentType = manager.getDocumentType(name); if (parentType == null) { - throw new IllegalArgumentException("Could not find document type '" + name.toString() + "'."); + throw new IllegalArgumentException("Could not find document type '" + name + "'."); } type.inherit(parentType); } @@ -164,18 +155,11 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub manager.register(type); } - private static void registerStructType(DocumentmanagerConfig config, DocumentTypeManager manager, int id, + private static void registerStructType(DocumentTypeManager manager, int id, DocumentmanagerConfig.Datatype.Structtype struct) { StructDataType type = new StructDataType(id, struct.name()); - if (config.enablecompression()) { - CompressionConfig comp = makeCompressionConfig(struct); - type.setCompressionConfig(comp); - } - - for (Object j : struct.field()) { - DocumentmanagerConfig.Datatype.Structtype.Field field = - (DocumentmanagerConfig.Datatype.Structtype.Field) j; + for (var field : struct.field()) { DataType fieldType = (field.datatype() == id) ? manager.getDataTypeAndReturnTemporary(field.datatype(), field.detailedtype()) : manager.getDataType(field.datatype(), field.detailedtype()); @@ -231,8 +215,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (int i = 0; i < config.datatype().size(); i++) { DocumentmanagerConfig.Datatype thisDataType = config.datatype(i); int id = thisDataType.id(); - for (Object o : thisDataType.annotationreftype()) { - DocumentmanagerConfig.Datatype.Annotationreftype annRefType = (DocumentmanagerConfig.Datatype.Annotationreftype) o; + for (var annRefType : thisDataType.annotationreftype()) { AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annRefType.annotation()); if (annotationType == null) { throw new IllegalArgumentException("Found reference to " + annRefType.annotation() + ", which does not exist!"); @@ -275,11 +258,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (int i = 0; i < config.datatype().size(); i++) { DocumentmanagerConfig.Datatype thisDataType = config.datatype(i); int id = thisDataType.id(); - for (Object o : thisDataType.structtype()) { - DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o; + for (var struct : thisDataType.structtype()) { StructDataType thisStruct = (StructDataType) manager.getDataType(id, ""); - for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) { + for (var parent : struct.inherits()) { StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name()); thisStruct.inherit(parentStruct); } diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index fd13885ac36..b47dbe69199 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -65,25 +65,20 @@ public class Struct extends StructuredFieldValue { return this.version; } + /** @deprecated Will go away on Vespa 8 */ + @Deprecated public com.yahoo.compress.CompressionType getCompressionType() { - if (getDataType().getCompressionConfig() == null) { - return com.yahoo.compress.CompressionType.NONE; - } - return getDataType().getCompressionConfig().type; + return com.yahoo.compress.CompressionType.NONE; } - public int getCompressionLevel() { - if ( getDataType().getCompressionConfig() == null) { - return 9; - } - return getDataType().getCompressionConfig().compressionLevel; - } + /** @deprecated Will go away on Vespa 8 */ + @Deprecated + public int getCompressionLevel() { return 9; } + /** @deprecated Will go away on Vespa 8 */ + @Deprecated public float getCompressionThreshold() { - if (getDataType().getCompressionConfig() == null) { - return .95f; - } - return getDataType().getCompressionConfig().threshold; + return .95f; } @Override diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index cbcd0c64bd0..09e41a0e8bf 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import com.yahoo.compress.Compressor; +import com.yahoo.compress.CompressionType; import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; @@ -53,7 +53,6 @@ import com.yahoo.document.update.TensorModifyUpdate; import com.yahoo.document.update.TensorRemoveUpdate; import com.yahoo.document.update.ValueUpdate; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.FieldBase; @@ -344,19 +343,10 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume buffer.flip(); buf = bigBuffer; - int uncompressedSize = buffer.remaining(); - Compressor.Compression compression = - s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining()); - // Actual serialization starts here. int lenPos = buf.position(); putInt(null, 0); // Move back to this after compression is done. - buf.put(compression.type().getCode()); - - if (compression.data() != null && compression.type().isCompressed()) { - buf.putInt2_4_8Bytes(uncompressedSize); - } - + buf.put(CompressionType.NONE.getCode()); buf.putInt1_4Bytes(s.getFieldCount()); for (int i = 0; i < s.getFieldCount(); ++i) { @@ -365,11 +355,7 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume } int pos = buf.position(); - if (compression.data() != null && compression.type().isCompressed()) { - put(null, compression.data()); - } else { - put(null, buffer.getByteBuffer()); - } + put(null, buffer.getByteBuffer()); int dataLength = buf.position() - pos; int posNow = buf.position(); diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index 5f283df9614..db877bb9712 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -112,21 +112,11 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { weightedSet.put(new StringFieldValue("Weighted 1"), 199); doc.setFieldValue("wsfield", weightedSet); - CompressionConfig noncomp = new CompressionConfig(); - CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); { - doc.getDataType().contentStruct().setCompressionConfig(noncomp); FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false); doc.serialize(fout); fout.close(); } - { - doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false); - doc.serialize(fout); - doc.getDataType().contentStruct().setCompressionConfig(noncomp); - fout.close(); - } } class TestDoc { diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 5a694d21dd8..144c7d62894 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -428,7 +428,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } } - class ModifyIteratorHandler extends FieldPathIteratorHandler { + static class ModifyIteratorHandler extends FieldPathIteratorHandler { public ModificationStatus doModify(FieldValue fv) { if (fv instanceof StringFieldValue) { @@ -444,7 +444,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } } - class AddIteratorHandler extends FieldPathIteratorHandler { + static class AddIteratorHandler extends FieldPathIteratorHandler { @SuppressWarnings("unchecked") public ModificationStatus doModify(FieldValue fv) { @@ -658,7 +658,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { validateCppDoc(doc); } - public void validateCppDoc(Document doc) throws IOException { + public void validateCppDoc(Document doc) { validateCppDocNotMap(doc); MapFieldValue map = (MapFieldValue)doc.getFieldValue("mapfield"); assertEquals(map.get(new StringFieldValue("foo1")), new StringFieldValue("bar1")); @@ -666,7 +666,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @SuppressWarnings("unchecked") - public void validateCppDocNotMap(Document doc) throws IOException { + public void validateCppDocNotMap(Document doc) { // in practice to validate v6 serialization assertEquals("id:ns:serializetest::http://test.doc.id/", doc.getId().toString()); assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield")); @@ -694,7 +694,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - @SuppressWarnings("deprecation") public void testGenerateSerializedFile() throws IOException { docMan = setUpCppDocType(); @@ -744,18 +743,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { FileOutputStream fos = new FileOutputStream("src/tests/data/serializejava.dat"); fos.write(buf.array(), 0, size); fos.close(); - - CompressionConfig noncomp = new CompressionConfig(); - CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); - - doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - buf = new GrowableByteBuffer(size, 2.0f); - - doc.serialize(buf); - doc.getDataType().contentStruct().setCompressionConfig(noncomp); - fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat"); - fos.write(buf.array(), 0, buf.position()); - fos.close(); } @Test @@ -801,53 +788,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - @SuppressWarnings("deprecation") - public void testSerializeDeserializeCompressed() { - setUpSertestDocType(); - Document doc = getSertestDocument(); - - CompressionConfig noncomp = new CompressionConfig(); - CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); - - doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - - GrowableByteBuffer data = new GrowableByteBuffer(); - doc.serialize(data); - int size = doc.getSerializedSize(); - doc.getDataType().contentStruct().setCompressionConfig(noncomp); - - assertEquals(size, data.position()); - - data.flip(); - - try { - FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat"); - fos.write(data.array(), 0, data.remaining()); - fos.close(); - } catch (Exception e) { - } - - Document doc2 = docMan.createDocument(data); - - assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid")); - assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date")); - assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from")); - assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to")); - assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject")); - assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body")); - assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount")); - assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments")); - byte[] docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array(); - byte[] doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array(); - assertEquals(docRawBytes.length, doc2RawBytes.length); - for (int i = 0; i < docRawBytes.length; i++) { - assertEquals(docRawBytes[i], doc2RawBytes[i]); - } - assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); - assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); - } - - @Test public void testDeserialize() { setUpSertestDocType(); @@ -937,7 +877,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - public void testCompressionConfigured() { + public void testCompressionConfiguredIsIgnored() { int size_uncompressed; { @@ -966,7 +906,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.serialize(data); int size_compressed = data.position(); - assertTrue(size_compressed + " < " + size_uncompressed, size_compressed < size_uncompressed); + assertEquals(size_compressed, size_uncompressed); } @Test diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java index 379089f1c79..ff66da9a3d7 100644 --- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java @@ -68,7 +68,6 @@ public class VespaDocumentSerializerTestCase { CompressionFixture() { docType = new DocumentType("map_of_structs"); - docType.contentStruct().setCompressionConfig(new CompressionConfig(CompressionType.LZ4)); nestedType = new StructDataType("nested_type"); nestedType.addField(new Field("str", DataType.STRING)); |