Moved databuffer and compresssion to vespalib

23 files changed, 22 insertions, 499 deletions

diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt
index e9694390b4b..ca2ee029c87 100644
--- a/document/CMakeLists.txt
+++ b/document/CMakeLists.txt
@@ -8,10 +8,6 @@ vespa_define_module(
     config_cloudconfig
     vespaeval
 
-    EXTERNAL_DEPENDS
-    lz4
-    zstd
-
     LIBS
     src/vespa/document
     src/vespa/document/annotation
diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp
index 44cff2c80de..c3f88ac3696 100644
--- a/document/src/tests/documenttestcase.cpp
+++ b/document/src/tests/documenttestcase.cpp
@@ -16,6 +16,7 @@
 #include <fcntl.h>
 
 using vespalib::nbostream;
+using vespalib::compression::CompressionConfig;
 
 using namespace document::config_builder;
 
diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp
index 508e6f237cf..1e8fd9ec470 100644
--- a/document/src/tests/repo/documenttyperepo_test.cpp
+++ b/document/src/tests/repo/documenttyperepo_test.cpp
@@ -10,7 +10,6 @@
 #include <vespa/document/datatype/weightedsetdatatype.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
-#include <stdlib.h>
 #include <vespa/vespalib/objects/identifiable.h>
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/testkit/testapp.h>
@@ -25,6 +24,7 @@ using std::vector;
 using vespalib::Identifiable;
 using vespalib::IllegalArgumentException;
 using vespalib::string;
+using vespalib::compression::CompressionConfig;
 
 using namespace document::config_builder;
 using namespace document;
diff --git a/document/src/tests/serialization/.gitignore b/document/src/tests/serialization/.gitignore
index 9f5bc440533..7e94d2757ae 100644
--- a/document/src/tests/serialization/.gitignore
+++ b/document/src/tests/serialization/.gitignore
@@ -2,5 +2,4 @@
 .depend
 Makefile
 document_annotationserializer_test_app
-document_compression_test_app
 document_vespadocumentserializer_test_app
diff --git a/document/src/tests/serialization/CMakeLists.txt b/document/src/tests/serialization/CMakeLists.txt
index 5b23bffad26..e1ce43b12d4 100644
--- a/document/src/tests/serialization/CMakeLists.txt
+++ b/document/src/tests/serialization/CMakeLists.txt
@@ -17,12 +17,3 @@ vespa_add_executable(document_annotationserializer_test_app TEST
     document_documentconfig
 )
 vespa_add_test(NAME document_annotationserializer_test_app COMMAND document_annotationserializer_test_app)
-vespa_add_executable(document_compression_test_app TEST
-    SOURCES
-    compression_test.cpp
-    DEPENDS
-    document
-    AFTER
-    document_documentconfig
-)
-vespa_add_test(NAME document_compression_test_app COMMAND document_compression_test_app)
diff --git a/document/src/tests/serialization/compression_test.cpp b/document/src/tests/serialization/compression_test.cpp
deleted file mode 100644
index 6574b4ac34c..00000000000
--- a/document/src/tests/serialization/compression_test.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/vespalib/testkit/test_kit.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/document/util/compressor.h>
-#include <vespa/vespalib/data/databuffer.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP("compression_test");
-
-using namespace document;
-using namespace document::compression;
-using namespace vespalib;
-
-static vespalib::string _G_compressableText("AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "XYZABCDEFGHIJGJMNOPQRSTUVW"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
-                                            "XYZABCDEFGHIJGJMNOPQRSTUVW");
-
-TEST("requireThatLZ4CompressFine") {
-    CompressionConfig cfg(CompressionConfig::Type::LZ4);
-    ConstBufferRef ref(_G_compressableText.c_str(), _G_compressableText.size());
-    DataBuffer compressed;
-    EXPECT_EQUAL(CompressionConfig::Type::LZ4, compress(cfg, ref, compressed, false));
-    EXPECT_EQUAL(66u, compressed.getDataLen());
-}
-
-TEST("requireThatZStdCompressFine") {
-    CompressionConfig cfg(CompressionConfig::Type::ZSTD);
-    ConstBufferRef ref(_G_compressableText.c_str(), _G_compressableText.size());
-    DataBuffer compressed;
-    EXPECT_EQUAL(CompressionConfig::Type::ZSTD, compress(cfg, ref, compressed, false));
-    EXPECT_EQUAL(64u, compressed.getDataLen());
-}
-
-TEST_MAIN() {
-    TEST_RUN_ALL();
-}
diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp
index d09012a4e4b..9da20e5a84c 100644
--- a/document/src/tests/serialization/vespadocumentserializer_test.cpp
+++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp
@@ -55,6 +55,7 @@ using vespalib::tensor::Tensor;
 using vespalib::tensor::TensorBuilder;
 using vespalib::tensor::TensorCells;
 using vespalib::tensor::TensorDimensions;
+using vespalib::compression::CompressionConfig;
 using namespace document;
 using std::string;
 using std::vector;
diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h
index f6c77f18f47..0f1c58316c9 100644
--- a/document/src/vespa/document/datatype/structdatatype.h
+++ b/document/src/vespa/document/datatype/structdatatype.h
@@ -11,15 +11,16 @@
 
 #include <vespa/document/datatype/structureddatatype.h>
 #include <vespa/vespalib/stllike/hash_map.h>
-#include <vespa/document/util/compressionconfig.h>
+#include <vespa/vespalib/util/compressionconfig.h>
 #include <memory>
 
 namespace document {
 
 class StructDataType final : public StructuredDataType {
 public:
-    typedef std::unique_ptr<StructDataType> UP;
-    typedef std::shared_ptr<StructDataType> SP;
+    using UP = std::unique_ptr<StructDataType>;
+    using SP = std::shared_ptr<StructDataType>;
+    using CompressionConfig = vespalib::compression::CompressionConfig;
 
     StructDataType();
     StructDataType(const vespalib::stringref &name);
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp
index 76216d75f1b..5dfd8eff891 100644
--- a/document/src/vespa/document/fieldvalue/serializablearray.cpp
+++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp
@@ -2,7 +2,7 @@
 #include "serializablearray.h"
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/util/bytebuffer.h>
-#include <vespa/document/util/compressor.h>
+#include <vespa/vespalib/util/compressor.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/data/databuffer.h>
 #include <algorithm>
@@ -181,7 +181,7 @@ SerializableArray::clear(int id)
 void
 SerializableArray::deCompress() // throw (DeserializeException)
 {
-    using document::compression::decompress;
+    using vespalib::compression::decompress;
     // will only do this once
 
     LOG_ASSERT(_compSerData);
@@ -239,7 +239,7 @@ void SerializableArray::assign(EntryMap & entries,
     }
 }
 
-CompressionInfo
+vespalib::compression::CompressionInfo
 SerializableArray::getCompressionInfo() const {
     return CompressionInfo(_uncompressedLength, _compSerData->getRemaining());
 }
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h
index 2d12c1191a6..2f7d65938aa 100644
--- a/document/src/vespa/document/fieldvalue/serializablearray.h
+++ b/document/src/vespa/document/fieldvalue/serializablearray.h
@@ -16,7 +16,7 @@
 
 #pragma once
 
-#include <vespa/document/util/compressionconfig.h>
+#include <vespa/vespalib/util/compressionconfig.h>
 #include <vespa/vespalib/objects/cloneable.h>
 #include <vespa/vespalib/util/buffer.h>
 #include <vespa/vespalib/util/memory.h>
@@ -107,6 +107,8 @@ public:
     using CP = vespalib::CloneablePtr<SerializableArray>;
     using UP = std::unique_ptr<SerializableArray>;
     using ByteBufferUP = std::unique_ptr<ByteBuffer>;
+    using CompressionConfig = vespalib::compression::CompressionConfig;
+    using CompressionInfo = vespalib::compression::CompressionInfo;
 
     SerializableArray();
     virtual ~SerializableArray();
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
index c0ae342fd34..0a05ae60600 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
@@ -21,6 +21,7 @@ using std::vector;
 using vespalib::nbostream;
 using vespalib::nbostream_longlivedbuf;
 using vespalib::make_string;
+using vespalib::compression::CompressionConfig;
 using namespace vespalib::xml;
 
 namespace document {
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h
index 6ab6f71ce56..bb6956cf012 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h
@@ -52,7 +52,9 @@ private:
     mutable bool            _hasChanged;
 
 public:
-    typedef std::unique_ptr<StructFieldValue> UP;
+    using UP = std::unique_ptr<StructFieldValue>;
+    using CompressionConfig = vespalib::compression::CompressionConfig;
+
     StructFieldValue(const DataType &type);
     ~StructFieldValue();
     void swap(StructFieldValue & rhs);
diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp
index 870e88e5036..6bfae246c10 100644
--- a/document/src/vespa/document/repo/documenttyperepo.cpp
+++ b/document/src/vespa/document/repo/documenttyperepo.cpp
@@ -30,6 +30,7 @@ using vespalib::hash_map;
 using vespalib::make_string;
 using vespalib::string;
 using vespalib::stringref;
+using vespalib::compression::CompressionConfig;
 
 namespace document {
 
diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
index bee1c258e4a..2b45e8a298c 100644
--- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
+++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
@@ -38,6 +38,7 @@ using vespalib::asciistream;
 using vespalib::nbostream;
 using vespalib::Memory;
 using vespalib::stringref;
+using vespalib::compression::CompressionConfig;
 
 namespace document {
 
diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
index 2534690b014..6c9e95a9dd6 100644
--- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp
+++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
@@ -27,7 +27,7 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/data/databuffer.h>
 #include <vespa/eval/tensor/serialization/typed_binary_format.h>
-#include <vespa/document/util/compressor.h>
+#include <vespa/vespalib/util/compressor.h>
 
 using std::make_pair;
 using std::pair;
@@ -36,6 +36,7 @@ using vespalib::nbostream;
 using vespalib::stringref;
 using vespalib::string;
 using vespalib::slime::BinaryFormat;
+using vespalib::compression::CompressionConfig;
 
 namespace document {
 
@@ -275,7 +276,7 @@ vespalib::ConstBufferRef
 compressStream(const CompressionConfig &config, nbostream &stream,
                     vespalib::DataBuffer & compressed_data)
 {
-    using compression::compress;
+    using vespalib::compression::compress;
     vespalib::ConstBufferRef buf(stream.c_str(), stream.size());
     if (config.useCompression() && bigEnough(stream.size(), config)) {
         CompressionConfig::Type compressedType = compress(config, vespalib::ConstBufferRef(stream.c_str(), stream.size()), compressed_data, false);
diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt
index 2179b1307d3..8cb148abe25 100644
--- a/document/src/vespa/document/util/CMakeLists.txt
+++ b/document/src/vespa/document/util/CMakeLists.txt
@@ -2,9 +2,6 @@
 vespa_add_library(document_util OBJECT
     SOURCES
     bytebuffer.cpp
-    compressor.cpp
-    lz4compressor.cpp
-    zstdcompressor.cpp
     printable.cpp
     serializable.cpp
     stringutil.cpp
diff --git a/document/src/vespa/document/util/compressionconfig.h b/document/src/vespa/document/util/compressionconfig.h
deleted file mode 100644
index 413bdc1fb3b..00000000000
--- a/document/src/vespa/document/util/compressionconfig.h
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include <cmath>
-#include <cstdint>
-#include <cstddef>
-
-namespace document {
-
-struct CompressionConfig {
-    enum Type {
-        NONE = 0,
-        HISTORIC_1 = 1,
-        HISTORIC_2 = 2,
-        HISTORIC_3 = 3,
-        HISTORIC_4 = 4,
-        UNCOMPRESSABLE = 5,
-        LZ4 = 6,
-        ZSTD = 7
-    };
-
-    CompressionConfig()
-        : type(NONE), compressionLevel(0), threshold(90), minSize(0) {}
-    CompressionConfig(Type t)
-        : type(t), compressionLevel(9), threshold(90), minSize(0) {}
-
-    CompressionConfig(Type t, uint8_t level, uint8_t minRes)
-        : type(t), compressionLevel(level), threshold(minRes), minSize(0) {}
-
-    CompressionConfig(Type t, uint8_t lvl, uint8_t minRes, size_t minSz)
-        : type(t), compressionLevel(lvl), threshold(minRes), minSize(minSz) {}
-
-    bool operator==(const CompressionConfig& o) const {
-        return (type == o.type
-                && compressionLevel == o.compressionLevel
-                && threshold == o.threshold);
-    }
-    bool operator!=(const CompressionConfig& o) const {
-        return !operator==(o);
-    }
-
-    static Type toType(uint32_t val) {
-        switch (val) {
-        case 1: return HISTORIC_1;
-        case 2: return HISTORIC_2;
-        case 3: return HISTORIC_3;
-        case 4: return HISTORIC_4;
-        case 5: return UNCOMPRESSABLE;
-        case 6: return LZ4;
-        case 7: return ZSTD;
-        default: return NONE;
-        }
-    }
-    static bool isCompressed(Type type) {
-        return (type != CompressionConfig::NONE &&
-                type != CompressionConfig::UNCOMPRESSABLE);
-    }
-    bool useCompression() const { return isCompressed(type); }
-
-    Type type;
-    uint8_t compressionLevel;
-    uint8_t threshold;
-    size_t minSize;
-};
-
-class CompressionInfo
-{
-public:
-    CompressionInfo(size_t uncompressedSize)
-        : _uncompressedSize(uncompressedSize), _compressedSize(uncompressedSize) { }
-    CompressionInfo(size_t uncompressedSize, size_t compressedSize)
-        : _uncompressedSize(uncompressedSize), _compressedSize(compressedSize) { }
-    size_t getUncompressedSize() const { return _uncompressedSize; }
-    size_t getCompressedSize()   const { return _compressedSize; }
-    double getCompressionRatio() const { return _uncompressedSize/_compressedSize; }
-private:
-    size_t _uncompressedSize;
-    size_t _compressedSize;
-};
-
-inline CompressionInfo operator + (const CompressionInfo & a, const CompressionInfo & b)
-{
-    return CompressionInfo(a.getUncompressedSize() + b.getUncompressedSize(), a.getCompressedSize() + b.getCompressedSize());
-}
-
-}
-
-
diff --git a/document/src/vespa/document/util/compressor.cpp b/document/src/vespa/document/util/compressor.cpp
deleted file mode 100644
index cd45017dd69..00000000000
--- a/document/src/vespa/document/util/compressor.cpp
+++ /dev/null
@@ -1,141 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "lz4compressor.h"
-#include "zstdcompressor.h"
-#include <vespa/vespalib/util/memory.h>
-#include <vespa/vespalib/util/stringfmt.h>
-#include <vespa/vespalib/data/databuffer.h>
-
-using vespalib::alloc::Alloc;
-using vespalib::ConstBufferRef;
-using vespalib::DataBuffer;
-using vespalib::make_string;
-
-namespace document::compression {
-
-CompressionConfig::Type
-compress(ICompressor & compressor, const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest)
-{
-    CompressionConfig::Type type(CompressionConfig::NONE);
-    dest.ensureFree(compressor.adjustProcessLen(0, org.size()));
-    size_t compressedSize(dest.getFreeLen());
-    if (compressor.process(compression, org.c_str(), org.size(), dest.getFree(), compressedSize)) {
-        if (compressedSize < ((org.size() * compression.threshold)/100)) {
-            dest.moveFreeToData(compressedSize);
-            type = compression.type;
-        }
-    }
-    return type;
-}
-
-CompressionConfig::Type
-docompress(const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest)
-{
-    CompressionConfig::Type type(CompressionConfig::NONE);
-    switch (compression.type) {
-    case CompressionConfig::LZ4:
-        {
-            LZ4Compressor lz4;
-            type = compress(lz4, compression, org, dest);
-        }
-        break;
-    case CompressionConfig::ZSTD:
-        {
-            ZStdCompressor zstd;
-            type = compress(zstd, compression, org, dest);
-        }
-        break;
-    case CompressionConfig::NONE:
-    default:
-        break;
-    }
-    return type;
-}
-
-CompressionConfig::Type
-compress(const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap)
-{
-    CompressionConfig::Type type(CompressionConfig::NONE);
-    if (org.size() >= compression.minSize) {
-        type = docompress(compression, org, dest);
-    }
-    if (type == CompressionConfig::NONE) {
-        if (allowSwap) {
-            DataBuffer tmp(const_cast<char *>(org.c_str()), org.size());
-            tmp.moveFreeToData(org.size());
-            dest.swap(tmp);
-        } else {
-            dest.writeBytes(org.c_str(), org.size());
-        }
-    }
-    return type;
-}
-
-
-void
-decompress(ICompressor & decompressor, size_t uncompressedLen, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap)
-{
-    dest.ensureFree(uncompressedLen);
-    size_t realUncompressedLen(dest.getFreeLen());
-    if ( ! decompressor.unprocess(org.c_str(), org.size(), dest.getFree(), realUncompressedLen) ) {
-        if ( uncompressedLen < realUncompressedLen) {
-            if (allowSwap) {
-                DataBuffer tmp(const_cast<char *>(org.c_str()), org.size());
-                tmp.moveFreeToData(org.size());
-                dest.swap(tmp);
-            } else {
-                dest.writeBytes(org.c_str(), org.size());
-            }
-        } else {
-            throw std::runtime_error(make_string("unprocess failed had %" PRIu64 ", wanted %" PRId64 ", got %" PRIu64,
-                                                 org.size(), uncompressedLen, realUncompressedLen));
-        }
-    } else {
-        dest.moveFreeToData(realUncompressedLen);
-    }
-}
-
-void
-decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap)
-{
-    switch (type) {
-    case CompressionConfig::LZ4:
-        {
-            LZ4Compressor lz4;
-            decompress(lz4, uncompressedLen, org, dest, allowSwap);
-        }
-        break;
-        case CompressionConfig::ZSTD:
-        {
-            ZStdCompressor zstd;
-            decompress(zstd, uncompressedLen, org, dest, allowSwap);
-        }
-        break;
-    case CompressionConfig::NONE:
-    case CompressionConfig::UNCOMPRESSABLE:
-        if (allowSwap) {
-            DataBuffer tmp(const_cast<char *>(org.c_str()), org.size());
-            tmp.moveFreeToData(org.size());
-            dest.swap(tmp);
-        } else {
-            dest.writeBytes(org.c_str(), org.size());
-        }
-        break;
-    default:
-        throw std::runtime_error(make_string("Unable to handle decompression of type '%d'", type));
-        break;
-    }
-}
-
-size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t payloadSize) {
-    if (type == CompressionConfig::LZ4) {
-        document::LZ4Compressor lz4;
-        return lz4.adjustProcessLen(0, payloadSize);
-    } else if (type == CompressionConfig::ZSTD) {
-        document::ZStdCompressor zstd;
-        return zstd.adjustProcessLen(0, payloadSize);
-    }
-    return payloadSize;
-}
-
-}
diff --git a/document/src/vespa/document/util/compressor.h b/document/src/vespa/document/util/compressor.h
deleted file mode 100644
index a8d4803e038..00000000000
--- a/document/src/vespa/document/util/compressor.h
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "compressionconfig.h"
-#include <vespa/vespalib/util/buffer.h>
-
-namespace vespalib { class DataBuffer; }
-
-namespace document {
-
-class ICompressor
-{
-public:
-    virtual ~ICompressor() { }
-    virtual bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) = 0;
-    virtual bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) = 0;
-    virtual size_t adjustProcessLen(uint16_t options, size_t len)   const = 0;
-};
-
-namespace compression {
-
-/**
- * Will try to compress a buffer according to the config. If the criteria can not
- * be met it will return NONE and dest will get the input buffer.
- * @param compression is config for how to compress and what criteria to meet.
- * @param org is the original input buffer.
- * @param dest is the destination buffer. The compressed data will be appended unless allowSwap is true
- *             and it is not compressable. Then it will be swapped in.
- * @param allowSwap will tell it the data must be appended or if it can be swapped in if it is uncompressable or config is NONE.
- */
-CompressionConfig::Type compress(const CompressionConfig & compression, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap);
-
-/**
- * Will try to decompress a buffer according to the config.
- * be met it will return NONE and dest will get the input buffer.
- * @param compression is the compression type used for the buffer.
- * @param uncompressedLen is the length of the uncompressed data.
- * @param org is the original input buffer.
- * @param dest is the destination buffer. The decompressed data will be
- *             appended unless allowSwap is true and compression is NONE.
- *             Then it will be swapped in.
- * @param allowSwap will tell it the data must be appended or if it can be swapped in if compression type is NONE.
- */
-void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap);
-
-
-size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t uncompressedSize);
-
-}
-
-}
diff --git a/document/src/vespa/document/util/lz4compressor.cpp b/document/src/vespa/document/util/lz4compressor.cpp
deleted file mode 100644
index 04d68f394a1..00000000000
--- a/document/src/vespa/document/util/lz4compressor.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "lz4compressor.h"
-#include <vespa/vespalib/util/alloc.h>
-#include <lz4.h>
-#include <lz4hc.h>
-#include <cassert>
-
-using vespalib::alloc::Alloc;
-
-namespace document {
-
-size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len)   const { return LZ4_compressBound(len); }
-
-bool
-LZ4Compressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
-{
-    const char * input(static_cast<const char *>(inputV));
-    char * output(static_cast<char *>(outputV));
-    int sz(-1);
-    int maxOutputLen = LZ4_compressBound(inputLen);
-    if (config.compressionLevel > 6) {
-        Alloc state = Alloc::alloc(LZ4_sizeofStateHC());
-        sz = LZ4_compress_HC_extStateHC(state.get(), input, output, inputLen, maxOutputLen, config.compressionLevel);
-    } else {
-        Alloc state = Alloc::alloc(LZ4_sizeofState());
-        sz = LZ4_compress_fast_extState(state.get(), input, output, inputLen, maxOutputLen, 1);
-    }
-    assert(sz != 0);
-    outputLenV = sz;
-    return (sz != 0);
-    
-}
-
-bool
-LZ4Compressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
-{
-    const char * input(static_cast<const char *>(inputV));
-    char * output(static_cast<char *>(outputV));
-    int sz = LZ4_decompress_safe(input, output, inputLen, outputLenV);
-    assert(sz > 0);
-    outputLenV = sz;
-    return (sz > 0);
-}
-
-}
diff --git a/document/src/vespa/document/util/lz4compressor.h b/document/src/vespa/document/util/lz4compressor.h
deleted file mode 100644
index 6dd8f8fd1cb..00000000000
--- a/document/src/vespa/document/util/lz4compressor.h
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "compressor.h"
-
-namespace document {
-
-class LZ4Compressor : public ICompressor
-{
-public:
-    bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override;
-    bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override;
-    size_t adjustProcessLen(uint16_t options, size_t len)   const override;
-};
-
-}
-
diff --git a/document/src/vespa/document/util/zstdcompressor.cpp b/document/src/vespa/document/util/zstdcompressor.cpp
deleted file mode 100644
index 74a17212a2e..00000000000
--- a/document/src/vespa/document/util/zstdcompressor.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "zstdcompressor.h"
-#include <vespa/vespalib/util/alloc.h>
-#include <vespa/vespalib/util/sync.h>
-#include <zstd.h>
-#include <vector>
-#include <cassert>
-
-using vespalib::alloc::Alloc;
-
-namespace document {
-
-namespace {
-
-class CompressContext {
-public:
-    CompressContext() : _ctx(ZSTD_createCCtx()) {}
-    ~CompressContext() { ZSTD_freeCCtx(_ctx); }
-    ZSTD_CCtx * get() { return _ctx; }
-private:
-    ZSTD_CCtx * _ctx;
-};
-class DecompressContext {
-public:
-    DecompressContext() : _ctx(ZSTD_createDCtx()) {}
-    ~DecompressContext() { ZSTD_freeDCtx(_ctx); }
-    ZSTD_DCtx * get() { return _ctx; }
-private:
-    ZSTD_DCtx * _ctx;
-};
-
-thread_local std::unique_ptr<CompressContext>  _tlCompressState;
-thread_local std::unique_ptr<DecompressContext> _tlDecompressState;
-
-}
-
-size_t ZStdCompressor::adjustProcessLen(uint16_t, size_t len)   const { return ZSTD_compressBound(len); }
-
-bool
-ZStdCompressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
-{
-    size_t maxOutputLen = ZSTD_compressBound(inputLen);
-    if ( ! _tlCompressState) {
-        _tlCompressState = std::make_unique<CompressContext>();
-    }
-    size_t sz = ZSTD_compressCCtx(_tlCompressState->get(), outputV, maxOutputLen, inputV, inputLen, config.compressionLevel);
-    assert( ! ZSTD_isError(sz) );
-    outputLenV = sz;
-    return ! ZSTD_isError(sz);
-}
-
-bool
-ZStdCompressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
-{
-    if ( ! _tlDecompressState) {
-        _tlDecompressState = std::make_unique<DecompressContext>();
-    }
-    size_t sz = ZSTD_decompressDCtx(_tlDecompressState->get(), outputV, outputLenV, inputV, inputLen);
-    assert( ! ZSTD_isError(sz) );
-    outputLenV = sz;
-    return ! ZSTD_isError(sz);
-}
-
-}
diff --git a/document/src/vespa/document/util/zstdcompressor.h b/document/src/vespa/document/util/zstdcompressor.h
deleted file mode 100644
index ba12ed2594c..00000000000
--- a/document/src/vespa/document/util/zstdcompressor.h
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "compressor.h"
-
-namespace document {
-
-class ZStdCompressor : public ICompressor
-{
-public:
-    bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override;
-    bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override;
-    size_t adjustProcessLen(uint16_t options, size_t len)   const override;
-};
-
-}
-