Unit test compaction triggered by address space usage.

1 files changed, 109 insertions, 15 deletions

diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp
index 7ba290c967d..3a4446b4cb2 100644
--- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp
+++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/attribute/attributevector.hpp>
 #include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/address_space_usage.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("attribute_compaction_test");
@@ -14,6 +15,7 @@ using search::AttributeVector;
 using search::attribute::Config;
 using search::attribute::BasicType;
 using search::attribute::CollectionType;
+using search::AddressSpace;
 
 using AttributePtr = AttributeVector::SP;
 using AttributeStatus = search::attribute::Status;
@@ -21,6 +23,20 @@ using AttributeStatus = search::attribute::Status;
 namespace
 {
 
+struct DocIdRange {
+    uint32_t docIdStart;
+    uint32_t docIdLimit;
+    DocIdRange(uint32_t docIdStart_, uint32_t docIdLimit_)
+        : docIdStart(docIdStart_),
+          docIdLimit(docIdLimit_)
+    {
+    }
+    uint32_t begin() { return docIdStart; }
+    uint32_t end() { return docIdLimit; }
+    uint32_t size() { return end() - begin(); }
+};
+
+
 template <typename VectorType>
 bool is(AttributePtr &v)
 {
@@ -33,37 +49,82 @@ VectorType &as(AttributePtr &v)
     return dynamic_cast<VectorType &>(*v);
 }
 
-void populateAttribute(IntegerAttribute &v, uint32_t docIdLimit)
+void cleanAttribute(AttributeVector &v, DocIdRange range)
 {
-    for(uint32_t docId = 0; docId < docIdLimit; ++docId) {
-        uint32_t checkDocId = 0;
-        EXPECT_TRUE(v.addDoc(checkDocId));
-        EXPECT_EQUAL(docId, checkDocId);
+    for (uint32_t docId = range.begin(); docId < range.end(); ++docId) {
         v.clearDoc(docId);
-        for (size_t vi = 0; vi <= 40; ++vi) {
+    }
+    v.commit(true);
+    v.incGeneration();
+}
+
+DocIdRange addAttributeDocs(AttributePtr &v, uint32_t numDocs)
+{
+    uint32_t startDoc = 0;
+    uint32_t lastDoc = 0;
+    EXPECT_TRUE(v->addDocs(startDoc, lastDoc, numDocs));
+    EXPECT_EQUAL(startDoc + numDocs - 1, lastDoc);
+    DocIdRange range(startDoc, startDoc + numDocs);
+    cleanAttribute(*v, range);
+    return range;
+}
+
+void populateAttribute(IntegerAttribute &v, DocIdRange range, uint32_t values)
+{
+    for(uint32_t docId = range.begin(); docId < range.end(); ++docId) {
+        v.clearDoc(docId);
+        for (uint32_t vi = 0; vi <= values; ++vi) {
             EXPECT_TRUE(v.append(docId, 42, 1) );
         }
+        if ((docId % 100) == 0) {
+            v.commit();
+        }
     }
     v.commit(true);
     v.incGeneration();
 }
 
-void populateAttribute(AttributePtr &v, uint32_t docIdLimit)
+void populateAttribute(AttributePtr &v, DocIdRange range, uint32_t values)
 {
     if (is<IntegerAttribute>(v)) {
-        populateAttribute(as<IntegerAttribute>(v), docIdLimit);
+        populateAttribute(as<IntegerAttribute>(v), range, values);
     }
 }
 
-void cleanAttribute(AttributeVector &v, uint32_t docIdLimit)
+void hammerAttribute(IntegerAttribute &v, DocIdRange range, uint32_t count)
 {
-    for (uint32_t docId = 0; docId < docIdLimit; ++docId) {
-        v.clearDoc(docId);
+    uint32_t work = 0;
+    for (uint32_t i = 0; i < count; ++i) {
+        for (uint32_t docId = range.begin(); docId < range.end(); ++docId) {
+            v.clearDoc(docId);
+            EXPECT_TRUE(v.append(docId, 42, 1));
+        }
+        work += range.size();
+        if (work >= 100000) {
+            v.commit(true);
+            work = 0;
+        } else {
+            v.commit();
+        }
     }
     v.commit(true);
     v.incGeneration();
 }
 
+void hammerAttribute(AttributePtr &v, DocIdRange range, uint32_t count)
+{
+    if (is<IntegerAttribute>(v)) {
+        hammerAttribute(as<IntegerAttribute>(v), range, count);
+    }
+}
+
+Config compactAddressSpaceAttributeConfig(bool enableAddressSpaceCompact)
+{
+    Config cfg(BasicType::INT8, CollectionType::ARRAY);
+    cfg.setCompactionStrategy({ 1.0, (enableAddressSpaceCompact ? 0.2 : 1.0) });
+    return cfg;
+}
+
 }
 
 class Fixture {
@@ -74,8 +135,10 @@ public:
         : _v()
     { _v = search::AttributeFactory::createAttribute("test", cfg); }
     ~Fixture() { }
-    void populate(uint32_t docIdLimit) { populateAttribute(_v, docIdLimit); }
-    void clean(uint32_t docIdLimit) { cleanAttribute(*_v, docIdLimit); }
+    DocIdRange addDocs(uint32_t numDocs) { return addAttributeDocs(_v, numDocs); }
+    void populate(DocIdRange range, uint32_t values) { populateAttribute(_v, range, values); }
+    void hammer(DocIdRange range, uint32_t count) { hammerAttribute(_v, range, count); }
+    void clean(DocIdRange range) { cleanAttribute(*_v, range); }
     AttributeStatus getStatus() { _v->commit(true); return _v->getStatus(); }
     AttributeStatus getStatus(const vespalib::string &prefix) {
         AttributeStatus status(getStatus());
@@ -83,15 +146,46 @@ public:
             prefix.c_str(), status.getUsed(), status.getDead(), status.getOnHold());
         return status;
     }
+    const Config &getConfig() const { return _v->getConfig(); }
+    AddressSpace getMultiValueAddressSpaceUsage() const {return _v->getAddressSpaceUsage().multiValueUsage(); }
+    AddressSpace getMultiValueAddressSpaceUsage(const vespalib::string &prefix) {
+        AddressSpace usage(getMultiValueAddressSpaceUsage());
+        LOG(info, "address space usage %s: used=%zu, dead=%zu, limit=%zu, usage=%12.8f",
+            prefix.c_str(), usage.used(), usage.dead(), usage.limit(), usage.usage());
+        return usage;
+    }
 };
 
 TEST_F("Test that compaction of integer array attribute reduces memory usage", Fixture({ BasicType::INT64, CollectionType::ARRAY }))
 {
-    f.populate(3000);
+    DocIdRange range1 = f.addDocs(2000);
+    DocIdRange range2 = f.addDocs(1000);
+    f.populate(range1, 40);
+    f.populate(range2, 40);
     AttributeStatus beforeStatus = f.getStatus("before");
-    f.clean(2000);
+    f.clean(range1);
     AttributeStatus afterStatus = f.getStatus("after");
     EXPECT_LESS(afterStatus.getUsed(), beforeStatus.getUsed());
 }
 
+TEST_F("Test that no compaction of int8 array attribute increases address space usage", Fixture(compactAddressSpaceAttributeConfig(false)))
+{
+    DocIdRange range1 = f.addDocs(1000);
+    DocIdRange range2 = f.addDocs(1000);
+    f.populate(range1, 1000);
+    f.hammer(range2, 101);
+    AddressSpace afterSpace = f.getMultiValueAddressSpaceUsage("after");
+    EXPECT_EQUAL(100001, afterSpace.dead());
+}
+
+TEST_F("Test that compaction of int8 array attribute limits address space usage", Fixture(compactAddressSpaceAttributeConfig(true)))
+{
+    DocIdRange range1 = f.addDocs(1000);
+    DocIdRange range2 = f.addDocs(1000);
+    f.populate(range1, 1000);
+    f.hammer(range2, 101);
+    AddressSpace afterSpace = f.getMultiValueAddressSpaceUsage("after");
+    EXPECT_GREATER(65536, afterSpace.dead());
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }