Unit test attribute vector compaction.

5 files changed, 108 insertions, 0 deletions

diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index 0ca4fe41d2e..7a5b654e2c0 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -75,6 +75,7 @@ vespa_define_module(
     src/tests/attribute/benchmark
     src/tests/attribute/bitvector
     src/tests/attribute/changevector
+    src/tests/attribute/compaction
     src/tests/attribute/comparator
     src/tests/attribute/document_weight_iterator
     src/tests/attribute/enumeratedsave
diff --git a/searchlib/src/tests/attribute/compaction/CMakeLists.txt b/searchlib/src/tests/attribute/compaction/CMakeLists.txt
new file mode 100644
index 00000000000..994d12761d1
--- /dev/null
+++ b/searchlib/src/tests/attribute/compaction/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_attribute_compaction_test_app TEST
+    SOURCES
+    attribute_compaction_test.cpp
+    DEPENDS
+    searchlib
+)
+vespa_add_test(NAME searchlib_attribute_compaction_test_app COMMAND searchlib_attribute_compaction_test_app)
diff --git a/searchlib/src/tests/attribute/compaction/DESC b/searchlib/src/tests/attribute/compaction/DESC
new file mode 100644
index 00000000000..dfed48a02fb
--- /dev/null
+++ b/searchlib/src/tests/attribute/compaction/DESC
@@ -0,0 +1 @@
+Unit tests for attribute vector compaction.
diff --git a/searchlib/src/tests/attribute/compaction/FILES b/searchlib/src/tests/attribute/compaction/FILES
new file mode 100644
index 00000000000..7dbc2016f5c
--- /dev/null
+++ b/searchlib/src/tests/attribute/compaction/FILES
@@ -0,0 +1 @@
+atttribute_compaction_test.cpp
diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp
new file mode 100644
index 00000000000..7ba290c967d
--- /dev/null
+++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp
@@ -0,0 +1,97 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchlib/attribute/attribute.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/integerbase.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_compaction_test");
+
+using search::IntegerAttribute;
+using search::AttributeVector;
+using search::attribute::Config;
+using search::attribute::BasicType;
+using search::attribute::CollectionType;
+
+using AttributePtr = AttributeVector::SP;
+using AttributeStatus = search::attribute::Status;
+
+namespace
+{
+
+template <typename VectorType>
+bool is(AttributePtr &v)
+{
+    return dynamic_cast<VectorType *>(v.get());
+}
+
+template <typename VectorType>
+VectorType &as(AttributePtr &v)
+{
+    return dynamic_cast<VectorType &>(*v);
+}
+
+void populateAttribute(IntegerAttribute &v, uint32_t docIdLimit)
+{
+    for(uint32_t docId = 0; docId < docIdLimit; ++docId) {
+        uint32_t checkDocId = 0;
+        EXPECT_TRUE(v.addDoc(checkDocId));
+        EXPECT_EQUAL(docId, checkDocId);
+        v.clearDoc(docId);
+        for (size_t vi = 0; vi <= 40; ++vi) {
+            EXPECT_TRUE(v.append(docId, 42, 1) );
+        }
+    }
+    v.commit(true);
+    v.incGeneration();
+}
+
+void populateAttribute(AttributePtr &v, uint32_t docIdLimit)
+{
+    if (is<IntegerAttribute>(v)) {
+        populateAttribute(as<IntegerAttribute>(v), docIdLimit);
+    }
+}
+
+void cleanAttribute(AttributeVector &v, uint32_t docIdLimit)
+{
+    for (uint32_t docId = 0; docId < docIdLimit; ++docId) {
+        v.clearDoc(docId);
+    }
+    v.commit(true);
+    v.incGeneration();
+}
+
+}
+
+class Fixture {
+public:
+    AttributePtr _v;
+
+    Fixture(Config cfg)
+        : _v()
+    { _v = search::AttributeFactory::createAttribute("test", cfg); }
+    ~Fixture() { }
+    void populate(uint32_t docIdLimit) { populateAttribute(_v, docIdLimit); }
+    void clean(uint32_t docIdLimit) { cleanAttribute(*_v, docIdLimit); }
+    AttributeStatus getStatus() { _v->commit(true); return _v->getStatus(); }
+    AttributeStatus getStatus(const vespalib::string &prefix) {
+        AttributeStatus status(getStatus());
+        LOG(info, "status %s: used=%zu, dead=%zu, onHold=%zu",
+            prefix.c_str(), status.getUsed(), status.getDead(), status.getOnHold());
+        return status;
+    }
+};
+
+TEST_F("Test that compaction of integer array attribute reduces memory usage", Fixture({ BasicType::INT64, CollectionType::ARRAY }))
+{
+    f.populate(3000);
+    AttributeStatus beforeStatus = f.getStatus("before");
+    f.clean(2000);
+    AttributeStatus afterStatus = f.getStatus("after");
+    EXPECT_LESS(afterStatus.getUsed(), beforeStatus.getUsed());
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }