Publish

39 files changed, 2420 insertions, 0 deletions

diff --git a/searchcore/src/tests/proton/attribute/.gitignore b/searchcore/src/tests/proton/attribute/.gitignore
new file mode 100644
index 00000000000..794f5f454f8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/.gitignore
@@ -0,0 +1,9 @@
+.depend
+Makefile
+*_test
+test
+test_output
+flush
+
+searchcore_attribute_test_app
+searchcore_attributeflush_test_app
diff --git a/searchcore/src/tests/proton/attribute/CMakeLists.txt b/searchcore/src/tests/proton/attribute/CMakeLists.txt
new file mode 100644
index 00000000000..1439c2b2646
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/CMakeLists.txt
@@ -0,0 +1,21 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_test_app
+    SOURCES
+    attribute_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_attribute
+    searchcore_flushengine
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attribute_test_app COMMAND sh attribute_test.sh)
+vespa_add_executable(searchcore_attributeflush_test_app
+    SOURCES
+    attributeflush_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_attribute
+    searchcore_flushengine
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attributeflush_test_app COMMAND sh attributeflush_test.sh)
diff --git a/searchcore/src/tests/proton/attribute/DESC b/searchcore/src/tests/proton/attribute/DESC
new file mode 100644
index 00000000000..bd71a808c51
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/DESC
@@ -0,0 +1 @@
+attribute test. Take a look at attribute.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/FILES b/searchcore/src/tests/proton/attribute/FILES
new file mode 100644
index 00000000000..84bc710d58b
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/FILES
@@ -0,0 +1 @@
+attribute.cpp
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore b/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore
new file mode 100644
index 00000000000..3e77da66466
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore
@@ -0,0 +1 @@
+searchcore_attribute_manager_test_app
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt
new file mode 100644
index 00000000000..7e8ab14a13b
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt
@@ -0,0 +1,14 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_manager_test_app
+    SOURCES
+    attribute_manager_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_initializer
+    searchcore_flushengine
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attribute_manager_test_app COMMAND searchcore_attribute_manager_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/DESC b/searchcore/src/tests/proton/attribute/attribute_manager/DESC
new file mode 100644
index 00000000000..f1cdc01fd47
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/DESC
@@ -0,0 +1 @@
+attribute manager test. Take a look at attribute_manager_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/FILES b/searchcore/src/tests/proton/attribute/attribute_manager/FILES
new file mode 100644
index 00000000000..8e4fbdcb888
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/FILES
@@ -0,0 +1 @@
+attribute_manager_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
new file mode 100644
index 00000000000..34c67da4ac8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -0,0 +1,686 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_manager_test");
+
+#include <vespa/fastos/file.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h>
+#include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_functor.h>
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+#include <vespa/searchcore/proton/initializer/task_runner.h>
+#include <vespa/searchcore/proton/test/attribute_utils.h>
+#include <vespa/searchcore/proton/test/attribute_vectors.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/util/filekit.h>
+
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/predicate_attribute.h>
+#include <vespa/searchlib/predicate/predicate_index.h>
+#include <vespa/searchlib/predicate/predicate_tree_annotator.h>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/config-attributes.h>
+
+namespace vespa { namespace config { namespace search {}}}
+
+using std::string;
+using namespace vespa::config::search;
+using namespace config;
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+using proton::initializer::InitializerTask;
+using proton::test::AttributeUtils;
+using proton::test::Int32Attribute;
+using search::TuneFileAttributes;
+using search::index::DummyFileHeaderContext;
+using search::ForegroundTaskExecutor;
+using search::predicate::PredicateIndex;
+using search::predicate::PredicateTreeAnnotations;
+using vespa::config::search::AttributesConfig;
+using vespa::config::search::AttributesConfigBuilder;
+
+typedef search::attribute::Config AVConfig;
+typedef proton::AttributeCollectionSpec::Attribute AttrSpec;
+typedef proton::AttributeCollectionSpec::AttributeList AttrSpecList;
+typedef proton::AttributeCollectionSpec AttrMgrSpec;
+
+namespace {
+
+const uint64_t createSerialNum = 42u;
+
+class MyAttributeFunctor : public proton::IAttributeFunctor
+{
+    std::vector<vespalib::string> _names;
+
+public:
+    virtual void
+    operator()(const search::AttributeVector &attributeVector) override {
+        _names.push_back(attributeVector.getName());
+    }
+
+    std::string getSortedNames() {
+        std::ostringstream os;
+        std::sort(_names.begin(), _names.end());
+        for (const vespalib::string &name : _names) {
+            if (!os.str().empty())
+                os << ",";
+            os << name;
+        }
+        return os.str();
+    }
+};
+
+}
+
+const string test_dir = "test_output";
+const AVConfig INT32_SINGLE = AttributeUtils::getInt32Config();
+const AVConfig INT32_ARRAY = AttributeUtils::getInt32ArrayConfig();
+
+void
+fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken)
+{
+    test::AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken);
+}
+
+void
+fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken)
+{
+    test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken);
+}
+
+struct BaseFixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext   _fileHeaderContext;
+    ForegroundTaskExecutor   _attributeFieldWriter;
+    BaseFixture()
+        : _dirHandler(test_dir),
+          _fileHeaderContext(),
+          _attributeFieldWriter()
+    {
+    }
+};
+
+
+struct AttributeManagerFixture
+{
+    proton::AttributeManager::SP _msp;
+    proton::AttributeManager &_m;
+    AttributeWriter _aw;
+    AttributeManagerFixture(BaseFixture &bf)
+        : _msp(std::make_shared<proton::AttributeManager>
+               (test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext,
+                bf._attributeFieldWriter)),
+          _m(*_msp),
+          _aw(_msp)
+    {
+    }
+    AttributeVector::SP addAttribute(const vespalib::string &name) {
+        return _m.addAttribute(name, INT32_SINGLE, createSerialNum);
+    }
+};
+
+struct Fixture : public BaseFixture, public AttributeManagerFixture
+{
+    Fixture()
+        : BaseFixture(),
+          AttributeManagerFixture(*static_cast<BaseFixture *>(this))
+    {
+    }
+};
+
+struct SequentialAttributeManager
+{
+    SequentialAttributesInitializer initializer;
+    proton::AttributeManager mgr;
+    SequentialAttributeManager(const AttributeManager &currMgr,
+                               const AttrMgrSpec &newSpec)
+        : initializer(newSpec.getDocIdLimit()),
+          mgr(currMgr, newSpec, initializer)
+    {
+        mgr.addInitializedAttributes(initializer.getInitializedAttributes());
+    }
+};
+
+struct DummyInitializerTask : public InitializerTask
+{
+    virtual void run() override {}
+};
+
+struct ParallelAttributeManager
+{
+    InitializerTask::SP documentMetaStoreInitTask;
+    BucketDBOwner::SP bucketDbOwner;
+    DocumentMetaStore::SP documentMetaStore;
+    search::GrowStrategy attributeGrow;
+    size_t attributeGrowNumDocs;
+    bool fastAccessAttributesOnly;
+    std::shared_ptr<AttributeManager::SP> mgr;
+    AttributeManagerInitializer::SP initializer;
+
+    ParallelAttributeManager(search::SerialNum configSerialNum,
+                             AttributeManager::SP baseAttrMgr,
+                             const AttributesConfig &attrCfg,
+                             uint32_t docIdLimit)
+        : documentMetaStoreInitTask(std::make_shared<DummyInitializerTask>()),
+          bucketDbOwner(std::make_shared<BucketDBOwner>()),
+          documentMetaStore(std::make_shared<DocumentMetaStore>(bucketDbOwner)),
+          attributeGrow(),
+          attributeGrowNumDocs(1),
+          fastAccessAttributesOnly(false),
+          mgr(std::make_shared<AttributeManager::SP>()),
+          initializer(std::make_shared<AttributeManagerInitializer>
+                              (configSerialNum, documentMetaStoreInitTask, documentMetaStore, baseAttrMgr, attrCfg,
+                               attributeGrow, attributeGrowNumDocs, fastAccessAttributesOnly, mgr))
+    {
+        documentMetaStore->setCommittedDocIdLimit(docIdLimit);
+        vespalib::ThreadStackExecutor executor(3, 128 * 1024);
+        initializer::TaskRunner taskRunner(executor);
+        taskRunner.runTask(initializer);
+    }
+};
+
+
+TEST_F("require that attributes are added", Fixture)
+{
+    EXPECT_TRUE(f.addAttribute("a1").get() != NULL);
+    EXPECT_TRUE(f.addAttribute("a2").get() != NULL);
+    EXPECT_EQUAL("a1", (*f._m.getAttribute("a1"))->getName());
+    EXPECT_EQUAL("a1", (*f._m.getAttributeStableEnum("a1"))->getName());
+    EXPECT_EQUAL("a2", (*f._m.getAttribute("a2"))->getName());
+    EXPECT_EQUAL("a2", (*f._m.getAttributeStableEnum("a2"))->getName());
+    EXPECT_TRUE(!f._m.getAttribute("not")->valid());
+}
+
+TEST_F("require that predicate attributes are added", Fixture)
+{
+    EXPECT_TRUE(f._m.addAttribute("p1", AttributeUtils::getPredicateConfig(),
+                                  createSerialNum).get() != NULL);
+    EXPECT_EQUAL("p1", (*f._m.getAttribute("p1"))->getName());
+    EXPECT_EQUAL("p1", (*f._m.getAttributeStableEnum("p1"))->getName());
+}
+
+TEST_F("require that attributes are flushed and loaded", BaseFixture)
+{
+    IndexMetaInfo ia1(test_dir + "/a1");
+    IndexMetaInfo ia2(test_dir + "/a2");
+    IndexMetaInfo ia3(test_dir + "/a3");
+    {
+        AttributeManagerFixture amf(f);
+        proton::AttributeManager &am = amf._m;
+        AttributeVector::SP a1 = amf.addAttribute("a1");
+        EXPECT_EQUAL(1u, a1->getNumDocs());	// Resized to size of attributemanager
+        fillAttribute(a1, 1, 3, 2, 10);
+        EXPECT_EQUAL(3u, a1->getNumDocs());	// Resized to size of attributemanager
+        AttributeVector::SP a2 = amf.addAttribute("a2");
+        EXPECT_EQUAL(1u, a2->getNumDocs());	// Not resized to size of attributemanager
+        fillAttribute(a2, 1, 5, 4, 10);
+        EXPECT_EQUAL(5u, a2->getNumDocs());	// Increased
+        EXPECT_TRUE(ia1.load());
+        EXPECT_TRUE(!ia1.getBestSnapshot().valid);
+        EXPECT_TRUE(ia2.load());
+        EXPECT_TRUE(!ia2.getBestSnapshot().valid);
+        EXPECT_TRUE(!ia3.load());
+        am.flushAll(0);
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia2.load());
+        EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken);
+    }
+    {
+        AttributeManagerFixture amf(f);
+        proton::AttributeManager &am = amf._m;
+        AttributeVector::SP a1 = amf.addAttribute("a1"); // loaded
+
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+        fillAttribute(a1, 1, 2, 20);
+        EXPECT_EQUAL(4u, a1->getNumDocs());
+        AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded
+        EXPECT_EQUAL(5u, a2->getNumDocs());
+        EXPECT_EQUAL(4u, a1->getNumDocs());
+        amf._aw.onReplayDone(5u);
+        EXPECT_EQUAL(5u, a2->getNumDocs());
+        EXPECT_EQUAL(5u, a1->getNumDocs());
+        fillAttribute(a2, 1, 4, 20);
+        EXPECT_EQUAL(6u, a2->getNumDocs());
+        AttributeVector::SP a3 = amf.addAttribute("a3"); // not-loaded
+        EXPECT_EQUAL(1u, a3->getNumDocs());
+        amf._aw.onReplayDone(6);
+        EXPECT_EQUAL(6u, a3->getNumDocs());
+        fillAttribute(a3, 1, 7, 6, 20);
+        EXPECT_EQUAL(7u, a3->getNumDocs());
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia2.load());
+        EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia3.load());
+        EXPECT_TRUE(!ia3.getBestSnapshot().valid);
+        am.flushAll(0);
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia2.load());
+        EXPECT_EQUAL(20u, ia2.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia3.load());
+        EXPECT_EQUAL(20u, ia3.getBestSnapshot().syncToken);
+    }
+    {
+        AttributeManagerFixture amf(f);
+        AttributeVector::SP a1 = amf.addAttribute("a1"); // loaded
+        EXPECT_EQUAL(6u, a1->getNumDocs());
+        AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded
+        EXPECT_EQUAL(6u, a1->getNumDocs());
+        EXPECT_EQUAL(6u, a2->getNumDocs());
+        AttributeVector::SP a3 = amf.addAttribute("a3"); // loaded
+        EXPECT_EQUAL(6u, a1->getNumDocs());
+        EXPECT_EQUAL(6u, a2->getNumDocs());
+        EXPECT_EQUAL(7u, a3->getNumDocs());
+        amf._aw.onReplayDone(7);
+        EXPECT_EQUAL(7u, a1->getNumDocs());
+        EXPECT_EQUAL(7u, a2->getNumDocs());
+        EXPECT_EQUAL(7u, a3->getNumDocs());
+    }
+}
+
+TEST_F("require that predicate attributes are flushed and loaded", BaseFixture)
+{
+    IndexMetaInfo ia1(test_dir + "/a1");
+    {
+        AttributeManagerFixture amf(f);
+        proton::AttributeManager &am = amf._m;
+        AttributeVector::SP a1 =
+            am.addAttribute("a1",
+                            AttributeUtils::getPredicateConfig(),
+                            createSerialNum);
+        EXPECT_EQUAL(1u, a1->getNumDocs());
+
+        PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1);
+        PredicateIndex &index = pa.getIndex();
+        uint32_t doc_id;
+        a1->addDoc(doc_id);
+        index.indexEmptyDocument(doc_id);
+        pa.commit(10, 10);
+
+        EXPECT_EQUAL(2u, a1->getNumDocs());
+
+        EXPECT_TRUE(ia1.load());
+        EXPECT_TRUE(!ia1.getBestSnapshot().valid);
+        am.flushAll(0);
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+    }
+    {
+        AttributeManagerFixture amf(f);
+        proton::AttributeManager &am = amf._m;
+        AttributeVector::SP a1 =
+            am.addAttribute("a1", AttributeUtils::getPredicateConfig(),
+                            createSerialNum); // loaded
+        EXPECT_EQUAL(2u, a1->getNumDocs());
+
+        PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1);
+        PredicateIndex &index = pa.getIndex();
+        uint32_t doc_id;
+        a1->addDoc(doc_id);
+        PredicateTreeAnnotations annotations(3);
+        annotations.interval_map[123] = {{ 0x0001ffff }};
+        index.indexDocument(1, annotations);
+        pa.commit(20, 20);
+
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        am.flushAll(0);
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken);
+    }
+}
+
+TEST_F("require that extra attribute is added", Fixture)
+{
+    AttributeVector::SP extra(new Int32Attribute("extra"));
+    f._m.addExtraAttribute(extra);
+    AttributeGuard::UP exguard(f._m.getAttribute("extra"));
+    EXPECT_TRUE(dynamic_cast<Int32Attribute *>(exguard->operator->()) !=
+                NULL);
+}
+
+TEST_F("require that reconfig can add attributes", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+
+    AttrSpecList newSpec;
+    newSpec.push_back(AttrSpec("a1", INT32_SINGLE));
+    newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+    newSpec.push_back(AttrSpec("a3", INT32_SINGLE));
+
+    SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, f._m.getNumDocs(), 0));
+    std::vector<AttributeGuard> list;
+    sam.mgr.getAttributeList(list);
+    std::sort(list.begin(), list.end(), [](const AttributeGuard & a, const AttributeGuard & b) {
+        return a->getName() < b->getName();
+    });
+    EXPECT_EQUAL(3u, list.size());
+    EXPECT_EQUAL("a1", list[0]->getName());
+    EXPECT_TRUE(list[0].operator->() == a1.get()); // reuse
+    EXPECT_EQUAL("a2", list[1]->getName());
+    EXPECT_EQUAL("a3", list[2]->getName());
+    EXPECT_TRUE(sam.mgr.getAttribute("ex")->operator->() == ex.get()); // reuse
+}
+
+TEST_F("require that reconfig can remove attributes", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    AttributeVector::SP a3 = f.addAttribute("a3");
+
+    AttrSpecList newSpec;
+    newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+
+    SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 1, 0));
+    std::vector<AttributeGuard> list;
+    sam.mgr.getAttributeList(list);
+    EXPECT_EQUAL(1u, list.size());
+    EXPECT_EQUAL("a2", list[0]->getName());
+    EXPECT_TRUE(list[0].operator->() == a2.get()); // reuse
+}
+
+TEST_F("require that new attributes after reconfig are initialized", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    uint32_t docId(0);
+    a1->addDoc(docId);
+    EXPECT_EQUAL(1u, docId);
+    a1->addDoc(docId);
+    EXPECT_EQUAL(2u, docId);
+    EXPECT_EQUAL(3u, a1->getNumDocs());
+
+    AttrSpecList newSpec;
+    newSpec.push_back(AttrSpec("a1", INT32_SINGLE));
+    newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+    newSpec.push_back(AttrSpec("a3", INT32_ARRAY));
+
+    SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 3, 4));
+    AttributeGuard::UP a2ap = sam.mgr.getAttribute("a2");
+    AttributeGuard &a2(*a2ap);
+    EXPECT_EQUAL(3u, a2->getNumDocs());
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(1)));
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(2)));
+    EXPECT_EQUAL(0u, a2->getStatus().getLastSyncToken());
+    AttributeGuard::UP a3ap = sam.mgr.getAttribute("a3");
+    AttributeGuard &a3(*a3ap);
+    AttributeVector::largeint_t buf[1];
+    EXPECT_EQUAL(3u, a3->getNumDocs());
+    EXPECT_EQUAL(0u, a3->get(1, buf, 1));
+    EXPECT_EQUAL(0u, a3->get(2, buf, 1));
+    EXPECT_EQUAL(0u, a3->getStatus().getLastSyncToken());
+}
+
+TEST_F("require that removed attributes can resurrect", BaseFixture)
+{
+    proton::AttributeManager::SP am1(
+            new proton::AttributeManager(test_dir, "test.subdb",
+                                         TuneFileAttributes(),
+                                         f._fileHeaderContext,
+                                         f._attributeFieldWriter));
+    {
+        AttributeVector::SP a1 =
+            am1->addAttribute("a1", INT32_SINGLE,
+                              0);
+        fillAttribute(a1, 2, 10, 15);
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+    }
+
+    AttrSpecList ns1;
+    SequentialAttributeManager am2(*am1, AttrMgrSpec(ns1, 3, 16));
+    am1.reset();
+
+    AttrSpecList ns2;
+    ns2.push_back(AttrSpec("a1", INT32_SINGLE));
+    // 2 new documents added since a1 was removed
+    SequentialAttributeManager am3(am2.mgr, AttrMgrSpec(ns2, 5, 20));
+
+    AttributeGuard::UP ag1ap = am3.mgr.getAttribute("a1");
+    AttributeGuard &ag1(*ag1ap);
+    ASSERT_TRUE(ag1.valid());
+    EXPECT_EQUAL(5u, ag1->getNumDocs());
+    EXPECT_EQUAL(10, ag1->getInt(1));
+    EXPECT_EQUAL(10, ag1->getInt(2));
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ag1->getInt(3)));
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ag1->getInt(4)));
+    EXPECT_EQUAL(16u, ag1->getStatus().getLastSyncToken());
+}
+
+TEST_F("require that extra attribute is not treated as removed", Fixture)
+{
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+    ex->commit(1,1);
+
+    AttrSpecList ns;
+    SequentialAttributeManager am2(f._m, AttrMgrSpec(ns, 2, 1));
+    EXPECT_TRUE(am2.mgr.getAttribute("ex")->operator->() == ex.get()); // reuse
+}
+
+TEST_F("require that history can be wiped", Fixture)
+{
+    f.addAttribute("a1");
+    f.addAttribute("a2");
+    f.addAttribute("a3");
+    f._m.flushAll(10);
+    Schema hs;
+    hs.addAttributeField(Schema::AttributeField("a1", Schema::INT32));
+    hs.addAttributeField(Schema::AttributeField("a3", Schema::INT32));
+    f._m.wipeHistory(hs);
+    FastOS_StatInfo si;
+    EXPECT_TRUE(!FastOS_File::Stat(vespalib::string(test_dir + "/a1").c_str(), &si));
+    EXPECT_TRUE(FastOS_File::Stat(vespalib::string(test_dir + "/a2").c_str(), &si));
+    EXPECT_TRUE(!FastOS_File::Stat(vespalib::string(test_dir + "/a3").c_str(), &si));
+}
+
+TEST_F("require that lid space can be compacted", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+    const int64_t attrValue = 33;
+    fillAttribute(a1, 20, attrValue, 100);
+    fillAttribute(a2, 20, attrValue, 100);
+    fillAttribute(ex, 20, attrValue, 100);
+
+    EXPECT_EQUAL(21u, a1->getNumDocs());
+    EXPECT_EQUAL(21u, a2->getNumDocs());
+    EXPECT_EQUAL(20u, ex->getNumDocs());
+    EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit());
+    EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit());
+    EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit());
+
+    f._aw.compactLidSpace(10, 101);
+
+    EXPECT_EQUAL(21u, a1->getNumDocs());
+    EXPECT_EQUAL(21u, a2->getNumDocs());
+    EXPECT_EQUAL(20u, ex->getNumDocs());
+    EXPECT_EQUAL(10u, a1->getCommittedDocIdLimit());
+    EXPECT_EQUAL(10u, a2->getCommittedDocIdLimit());
+    EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit());
+}
+
+TEST_F("require that lid space compaction op can be ignored", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+    const int64_t attrValue = 33;
+    fillAttribute(a1, 20, attrValue, 200);
+    fillAttribute(a2, 20, attrValue, 100);
+    fillAttribute(ex, 20, attrValue, 100);
+
+    EXPECT_EQUAL(21u, a1->getNumDocs());
+    EXPECT_EQUAL(21u, a2->getNumDocs());
+    EXPECT_EQUAL(20u, ex->getNumDocs());
+    EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit());
+    EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit());
+    EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit());
+
+    f._aw.compactLidSpace(10, 101);
+
+    EXPECT_EQUAL(21u, a1->getNumDocs());
+    EXPECT_EQUAL(21u, a2->getNumDocs());
+    EXPECT_EQUAL(20u, ex->getNumDocs());
+    EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit());
+    EXPECT_EQUAL(10u, a2->getCommittedDocIdLimit());
+    EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit());
+}
+
+TEST_F("require that flushed serial number can be retrieved", Fixture)
+{
+    f.addAttribute("a1");
+    EXPECT_EQUAL(0u, f._m.getFlushedSerialNum("a1"));
+    f._m.flushAll(100);
+    EXPECT_EQUAL(100u, f._m.getFlushedSerialNum("a1"));
+    EXPECT_EQUAL(0u, f._m.getFlushedSerialNum("a2"));
+}
+
+
+TEST_F("require that writable attributes can be retrieved", Fixture)
+{
+    auto a1 = f.addAttribute("a1");
+    auto a2 = f.addAttribute("a2");
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+    auto &vec = f._m.getWritableAttributes();
+    EXPECT_EQUAL(2u, vec.size());
+    EXPECT_EQUAL(a1.get(), vec[0]);
+    EXPECT_EQUAL(a2.get(), vec[1]);
+    EXPECT_EQUAL(a1.get(), f._m.getWritableAttribute("a1"));
+    EXPECT_EQUAL(a2.get(), f._m.getWritableAttribute("a2"));
+    AttributeVector *noAttr = nullptr;
+    EXPECT_EQUAL(noAttr, f._m.getWritableAttribute("a3"));
+    EXPECT_EQUAL(noAttr, f._m.getWritableAttribute("ex"));
+}
+
+
+void
+populateAndFlushAttributes(AttributeManagerFixture &f)
+{
+    const int64_t attrValue = 7;
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    fillAttribute(a1, 1, 10, attrValue, createSerialNum);
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    fillAttribute(a2, 1, 10, attrValue, createSerialNum);
+    AttributeVector::SP a3 = f.addAttribute("a3");
+    fillAttribute(a3, 1, 10, attrValue, createSerialNum);
+    f._m.flushAll(createSerialNum + 3);
+}
+
+void
+validateAttribute(const AttributeVector &attr)
+{
+    ASSERT_EQUAL(10u, attr.getNumDocs());
+    EXPECT_EQUAL(createSerialNum + 3, attr.getStatus().getLastSyncToken());
+    for (uint32_t docId = 1; docId < 10; ++docId) {
+        EXPECT_EQUAL(7, attr.getInt(docId));
+    }
+}
+
+TEST_F("require that attributes can be initialized and loaded in sequence", BaseFixture)
+{
+    {
+        AttributeManagerFixture amf(f);
+        populateAndFlushAttributes(amf);
+    }
+    {
+        AttributeManagerFixture amf(f);
+
+        AttrSpecList newSpec;
+        newSpec.push_back(AttrSpec("a1", INT32_SINGLE));
+        newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+        newSpec.push_back(AttrSpec("a3", INT32_SINGLE));
+
+        SequentialAttributeManager newMgr(amf._m, AttrMgrSpec(newSpec, 10, createSerialNum + 5));
+
+        AttributeGuard::UP a1 = newMgr.mgr.getAttribute("a1");
+        TEST_DO(validateAttribute(a1->get()));
+        AttributeGuard::UP a2 = newMgr.mgr.getAttribute("a2");
+        TEST_DO(validateAttribute(a2->get()));
+        AttributeGuard::UP a3 = newMgr.mgr.getAttribute("a3");
+        TEST_DO(validateAttribute(a3->get()));
+    }
+}
+
+AttributesConfigBuilder::Attribute
+createAttributeConfig(const vespalib::string &name)
+{
+    AttributesConfigBuilder::Attribute result;
+    result.name = name;
+    result.datatype = AttributesConfigBuilder::Attribute::Datatype::INT32;
+    result.collectiontype = AttributesConfigBuilder::Attribute::Collectiontype::SINGLE;
+    return result;
+}
+
+TEST_F("require that attributes can be initialized and loaded in parallel", BaseFixture)
+{
+    {
+        AttributeManagerFixture amf(f);
+        populateAndFlushAttributes(amf);
+    }
+    {
+        AttributeManagerFixture amf(f);
+
+        AttributesConfigBuilder attrCfg;
+        attrCfg.attribute.push_back(createAttributeConfig("a1"));
+        attrCfg.attribute.push_back(createAttributeConfig("a2"));
+        attrCfg.attribute.push_back(createAttributeConfig("a3"));
+
+        ParallelAttributeManager newMgr(createSerialNum + 5, amf._msp, attrCfg, 10);
+
+        AttributeGuard::UP a1 = newMgr.mgr->get()->getAttribute("a1");
+        TEST_DO(validateAttribute(a1->get()));
+        AttributeGuard::UP a2 = newMgr.mgr->get()->getAttribute("a2");
+        TEST_DO(validateAttribute(a2->get()));
+        AttributeGuard::UP a3 = newMgr.mgr->get()->getAttribute("a3");
+        TEST_DO(validateAttribute(a3->get()));
+    }
+}
+
+TEST_F("require that we can call functions on all attributes via functor",
+       Fixture)
+{
+    f.addAttribute("a1");
+    f.addAttribute("a2");
+    f.addAttribute("a3");
+    std::shared_ptr<MyAttributeFunctor> functor =
+        std::make_shared<MyAttributeFunctor>();
+    f._m.asyncForEachAttribute(functor);
+    EXPECT_EQUAL("a1,a2,a3", functor->getSortedNames());
+}
+
+TEST_F("require that we can acquire exclusive read access to attribute", Fixture)
+{
+    f.addAttribute("attr");
+    ExclusiveAttributeReadAccessor::UP attrAccessor = f._m.getExclusiveReadAccessor("attr");
+    ExclusiveAttributeReadAccessor::UP noneAccessor = f._m.getExclusiveReadAccessor("none");
+    EXPECT_TRUE(attrAccessor.get() != nullptr);
+    EXPECT_TRUE(noneAccessor.get() == nullptr);
+}
+
+TEST_MAIN()
+{
+    vespalib::rmdir(test_dir, true);
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore b/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore
new file mode 100644
index 00000000000..2400fd559e6
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore
@@ -0,0 +1 @@
+searchcore_attribute_populator_test_app
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt
new file mode 100644
index 00000000000..064759b88d1
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_populator_test_app
+    SOURCES
+    attribute_populator_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attribute_populator_test_app COMMAND searchcore_attribute_populator_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/DESC b/searchcore/src/tests/proton/attribute/attribute_populator/DESC
new file mode 100644
index 00000000000..5ef9dcb2709
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/DESC
@@ -0,0 +1 @@
+attribute_populator test. Take a look at attribute_populator_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/FILES b/searchcore/src/tests/proton/attribute/attribute_populator/FILES
new file mode 100644
index 00000000000..b6bf0bf8458
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/FILES
@@ -0,0 +1 @@
+attribute_populator_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
new file mode 100644
index 00000000000..36e50249b89
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_populator_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/attribute_populator.h>
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+
+typedef search::attribute::Config AVConfig;
+typedef search::attribute::BasicType AVBasicType;
+
+const vespalib::string TEST_DIR = "testdir";
+const uint64_t CREATE_SERIAL_NUM = 8u;
+
+Schema
+createSchema()
+{
+    Schema schema;
+    schema.addAttributeField(Schema::AttributeField("a1", Schema::DataType::INT32));
+    return schema;
+}
+
+struct DocContext
+{
+    Schema _schema;
+    DocBuilder _builder;
+    DocContext()
+        : _schema(createSchema()),
+          _builder(_schema)
+    {
+    }
+    Document::UP create(uint32_t id, int64_t fieldValue) {
+        vespalib::string docId =
+                vespalib::make_string("id:searchdocument:searchdocument::%u", id);
+        return _builder.startDocument(docId).
+                startAttributeField("a1").addInt(fieldValue).endField().
+                endDocument();
+    }
+};
+
+struct Fixture
+{
+    test::DirectoryHandler _testDir;
+    DummyFileHeaderContext _fileHeader;
+    ForegroundTaskExecutor _attributeFieldWriter;
+    AttributeManager::SP _mgr;
+    AttributePopulator _pop;
+    DocContext _ctx;
+    Fixture()
+        : _testDir(TEST_DIR),
+          _fileHeader(),
+          _attributeFieldWriter(),
+          _mgr(new AttributeManager(TEST_DIR, "test.subdb",
+                  TuneFileAttributes(),
+                                    _fileHeader, _attributeFieldWriter)),
+          _pop(_mgr, 1, "test"),
+          _ctx()
+    {
+        _mgr->addAttribute("a1", AVConfig(AVBasicType::INT32),
+                CREATE_SERIAL_NUM);
+    }
+    AttributeGuard::UP getAttr() {
+        return _mgr->getAttribute("a1");
+    }
+};
+
+TEST_F("require that reprocess with document populates attribute", Fixture)
+{
+    AttributeGuard::UP attr = f.getAttr();
+    EXPECT_EQUAL(1u, attr->get().getNumDocs());
+
+    f._pop.handleExisting(5, *f._ctx.create(0, 33));
+    EXPECT_EQUAL(6u, attr->get().getNumDocs());
+    EXPECT_EQUAL(33, attr->get().getInt(5));
+    EXPECT_EQUAL(1u, attr->get().getStatus().getLastSyncToken());
+
+    f._pop.handleExisting(6, *f._ctx.create(1, 44));
+    EXPECT_EQUAL(7u, attr->get().getNumDocs());
+    EXPECT_EQUAL(44, attr->get().getInt(6));
+    EXPECT_EQUAL(2u, attr->get().getStatus().getLastSyncToken());
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
new file mode 100644
index 00000000000..d5084273c6c
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -0,0 +1,607 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_test");
+
+#include <vespa/fastos/file.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
+#include <vespa/searchcore/proton/test/attribute_utils.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/common/idestructorcallback.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/util/filekit.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+#include <vespa/document/predicate/predicate_slime_builder.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/predicate_attribute.h>
+#include <vespa/searchlib/predicate/predicate_index.h>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
+#include <vespa/searchlib/predicate/predicate_hash.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+#include <vespa/vespalib/tensor/tensor.h>
+#include <vespa/vespalib/tensor/types.h>
+#include <vespa/vespalib/tensor/default_tensor.h>
+#include <vespa/vespalib/tensor/tensor_factory.h>
+#include <vespa/searchlib/attribute/tensorattribute.h>
+
+
+namespace vespa { namespace config { namespace search {}}}
+
+using std::string;
+using namespace vespa::config::search;
+using namespace config;
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+using search::attribute::TensorAttribute;
+using search::TuneFileAttributes;
+using search::index::DummyFileHeaderContext;
+using search::predicate::PredicateIndex;
+using search::predicate::PredicateHash;
+using vespalib::tensor::Tensor;
+using vespalib::tensor::TensorType;
+using vespalib::tensor::TensorCells;
+using vespalib::tensor::TensorDimensions;
+
+typedef search::attribute::Config AVConfig;
+typedef search::attribute::BasicType AVBasicType;
+typedef search::attribute::CollectionType AVCollectionType;
+typedef proton::AttributeCollectionSpec::Attribute AttrSpec;
+typedef proton::AttributeCollectionSpec::AttributeList AttrSpecList;
+typedef proton::AttributeCollectionSpec AttrMgrSpec;
+typedef SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> > Int32AttributeVector;
+
+namespace
+{
+
+const uint64_t createSerialNum = 42u;
+
+}
+
+AVConfig
+unregister(const AVConfig & cfg)
+{
+    AVConfig retval = cfg;
+    return retval;
+}
+
+const string test_dir = "test_output";
+const AVConfig INT32_SINGLE = unregister(AVConfig(AVBasicType::INT32));
+const AVConfig INT32_ARRAY = unregister(AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY));
+
+void
+fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken)
+{
+    test::AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken);
+}
+
+void
+fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken)
+{
+    test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken);
+}
+
+const std::shared_ptr<IDestructorCallback> emptyCallback;
+
+
+struct Fixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext   _fileHeaderContext;
+    ForegroundTaskExecutor   _attributeFieldWriter;
+    proton::AttributeManager::SP _m;
+    AttributeWriter aw;
+
+    Fixture()
+        : _dirHandler(test_dir),
+          _fileHeaderContext(),
+          _attributeFieldWriter(),
+          _m(std::make_shared<proton::AttributeManager>
+             (test_dir, "test.subdb", TuneFileAttributes(),
+              _fileHeaderContext, _attributeFieldWriter)),
+          aw(_m)
+    {
+    }
+    AttributeVector::SP addAttribute(const vespalib::string &name) {
+        return _m->addAttribute(name, AVConfig(AVBasicType::INT32),
+                                createSerialNum);
+    }
+    void put(SerialNum serialNum, const Document &doc, DocumentIdT lid,
+             bool immediateCommit = true) {
+        aw.put(serialNum, doc, lid, immediateCommit, emptyCallback);
+    }
+    void update(SerialNum serialNum, const DocumentUpdate &upd,
+                DocumentIdT lid, bool immediateCommit) {
+        aw.update(serialNum, upd, lid, immediateCommit, emptyCallback);
+    }
+    void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit = true) {
+        aw.remove(serialNum, lid, immediateCommit, emptyCallback);
+    }
+    void commit(SerialNum serialNum) {
+        aw.commit(serialNum, emptyCallback);
+    }
+};
+
+
+TEST_F("require that attribute adapter handles put", Fixture)
+{
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("a1", Schema::INT32, Schema::SINGLE));
+    s.addAttributeField(Schema::AttributeField("a2", Schema::INT32, Schema::ARRAY));
+    s.addAttributeField(Schema::AttributeField("a3", Schema::FLOAT, Schema::SINGLE));
+    s.addAttributeField(Schema::AttributeField("a4", Schema::STRING, Schema::SINGLE));
+
+    DocBuilder idb(s);
+
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 =
+        am.addAttribute("a2",
+                        AVConfig(AVBasicType::INT32,
+                                 AVCollectionType::ARRAY),
+                        createSerialNum);
+    AttributeVector::SP a3 =
+        am.addAttribute("a3", AVConfig(AVBasicType::FLOAT),
+                        createSerialNum);
+    AttributeVector::SP a4 = am.addAttribute("a4",
+                                             AVConfig(AVBasicType::STRING),
+                                             createSerialNum);
+
+    attribute::IntegerContent ibuf;
+    attribute::FloatContent fbuf;
+    attribute::ConstCharContent sbuf;
+    { // empty document should give default values
+        EXPECT_EQUAL(1u, a1->getNumDocs());
+        f.put(1, *idb.startDocument("doc::1").endDocument(), 1);
+        EXPECT_EQUAL(2u, a1->getNumDocs());
+        EXPECT_EQUAL(2u, a2->getNumDocs());
+        EXPECT_EQUAL(2u, a3->getNumDocs());
+        EXPECT_EQUAL(2u, a4->getNumDocs());
+        EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(1u, a2->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(1u, a3->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(1u, a4->getStatus().getLastSyncToken());
+        ibuf.fill(*a1, 1);
+        EXPECT_EQUAL(1u, ibuf.size());
+        EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ibuf[0]));
+        ibuf.fill(*a2, 1);
+        EXPECT_EQUAL(0u, ibuf.size());
+        fbuf.fill(*a3, 1);
+        EXPECT_EQUAL(1u, fbuf.size());
+        EXPECT_TRUE(search::attribute::isUndefined<float>(fbuf[0]));
+        sbuf.fill(*a4, 1);
+        EXPECT_EQUAL(1u, sbuf.size());
+        EXPECT_EQUAL(strcmp("", sbuf[0]), 0);
+    }
+    { // document with single value & multi value attribute
+        Document::UP doc = idb.startDocument("doc::2").
+            startAttributeField("a1").addInt(10).endField().
+            startAttributeField("a2").startElement().addInt(20).endElement().
+                                      startElement().addInt(30).endElement().endField().endDocument();
+        f.put(2, *doc, 2);
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+        EXPECT_EQUAL(3u, a2->getNumDocs());
+        EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(2u, a2->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(2u, a3->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(2u, a4->getStatus().getLastSyncToken());
+        ibuf.fill(*a1, 2);
+        EXPECT_EQUAL(1u, ibuf.size());
+        EXPECT_EQUAL(10u, ibuf[0]);
+        ibuf.fill(*a2, 2);
+        EXPECT_EQUAL(2u, ibuf.size());
+        EXPECT_EQUAL(20u, ibuf[0]);
+        EXPECT_EQUAL(30u, ibuf[1]);
+    }
+    { // replace existing document
+        Document::UP doc = idb.startDocument("doc::2").
+            startAttributeField("a1").addInt(100).endField().
+            startAttributeField("a2").startElement().addInt(200).endElement().
+                                      startElement().addInt(300).endElement().
+                                      startElement().addInt(400).endElement().endField().endDocument();
+        f.put(3, *doc, 2);
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+        EXPECT_EQUAL(3u, a2->getNumDocs());
+        EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(3u, a2->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(3u, a3->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(3u, a4->getStatus().getLastSyncToken());
+        ibuf.fill(*a1, 2);
+        EXPECT_EQUAL(1u, ibuf.size());
+        EXPECT_EQUAL(100u, ibuf[0]);
+        ibuf.fill(*a2, 2);
+        EXPECT_EQUAL(3u, ibuf.size());
+        EXPECT_EQUAL(200u, ibuf[0]);
+        EXPECT_EQUAL(300u, ibuf[1]);
+        EXPECT_EQUAL(400u, ibuf[2]);
+    }
+}
+
+TEST_F("require that attribute adapter handles predicate put", Fixture)
+{
+    Schema s;
+    s.addAttributeField(
+            Schema::AttributeField("a1", Schema::BOOLEANTREE, Schema::SINGLE));
+    DocBuilder idb(s);
+
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             AVConfig(AVBasicType::PREDICATE),
+                                             createSerialNum);
+
+    PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex();
+
+    // empty document should give default values
+    EXPECT_EQUAL(1u, a1->getNumDocs());
+    f.put(1, *idb.startDocument("doc::1").endDocument(), 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+    EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken());
+    EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size());
+
+    // document with single value attribute
+    PredicateSlimeBuilder builder;
+    Document::UP doc =
+        idb.startDocument("doc::2").startAttributeField("a1")
+        .addPredicate(builder.true_predicate().build())
+        .endField().endDocument();
+    f.put(2, *doc, 2);
+    EXPECT_EQUAL(3u, a1->getNumDocs());
+    EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken());
+    EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size());
+
+    auto it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar"));
+    EXPECT_FALSE(it.valid());
+
+    // replace existing document
+    doc = idb.startDocument("doc::2").startAttributeField("a1")
+          .addPredicate(builder.feature("foo").value("bar").build())
+          .endField().endDocument();
+    f.put(3, *doc, 2);
+    EXPECT_EQUAL(3u, a1->getNumDocs());
+    EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+
+    it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar"));
+    EXPECT_TRUE(it.valid());
+}
+
+TEST_F("require that attribute adapter handles remove", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("a1", Schema::INT32, Schema::SINGLE));
+    s.addAttributeField(Schema::AttributeField("a2", Schema::INT32, Schema::SINGLE));
+
+    DocBuilder idb(s);
+
+    fillAttribute(a1, 1, 10, 1);
+    fillAttribute(a2, 1, 20, 1);
+
+    f.remove(2, 0);
+
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a1->getInt(0)));
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(0)));
+
+    f.remove(2, 0); // same sync token as previous
+    try {
+        f.remove(1, 0); // lower sync token than previous
+        EXPECT_TRUE(true);	// update is ignored
+    } catch (vespalib::IllegalStateException & e) {
+        LOG(info, "Got expected exception: '%s'", e.getMessage().c_str());
+        EXPECT_TRUE(true);
+    }
+}
+
+void verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected)
+{
+    attribute::ConstCharContent sbuf;
+    sbuf.fill(v, lid);
+    EXPECT_EQUAL(1u, sbuf.size());
+    EXPECT_EQUAL(expected, sbuf[0]);
+}
+
+TEST_F("require that visibilitydelay is honoured", Fixture)
+{
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             AVConfig(AVBasicType::STRING),
+                                             createSerialNum);
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("a1", Schema::STRING, Schema::SINGLE));
+    DocBuilder idb(s);
+    EXPECT_EQUAL(1u, a1->getNumDocs());
+    EXPECT_EQUAL(0u, a1->getStatus().getLastSyncToken());
+    Document::UP doc = idb.startDocument("doc::1")
+                              .startAttributeField("a1").addStr("10").endField()
+                          .endDocument();
+    f.put(3, *doc, 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+    EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+    AttributeWriter awDelayed(f._m);
+    awDelayed.put(4, *doc, 2, false, emptyCallback);
+    EXPECT_EQUAL(3u, a1->getNumDocs());
+    EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+    awDelayed.put(5, *doc, 4, false, emptyCallback);
+    EXPECT_EQUAL(5u, a1->getNumDocs());
+    EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+    awDelayed.commit(6, emptyCallback);
+    EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken());
+
+    AttributeWriter awDelayedShort(f._m);
+    awDelayedShort.put(7, *doc, 2, false, emptyCallback);
+    EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken());
+    awDelayedShort.put(8, *doc, 2, false, emptyCallback);
+    awDelayedShort.commit(8, emptyCallback);
+    EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken());
+
+    verifyAttributeContent(*a1, 2, "10");
+    awDelayed.put(9, *idb.startDocument("doc::1").startAttributeField("a1").addStr("11").endField().endDocument(),
+            2, false, emptyCallback);
+    awDelayed.put(10, *idb.startDocument("doc::1").startAttributeField("a1").addStr("20").endField().endDocument(),
+            2, false, emptyCallback);
+    awDelayed.put(11, *idb.startDocument("doc::1").startAttributeField("a1").addStr("30").endField().endDocument(),
+            2, false, emptyCallback);
+    EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken());
+    verifyAttributeContent(*a1, 2, "10");
+    awDelayed.commit(12, emptyCallback);
+    EXPECT_EQUAL(12u, a1->getStatus().getLastSyncToken());
+    verifyAttributeContent(*a1, 2, "30");
+    
+}
+
+TEST_F("require that attribute adapter handles predicate remove", Fixture)
+{
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             AVConfig(AVBasicType::PREDICATE),
+                                             createSerialNum);
+    Schema s;
+    s.addAttributeField(
+            Schema::AttributeField("a1", Schema::BOOLEANTREE, Schema::SINGLE));
+
+    DocBuilder idb(s);
+    PredicateSlimeBuilder builder;
+    Document::UP doc =
+        idb.startDocument("doc::1").startAttributeField("a1")
+        .addPredicate(builder.true_predicate().build())
+        .endField().endDocument();
+    f.put(1, *doc, 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+
+    PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex();
+    EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size());
+    f.remove(2, 1);
+    EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size());
+}
+
+TEST_F("require that attribute adapter handles update", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+
+    fillAttribute(a1, 1, 10, 1);
+    fillAttribute(a2, 1, 20, 1);
+
+    Schema schema;
+    schema.addAttributeField(Schema::AttributeField(
+                    "a1", Schema::INT32,
+                    Schema::SINGLE));
+    schema.addAttributeField(Schema::AttributeField(
+                    "a2", Schema::INT32,
+                    Schema::SINGLE));
+    DocBuilder idb(schema);
+    const document::DocumentType &dt(idb.getDocumentType());
+    DocumentUpdate upd(dt, DocumentId("doc::1"));
+    upd.addUpdate(FieldUpdate(upd.getType().getField("a1"))
+                  .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 5)));
+    upd.addUpdate(FieldUpdate(upd.getType().getField("a2"))
+                  .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)));
+
+    bool immediateCommit = true;
+    f.update(2, upd, 1, immediateCommit);
+
+    attribute::IntegerContent ibuf;
+    ibuf.fill(*a1, 1);
+    EXPECT_EQUAL(1u, ibuf.size());
+    EXPECT_EQUAL(15u, ibuf[0]);
+    ibuf.fill(*a2, 1);
+    EXPECT_EQUAL(1u, ibuf.size());
+    EXPECT_EQUAL(30u, ibuf[0]);
+
+    f.update(2, upd, 1, immediateCommit); // same sync token as previous
+    try {
+        f.update(1, upd, 1, immediateCommit); // lower sync token than previous
+        EXPECT_TRUE(true);	// update is ignored
+    } catch (vespalib::IllegalStateException & e) {
+        LOG(info, "Got expected exception: '%s'", e.getMessage().c_str());
+        EXPECT_TRUE(true);
+    }
+}
+
+TEST_F("require that attribute adapter handles predicate update", Fixture)
+{
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             AVConfig(AVBasicType::PREDICATE),
+                                             createSerialNum);
+    Schema schema;
+    schema.addAttributeField(Schema::AttributeField(
+                    "a1", Schema::BOOLEANTREE,
+                    Schema::SINGLE));
+
+    DocBuilder idb(schema);
+    PredicateSlimeBuilder builder;
+    Document::UP doc =
+        idb.startDocument("doc::1").startAttributeField("a1")
+        .addPredicate(builder.true_predicate().build())
+        .endField().endDocument();
+    f.put(1, *doc, 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+
+    const document::DocumentType &dt(idb.getDocumentType());
+    DocumentUpdate upd(dt, DocumentId("doc::1"));
+    PredicateFieldValue new_value(builder.feature("foo").value("bar").build());
+    upd.addUpdate(FieldUpdate(upd.getType().getField("a1"))
+                  .addUpdate(AssignValueUpdate(new_value)));
+
+    PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex();
+    EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size());
+    EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid());
+    bool immediateCommit = true;
+    f.update(2, upd, 1, immediateCommit);
+    EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size());
+    EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid());
+}
+
+struct AttributeCollectionSpecFixture
+{
+    AttributesConfigBuilder _builder;
+    AttributeCollectionSpecFactory _factory;
+    AttributeCollectionSpecFixture(bool fastAccessOnly)
+        : _builder(),
+          _factory(search::GrowStrategy(), 100, fastAccessOnly)
+    {
+        addAttribute("a1", false);
+        addAttribute("a2", true);
+    }
+    void addAttribute(const vespalib::string &name, bool fastAccess) {
+        AttributesConfigBuilder::Attribute attr;
+        attr.name = name;
+        attr.fastaccess = fastAccess;
+        _builder.attribute.push_back(attr);
+    }
+    AttributeCollectionSpec::UP create(uint32_t docIdLimit,
+                                       search::SerialNum serialNum) {
+        return _factory.create(_builder, docIdLimit, serialNum);
+    }
+};
+
+struct NormalAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture
+{
+    NormalAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(false) {}
+};
+
+struct FastAccessAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture
+{
+    FastAccessAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(true) {}
+};
+
+TEST_F("require that normal attribute collection spec can be created",
+        NormalAttributeCollectionSpecFixture)
+{
+    AttributeCollectionSpec::UP spec = f.create(10, 20);
+    EXPECT_EQUAL(2u, spec->getAttributes().size());
+    EXPECT_EQUAL("a1", spec->getAttributes()[0].getName());
+    EXPECT_EQUAL("a2", spec->getAttributes()[1].getName());
+    EXPECT_EQUAL(10u, spec->getDocIdLimit());
+    EXPECT_EQUAL(20u, spec->getCurrentSerialNum());
+}
+
+TEST_F("require that fast access attribute collection spec can be created",
+        FastAccessAttributeCollectionSpecFixture)
+{
+    AttributeCollectionSpec::UP spec = f.create(10, 20);
+    EXPECT_EQUAL(1u, spec->getAttributes().size());
+    EXPECT_EQUAL("a2", spec->getAttributes()[0].getName());
+    EXPECT_EQUAL(10u, spec->getDocIdLimit());
+    EXPECT_EQUAL(20u, spec->getCurrentSerialNum());
+}
+
+const FilterAttributeManager::AttributeSet ACCEPTED_ATTRIBUTES = {"a2"};
+
+struct FilterFixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext _fileHeaderContext;
+    ForegroundTaskExecutor _attributeFieldWriter;
+    proton::AttributeManager::SP _baseMgr;
+    FilterAttributeManager _filterMgr;
+    FilterFixture()
+        : _dirHandler(test_dir),
+          _fileHeaderContext(),
+          _attributeFieldWriter(),
+          _baseMgr(new proton::AttributeManager(test_dir, "test.subdb",
+                                                TuneFileAttributes(),
+                                                _fileHeaderContext,
+                                                _attributeFieldWriter)),
+          _filterMgr(ACCEPTED_ATTRIBUTES, _baseMgr)
+    {
+        _baseMgr->addAttribute("a1", INT32_SINGLE, createSerialNum);
+        _baseMgr->addAttribute("a2", INT32_SINGLE, createSerialNum);
+   }
+};
+
+TEST_F("require that filter attribute manager can filter attributes", FilterFixture)
+{
+    EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == NULL);
+    EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != NULL);
+    std::vector<AttributeGuard> attrs;
+    f._filterMgr.getAttributeList(attrs);
+    EXPECT_EQUAL(1u, attrs.size());
+    EXPECT_EQUAL("a2", attrs[0].get().getName());
+}
+
+TEST_F("require that filter attribute manager can return flushed serial number", FilterFixture)
+{
+    f._baseMgr->flushAll(100);
+    EXPECT_EQUAL(0u, f._filterMgr.getFlushedSerialNum("a1"));
+    EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2"));
+}
+
+namespace {
+
+Tensor::UP
+createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
+    vespalib::tensor::DefaultTensor::builder builder;
+    return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+}
+
+}
+
+
+TEST_F("Test that we can use attribute writer to write to tensor attribute",
+       Fixture)
+{
+    proton::AttributeManager & am = *f._m;
+    AVConfig cfg(AVBasicType::TENSOR);
+    cfg.setTensorType(TensorType::fromSpec("tensor(x{},y{})"));
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             cfg,
+                                             createSerialNum);
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("a1", Schema::TENSOR,
+                                               Schema::SINGLE));
+    DocBuilder builder(s);
+    auto tensor = createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} },
+                               {"x", "y"});
+    Document::UP doc = builder.startDocument("doc::1").
+                       startAttributeField("a1").
+                       addTensor(tensor->clone()).endField().endDocument();
+    f.put(1, *doc, 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+    TensorAttribute *tensorAttribute =
+        dynamic_cast<TensorAttribute *>(a1.get());
+    EXPECT_TRUE(tensorAttribute != nullptr);
+    auto tensor2 = tensorAttribute->getTensor(1);
+    EXPECT_TRUE(static_cast<bool>(tensor2));
+    EXPECT_TRUE(tensor->equals(*tensor2));
+}
+
+TEST_MAIN()
+{
+    vespalib::rmdir(test_dir, true);
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.sh b/searchcore/src/tests/proton/attribute/attribute_test.sh
new file mode 100755
index 00000000000..950a9f92bb8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+rm -rf test_output
+$VALGRIND ./searchcore_attribute_test_app
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore b/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore
new file mode 100644
index 00000000000..2642c637ea0
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore
@@ -0,0 +1 @@
+searchcore_attribute_usage_filter_test_app
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt
new file mode 100644
index 00000000000..2dd66c2a3ec
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_usage_filter_test_app
+    SOURCES
+    attribute_usage_filter_test.cpp
+    DEPENDS
+    searchcore_attribute
+)
+vespa_add_test(NAME searchcore_attribute_usage_filter_test_app COMMAND searchcore_attribute_usage_filter_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC b/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC
new file mode 100644
index 00000000000..31b3afbcdf7
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC
@@ -0,0 +1 @@
+AttributeUsageFilter test. Take a look at attribute_usage_filter_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES b/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES
new file mode 100644
index 00000000000..b63aeb79d02
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES
@@ -0,0 +1 @@
+attribute_usage_filter_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp
new file mode 100644
index 00000000000..d8ede8030e2
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp
@@ -0,0 +1,143 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_usage_filter_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
+
+using proton::AttributeUsageFilter;
+using proton::AttributeUsageStats;
+
+namespace
+{
+
+search::AddressSpace enumStoreOverLoad(30 * 1024 * 1024 * UINT64_C(1024),
+                                       32 * 1024 * 1024 * UINT64_C(1024));
+
+search::AddressSpace multiValueOverLoad(127 * 1024 * 1024,
+                                        128 * 1024 * 1024);
+
+
+
+class MyAttributeStats : public AttributeUsageStats
+{
+public:
+    void triggerEnumStoreLimit() {
+        merge({        enumStoreOverLoad,
+                       search::AddressSpaceUsage::defaultMultiValueUsage() },
+              "enumeratedName",
+              "ready");
+    }
+
+    void triggerMultiValueLimit() {
+        merge({         search::AddressSpaceUsage::defaultEnumStoreUsage(),
+                       multiValueOverLoad },
+              "multiValueName",
+              "ready");
+    }
+};
+
+struct Fixture
+{
+    AttributeUsageFilter _filter;
+    using State = AttributeUsageFilter::State;
+    using Config = AttributeUsageFilter::Config;
+
+    Fixture()
+        : _filter()
+    {
+    }
+
+    void testWrite(const vespalib::string &exp) {
+        if (exp.empty()) {
+            EXPECT_TRUE(_filter.acceptWriteOperation());
+            State state = _filter.getAcceptState();
+            EXPECT_TRUE(state.acceptWriteOperation());
+            EXPECT_EQUAL(exp, state.message());
+        } else {
+            EXPECT_FALSE(_filter.acceptWriteOperation());
+            State state = _filter.getAcceptState();
+            EXPECT_FALSE(state.acceptWriteOperation());
+            EXPECT_EQUAL(exp, state.message());
+        }
+    }
+
+    void setAttributeStats(const AttributeUsageStats &stats) {
+        _filter.setAttributeStats(stats);
+    }
+};
+
+}
+
+TEST_F("Check that default filter allows write", Fixture)
+{
+    f.testWrite("");
+}
+
+
+TEST_F("Check that enum store limit can be reached", Fixture)
+{
+    f._filter.setConfig(Fixture::Config(0.8, 1.0));
+    MyAttributeStats stats;
+    stats.triggerEnumStoreLimit();
+    f.setAttributeStats(stats);
+    f.testWrite("enumStoreLimitReached: { "
+                "action: \""
+                "add more content nodes"
+                "\", "
+                "reason: \""
+                "enum store address space used (0.9375) > limit (0.8)"
+                "\", "
+                "enumStore: { used: 32212254720, limit: 34359738368}, "
+                "attributeName: \"enumeratedName\", subdb: \"ready\"}");
+}
+
+TEST_F("Check that multivalue limit can be reached", Fixture)
+{
+    f._filter.setConfig(Fixture::Config(1.0, 0.8));
+    MyAttributeStats stats;
+    stats.triggerMultiValueLimit();
+    f.setAttributeStats(stats);
+    f.testWrite("multiValueLimitReached: { "
+                "action: \""
+                "use 'huge' setting on attribute field "
+                "or add more content nodes"
+                "\", "
+                "reason: \""
+                "multiValue address space used (0.992188) > limit (0.8)"
+                "\", "
+                "multiValue: { used: 133169152, limit: 134217728}, "
+                "attributeName: \"multiValueName\", subdb: \"ready\"}");
+}
+
+TEST_F("Check that both enumstore limit and multivalue limit can be reached",
+       Fixture)
+{
+    f._filter.setConfig(Fixture::Config(0.8, 0.8));
+    MyAttributeStats stats;
+    stats.triggerEnumStoreLimit();
+    stats.triggerMultiValueLimit();
+    f.setAttributeStats(stats);
+    f.testWrite("enumStoreLimitReached: { "
+                "action: \""
+                "add more content nodes"
+                "\", "
+                "reason: \""
+                "enum store address space used (0.9375) > limit (0.8)"
+                "\", "
+                "enumStore: { used: 32212254720, limit: 34359738368}, "
+                "attributeName: \"enumeratedName\", subdb: \"ready\"}"
+                ", "
+                "multiValueLimitReached: { "
+                "action: \""
+                "use 'huge' setting on attribute field "
+                "or add more content nodes"
+                "\", "
+                "reason: \""
+                "multiValue address space used (0.992188) > limit (0.8)"
+                "\", "
+                "multiValue: { used: 133169152, limit: 134217728}, "
+                "attributeName: \"multiValueName\", subdb: \"ready\"}");
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
new file mode 100644
index 00000000000..53904e14658
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
@@ -0,0 +1,564 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attributeflush_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/flushableattribute.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/common/indexmetainfo.h>
+#include <vespa/searchlib/util/dirtraverse.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+
+#include <vespa/searchlib/attribute/attributevector.hpp>
+
+using namespace document;
+using namespace search;
+using namespace vespalib;
+
+using search::index::DummyFileHeaderContext;
+
+typedef search::attribute::Config AVConfig;
+typedef search::attribute::BasicType AVBasicType;
+typedef search::attribute::CollectionType AVCollectionType;
+
+typedef std::shared_ptr<Gate> GateSP;
+
+namespace proton {
+
+namespace
+{
+
+const uint64_t createSerialNum = 42u;
+
+}
+
+class TaskWrapper : public Executor::Task
+{
+private:
+    Executor::Task::UP _task;
+    GateSP             _gate;
+public:
+    TaskWrapper(Executor::Task::UP task, const GateSP &gate)
+        : _task(std::move(task)),
+          _gate(gate)
+    {
+    }
+
+    virtual void
+    run(void)
+    {
+        _task->run();
+        _gate->countDown();
+        LOG(info, "doneFlushing");
+    }
+};
+
+
+class FlushHandler
+{
+private:
+    ThreadStackExecutor _executor;
+public:
+    GateSP gate;
+
+    FlushHandler()
+        : _executor(1, 65536),
+          gate()
+    {
+    }
+
+    void
+    doFlushing(Executor::Task::UP task)
+    {
+        Executor::Task::UP wrapper(new TaskWrapper(std::move(task), gate));
+        Executor::Task::UP ok = _executor.execute(std::move(wrapper));
+        assert(ok.get() == NULL);
+    }
+};
+
+
+class UpdaterTask
+{
+private:
+    proton::AttributeManager & _am;
+public:
+    UpdaterTask(proton::AttributeManager & am)
+        :
+        _am(am)
+    {
+    }
+
+    void
+    startFlushing(uint64_t syncToken, FlushHandler & handler);
+
+    void
+    run(void);
+};
+
+
+void
+UpdaterTask::startFlushing(uint64_t syncToken, FlushHandler & handler)
+{
+    handler.gate.reset(new Gate());
+    IFlushTarget::SP flushable = _am.getFlushable("a1");
+    LOG(info, "startFlushing(%" PRIu64 ")", syncToken);
+    handler.doFlushing(flushable->initFlush(syncToken));
+}
+
+
+void
+UpdaterTask::run(void)
+{
+    LOG(info, "UpdaterTask::run(begin)");
+    uint32_t totalDocs = 2000000;
+    uint32_t totalDocsMax = 125000000; // XXX: Timing dependent.
+    uint32_t slowdownUpdateLim = 4000000;
+    bool slowedDown = false;
+    uint32_t incDocs = 1000;
+    uint64_t commits = 0;
+    uint32_t flushCount = 0;
+    uint64_t flushedToken = 0;
+    uint64_t needFlushToken = 0;
+    FlushHandler flushHandler;
+    for (uint32_t i = incDocs;
+         i <= totalDocs || (flushCount + (flushedToken <
+                                          needFlushToken) <= 2 &&
+                            i <= totalDocsMax);
+         i += incDocs) {
+        uint32_t startDoc = 0;
+        uint32_t lastDoc = 0;
+        AttributeGuard::UP agap = _am.getAttribute("a1");
+        AttributeGuard &ag(*agap);
+        IntegerAttribute & ia = static_cast<IntegerAttribute &>(*ag);
+        for (uint32_t j = i - incDocs; j < i; ++j) {
+            if (j >= ag->getNumDocs()) {
+                ag->addDocs(startDoc, lastDoc, incDocs);
+                if (i % (totalDocs / 20) == 0) {
+                    LOG(info,
+                        "addDocs(%u, %u, %u)",
+                        startDoc, lastDoc, ag->getNumDocs());
+                }
+            }
+            ia.update(j, i);
+        }
+        ia.commit(i-1, i); // save i as last sync token
+        needFlushToken = i;
+        assert(i + 1 == ag->getNumDocs());
+        if ((commits++ % 20 == 0) &&
+            (flushHandler.gate.get() == NULL ||
+             flushHandler.gate->getCount() == 0)) {
+            startFlushing(i, flushHandler);
+            ++flushCount;
+            flushedToken = i;
+            slowedDown = false;
+        }
+        if (needFlushToken > flushedToken + slowdownUpdateLim) {
+            FastOS_Thread::Sleep(100);
+            if (!slowedDown) {
+                LOG(warning,
+                    "Slowing down updates due to slow flushing (slow disk ?)");
+            }
+            slowedDown = true;
+        }
+    }
+    if (flushHandler.gate.get() != NULL) {
+        flushHandler.gate->await();
+    }
+    if (flushedToken < needFlushToken) {
+        startFlushing(needFlushToken, flushHandler);
+        flushHandler.gate->await();
+    }
+    LOG(info, "UpdaterTask::run(end)");
+}
+
+
+AVConfig
+getInt32Config()
+{
+    return AVConfig(AVBasicType::INT32);
+}
+
+
+class Test : public vespalib::TestApp
+{
+private:
+    void
+    requireThatUpdaterAndFlusherCanRunConcurrently(void);
+
+    void
+    requireThatFlushableAttributeReportsMemoryUsage(void);
+
+    void
+    requireThatFlushableAttributeManagesSyncTokenInfo(void);
+
+    void
+    requireThatFlushTargetsCanBeRetrieved(void);
+
+    void
+    requireThatCleanUpIsPerformedAfterFlush(void);
+
+    void
+    requireThatFlushStatsAreUpdated(void);
+
+    void
+    requireThatOnlyOneFlusherCanRunAtTheSameTime(void);
+
+    void
+    requireThatLastFlushTimeIsReported(void);
+
+    void
+    requireThatShrinkWorks();
+public:
+    int
+    Main(void);
+};
+
+
+const string test_dir = "flush";
+
+struct BaseFixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext   _fileHeaderContext;
+    ForegroundTaskExecutor   _attributeFieldWriter;
+    BaseFixture()
+        : _dirHandler(test_dir),
+          _fileHeaderContext(),
+          _attributeFieldWriter()
+    {
+    }
+};
+
+
+struct AttributeManagerFixture
+{
+    AttributeManager::SP _msp;
+    AttributeManager &_m;
+    AttributeWriter _aw;
+    AttributeManagerFixture(BaseFixture &bf)
+        : _msp(std::make_shared<AttributeManager>
+               (test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext,
+                bf._attributeFieldWriter)),
+          _m(*_msp),
+          _aw(_msp)
+    {
+    }
+    AttributeVector::SP addAttribute(const vespalib::string &name) {
+        return _m.addAttribute(name, getInt32Config(), createSerialNum);
+    }
+};
+
+struct Fixture : public BaseFixture, public AttributeManagerFixture
+{
+    Fixture()
+        : BaseFixture(),
+          AttributeManagerFixture(*static_cast<BaseFixture *>(this))
+    {
+    }
+};
+
+
+
+void
+Test::requireThatUpdaterAndFlusherCanRunConcurrently(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    EXPECT_TRUE(f.addAttribute("a1").get() != NULL);
+    IFlushTarget::SP ft = am.getFlushable("a1");
+    (static_cast<FlushableAttribute *>(ft.get()))->setCleanUpAfterFlush(false);
+    UpdaterTask updaterTask(am);
+    updaterTask.run();
+
+    IndexMetaInfo info("flush/a1");
+    EXPECT_TRUE(info.load());
+    EXPECT_TRUE(info.snapshots().size() > 2);
+    for (size_t i = 0; i < info.snapshots().size(); ++i) {
+        const IndexMetaInfo::Snapshot & snap = info.snapshots()[i];
+        LOG(info,
+            "Snapshot(%" PRIu64 ", %s)",
+            snap.syncToken, snap.dirName.c_str());
+        if (snap.syncToken > 0) {
+            EXPECT_TRUE(snap.valid);
+            std::string baseFileName = "flush/a1/" + snap.dirName + "/a1";
+            AttributeVector::SP attr =
+                AttributeFactory::createAttribute(baseFileName,
+                        getInt32Config());
+            EXPECT_TRUE(attr->load());
+            EXPECT_EQUAL((uint32_t)snap.syncToken + 1, attr->getNumDocs());
+        }
+    }
+}
+
+
+void
+Test::requireThatFlushableAttributeReportsMemoryUsage(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a2");
+    av->addDocs(100);
+    av->commit();
+    IFlushTarget::SP fa = am.getFlushable("a2");
+    EXPECT_TRUE(av->getStatus().getAllocated() >= 100u * sizeof(int32_t));
+    EXPECT_EQUAL(av->getStatus().getUsed(),
+               fa->getApproxMemoryGain().getBefore()+0lu);
+    // attributes stay in memory
+    EXPECT_EQUAL(fa->getApproxMemoryGain().getBefore(),
+               fa->getApproxMemoryGain().getAfter());
+}
+
+
+void
+Test::requireThatFlushableAttributeManagesSyncTokenInfo(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a3");
+    av->addDocs(1);
+    IFlushTarget::SP fa = am.getFlushable("a3");
+
+    IndexMetaInfo info("flush/a3");
+    EXPECT_EQUAL(0u, fa->getFlushedSerialNum());
+    EXPECT_TRUE(fa->initFlush(0).get() == NULL);
+    EXPECT_TRUE(info.load());
+    EXPECT_EQUAL(0u, info.snapshots().size());
+
+    av->commit(10, 10); // last sync token = 10
+    EXPECT_EQUAL(0u, fa->getFlushedSerialNum());
+    EXPECT_TRUE(fa->initFlush(10).get() != NULL);
+    fa->initFlush(10)->run();
+    EXPECT_EQUAL(10u, fa->getFlushedSerialNum());
+    EXPECT_TRUE(info.load());
+    EXPECT_EQUAL(1u, info.snapshots().size());
+    EXPECT_TRUE(info.snapshots()[0].valid);
+    EXPECT_EQUAL(10u, info.snapshots()[0].syncToken);
+
+    av->commit(20, 20); // last sync token = 20
+    EXPECT_EQUAL(10u, fa->getFlushedSerialNum());
+    fa->initFlush(20)->run();
+    EXPECT_EQUAL(20u, fa->getFlushedSerialNum());
+    EXPECT_TRUE(info.load());
+    EXPECT_EQUAL(1u, info.snapshots().size()); // snapshot 10 removed
+    EXPECT_TRUE(info.snapshots()[0].valid);
+    EXPECT_EQUAL(20u, info.snapshots()[0].syncToken);
+}
+
+
+void
+Test::requireThatFlushTargetsCanBeRetrieved(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    f.addAttribute("a4");
+    f.addAttribute("a5");
+    std::vector<IFlushTarget::SP> ftl = am.getFlushTargets();
+    EXPECT_EQUAL(2u, ftl.size());
+    EXPECT_EQUAL(am.getFlushable("a4").get(), ftl[0].get());
+    EXPECT_EQUAL(am.getFlushable("a5").get(), ftl[1].get());
+}
+
+
+void
+Test::requireThatCleanUpIsPerformedAfterFlush(void)
+{
+    Fixture f;
+    AttributeVector::SP av = f.addAttribute("a6");
+    av->addDocs(1);
+    av->commit(30, 30);
+
+    // fake up some snapshots
+    std::string snap10 = "flush/a6/snapshot-10";
+    std::string snap20 = "flush/a6/snapshot-20";
+    vespalib::mkdir(snap10, false);
+    vespalib::mkdir(snap20, false);
+    IndexMetaInfo info("flush/a6");
+    info.addSnapshot(IndexMetaInfo::Snapshot(true, 10, "snapshot-10"));
+    info.addSnapshot(IndexMetaInfo::Snapshot(false, 20, "snapshot-20"));
+    EXPECT_TRUE(info.save());
+
+    FlushableAttribute fa(av, "flush", TuneFileAttributes(),
+                          f._fileHeaderContext, f._attributeFieldWriter);
+    fa.initFlush(30)->run();
+
+    EXPECT_TRUE(info.load());
+    EXPECT_EQUAL(1u, info.snapshots().size()); // snapshots 10 & 20 removed
+    EXPECT_TRUE(info.snapshots()[0].valid);
+    EXPECT_EQUAL(30u, info.snapshots()[0].syncToken);
+    FastOS_StatInfo statInfo;
+    EXPECT_TRUE(!FastOS_File::Stat(snap10.c_str(), &statInfo));
+    EXPECT_TRUE(!FastOS_File::Stat(snap20.c_str(), &statInfo));
+}
+
+
+void
+Test::requireThatFlushStatsAreUpdated(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a7");
+    av->addDocs(1);
+    av->commit(100,100);
+    IFlushTarget::SP ft = am.getFlushable("a7");
+    ft->initFlush(101)->run();
+    FlushStats stats = ft->getLastFlushStats();
+    EXPECT_EQUAL("flush/a7/snapshot-101", stats.getPath());
+    EXPECT_EQUAL(8u, stats.getPathElementsToLog());
+}
+
+
+void
+Test::requireThatOnlyOneFlusherCanRunAtTheSameTime(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a8");
+    av->addDocs(10000);
+    av->commit(9,9);
+    IFlushTarget::SP ft = am.getFlushable("a8");
+    (static_cast<FlushableAttribute *>(ft.get()))->setCleanUpAfterFlush(false);
+    vespalib::ThreadStackExecutor exec(16, 64000);
+
+    for (size_t i = 10; i < 100; ++i) {
+        av->commit(i, i);
+        vespalib::Executor::Task::UP task = ft->initFlush(i);
+        exec.execute(std::move(task));
+    }
+    exec.sync();
+    exec.shutdown();
+
+    IndexMetaInfo info("flush/a8");
+    ASSERT_TRUE(info.load());
+    LOG(info, "Found %zu snapshots", info.snapshots().size());
+    for (size_t i = 0; i < info.snapshots().size(); ++i) {
+        EXPECT_EQUAL(true, info.snapshots()[i].valid);
+    }
+    IndexMetaInfo::Snapshot best = info.getBestSnapshot();
+    EXPECT_EQUAL(true, best.valid);
+    EXPECT_EQUAL(99u, best.syncToken);
+    FlushStats stats = ft->getLastFlushStats();
+    EXPECT_EQUAL("flush/a8/snapshot-99", stats.getPath());
+}
+
+
+void
+Test::requireThatLastFlushTimeIsReported(void)
+{
+    BaseFixture f;
+    FastOS_StatInfo stat;
+    { // no meta info file yet
+        AttributeManagerFixture amf(f);
+        AttributeManager &am = amf._m;
+        AttributeVector::SP av = amf.addAttribute("a9");
+        EXPECT_EQUAL(0, am.getFlushable("a9")->getLastFlushTime().time());
+    }
+    { // no snapshot flushed yet
+        AttributeManagerFixture amf(f);
+        AttributeManager &am = amf._m;
+        AttributeVector::SP av = amf.addAttribute("a9");
+        IFlushTarget::SP ft = am.getFlushable("a9");
+        EXPECT_EQUAL(0, ft->getLastFlushTime().time());
+        ft->initFlush(5)->run();
+        EXPECT_TRUE(FastOS_File::Stat("flush/a9/snapshot-5", &stat));
+        EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime().time());
+    }
+    { // snapshot flushed
+        AttributeManagerFixture amf(f);
+        AttributeManager &am = amf._m;
+        amf.addAttribute("a9");
+        IFlushTarget::SP ft = am.getFlushable("a9");
+        EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime().time());
+        { // updated flush time after nothing to flush
+            FastOS_Thread::Sleep(8000);
+            fastos::TimeStamp now = fastos::ClockSystem::now();
+            Executor::Task::UP task = ft->initFlush(5);
+            EXPECT_TRUE(task.get() == NULL);
+            EXPECT_LESS(stat._modifiedTime, ft->getLastFlushTime().time());
+            EXPECT_APPROX(now.time(), ft->getLastFlushTime().time(), 8);
+        }
+    }
+}
+
+
+void
+Test::requireThatShrinkWorks()
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a10");
+    
+    av->addDocs(1000 - av->getNumDocs());
+    av->commit(10, 10);
+    IFlushTarget::SP ft = am.getFlushable("a10");
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+    AttributeGuard::UP g = am.getAttribute("a10");
+    EXPECT_FALSE(av->wantShrinkLidSpace());
+    EXPECT_FALSE(av->canShrinkLidSpace());
+    EXPECT_EQUAL(1000u, av->getNumDocs());
+    EXPECT_EQUAL(1000u, av->getCommittedDocIdLimit());
+    av->compactLidSpace(100);
+    EXPECT_TRUE(av->wantShrinkLidSpace());
+    EXPECT_FALSE(av->canShrinkLidSpace());
+    EXPECT_EQUAL(1000u, av->getNumDocs());
+    EXPECT_EQUAL(100u, av->getCommittedDocIdLimit());
+    f._aw.heartBeat(11);
+    EXPECT_TRUE(av->wantShrinkLidSpace());
+    EXPECT_FALSE(av->canShrinkLidSpace());
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+    g.reset();
+    f._aw.heartBeat(11);
+    EXPECT_TRUE(av->wantShrinkLidSpace());
+    EXPECT_TRUE(av->canShrinkLidSpace());
+    EXPECT_TRUE(ft->getApproxMemoryGain().getBefore() >
+                ft->getApproxMemoryGain().getAfter());
+    EXPECT_EQUAL(1000u, av->getNumDocs());
+    EXPECT_EQUAL(100u, av->getCommittedDocIdLimit());
+    vespalib::ThreadStackExecutor exec(1, 128 * 1024);
+    vespalib::Executor::Task::UP task = ft->initFlush(11);
+    exec.execute(std::move(task));
+    exec.sync();
+    exec.shutdown();
+    EXPECT_FALSE(av->wantShrinkLidSpace());
+    EXPECT_FALSE(av->canShrinkLidSpace());
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+    EXPECT_EQUAL(100u, av->getNumDocs());
+    EXPECT_EQUAL(100u, av->getCommittedDocIdLimit());
+}
+
+
+int
+Test::Main(void)
+{
+    TEST_INIT("attributeflush_test");
+
+    if (_argc > 0) {
+        DummyFileHeaderContext::setCreator(_argv[0]);
+    }
+    vespalib::rmdir(test_dir, true);
+    TEST_DO(requireThatUpdaterAndFlusherCanRunConcurrently());
+    TEST_DO(requireThatFlushableAttributeReportsMemoryUsage());
+    TEST_DO(requireThatFlushableAttributeManagesSyncTokenInfo());
+    TEST_DO(requireThatFlushTargetsCanBeRetrieved());
+    TEST_DO(requireThatCleanUpIsPerformedAfterFlush());
+    TEST_DO(requireThatFlushStatsAreUpdated());
+    TEST_DO(requireThatOnlyOneFlusherCanRunAtTheSameTime());
+    TEST_DO(requireThatLastFlushTimeIsReported());
+    TEST_DO(requireThatShrinkWorks());
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(proton::Test);
diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.sh b/searchcore/src/tests/proton/attribute/attributeflush_test.sh
new file mode 100755
index 00000000000..8ec2f5d8dd8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributeflush_test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+rm -rf flush
+$VALGRIND ./searchcore_attributeflush_test_app
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore b/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore
new file mode 100644
index 00000000000..3b612102a10
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore
@@ -0,0 +1 @@
+searchcore_attributes_state_explorer_test_app
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt
new file mode 100644
index 00000000000..322d22c8f0d
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attributes_state_explorer_test_app
+    SOURCES
+    attributes_state_explorer_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attributes_state_explorer_test_app COMMAND searchcore_attributes_state_explorer_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC b/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC
new file mode 100644
index 00000000000..1459d32ddae
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC
@@ -0,0 +1 @@
+attributes_state_explorer test. Take a look at attributes_state_explorer_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES b/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES
new file mode 100644
index 00000000000..f49eb2b8e86
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES
@@ -0,0 +1 @@
+attributes_state_explorer_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
new file mode 100644
index 00000000000..43eeec6086a
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attributes_state_explorer_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/test/attribute_vectors.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+
+using namespace proton;
+using namespace proton::test;
+using search::index::DummyFileHeaderContext;
+using search::AttributeVector;
+using search::TuneFileAttributes;
+using search::ForegroundTaskExecutor;
+
+const vespalib::string TEST_DIR = "test_output";
+
+struct Fixture
+{
+    DirectoryHandler _dirHandler;
+    DummyFileHeaderContext _fileHeaderContext;
+    ForegroundTaskExecutor _attributeFieldWriter;
+    AttributeManager::SP _mgr;
+    AttributeManagerExplorer _explorer;
+    Fixture()
+        : _dirHandler(TEST_DIR),
+          _fileHeaderContext(),
+          _attributeFieldWriter(),
+          _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(),
+                                    _fileHeaderContext,
+                                    _attributeFieldWriter)),
+          _explorer(_mgr)
+    {
+        addAttribute("regular");
+        addExtraAttribute("extra");
+    }
+    void addAttribute(const vespalib::string &name) {
+        _mgr->addAttribute(name, AttributeUtils::getInt32Config(), 1);
+    }
+    void addExtraAttribute(const vespalib::string &name) {
+        _mgr->addExtraAttribute(AttributeVector::SP(new Int32Attribute(name)));
+    }
+};
+
+typedef std::vector<vespalib::string> StringVector;
+
+TEST_F("require that attributes are exposed as children names", Fixture)
+{
+    StringVector children = f._explorer.get_children_names();
+    std::sort(children.begin(), children.end());
+    EXPECT_EQUAL(StringVector({"extra", "regular"}), children);
+}
+
+TEST_F("require that attributes are explorable", Fixture)
+{
+    EXPECT_TRUE(f._explorer.get_child("regular").get() != nullptr);
+    EXPECT_TRUE(f._explorer.get_child("extra").get() != nullptr);
+    EXPECT_TRUE(f._explorer.get_child("not").get() == nullptr);
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore b/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore
new file mode 100644
index 00000000000..45cd0a54f56
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore
@@ -0,0 +1 @@
+searchcore_document_field_populator_test_app
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt
new file mode 100644
index 00000000000..4c6da0a3397
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_document_field_populator_test_app
+    SOURCES
+    document_field_populator_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_document_field_populator_test_app COMMAND searchcore_document_field_populator_test_app)
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/DESC b/searchcore/src/tests/proton/attribute/document_field_populator/DESC
new file mode 100644
index 00000000000..cdc71250210
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/DESC
@@ -0,0 +1 @@
+document_field_populator test. Take a look at document_field_populator_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/FILES b/searchcore/src/tests/proton/attribute/document_field_populator/FILES
new file mode 100644
index 00000000000..21f62452acf
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/FILES
@@ -0,0 +1 @@
+document_field_populator_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp
new file mode 100644
index 00000000000..d0be50bfd2f
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("document_field_populator_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/attribute/document_field_populator.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+
+typedef search::attribute::Config AVConfig;
+typedef search::attribute::BasicType AVBasicType;
+
+Schema::AttributeField
+createAttributeField()
+{
+    return Schema::AttributeField("a1", Schema::DataType::INT32);
+}
+
+Schema
+createSchema()
+{
+    Schema schema;
+    schema.addAttributeField(createAttributeField());
+    return schema;
+}
+
+struct DocContext
+{
+    Schema _schema;
+    DocBuilder _builder;
+    DocContext()
+        : _schema(createSchema()),
+          _builder(_schema)
+    {
+    }
+    Document::UP create(uint32_t id) {
+        vespalib::string docId =
+                vespalib::make_string("id:searchdocument:searchdocument::%u", id);
+        return _builder.startDocument(docId).endDocument();
+    }
+};
+
+struct Fixture
+{
+    AttributeVector::SP _attr;
+    IntegerAttribute &_intAttr;
+    DocumentFieldPopulator _pop;
+    DocContext _ctx;
+    Fixture()
+        : _attr(search::AttributeFactory::createAttribute("a1", AVConfig(AVBasicType::INT32))),
+          _intAttr(dynamic_cast<IntegerAttribute &>(*_attr)),
+          _pop(createAttributeField(), _attr, "test"),
+          _ctx()
+    {
+        _intAttr.addDocs(2);
+        _intAttr.update(1, 100);
+        _intAttr.commit();
+    }
+};
+
+TEST_F("require that document field is populated based on attribute content", Fixture)
+{
+    // NOTE: DocumentFieldRetriever (used by DocumentFieldPopulator) is fully tested
+    // with all data types in searchcore/src/tests/proton/server/documentretriever_test.cpp.
+    {
+        Document::UP doc = f._ctx.create(1);
+        f._pop.handleExisting(1, *doc);
+        EXPECT_EQUAL(100, doc->getValue("a1")->getAsInt());
+    }
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore
new file mode 100644
index 00000000000..f3666eecb6e
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore
@@ -0,0 +1 @@
+searchcore_exclusive_attribute_read_accessor_test_app
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt
new file mode 100644
index 00000000000..c39025ae39f
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_exclusive_attribute_read_accessor_test_app
+    SOURCES
+    exclusive_attribute_read_accessor_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_exclusive_attribute_read_accessor_test_app COMMAND searchcore_exclusive_attribute_read_accessor_test_app)
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC
new file mode 100644
index 00000000000..ec5a407ddbd
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC
@@ -0,0 +1 @@
+exclusive_attribute_read_accessor test. Take a look at exclusive_attribute_read_accessor_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES
new file mode 100644
index 00000000000..74a9ab77547
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES
@@ -0,0 +1 @@
+exclusive_attribute_read_accessor_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp
new file mode 100644
index 00000000000..7cb6a503ae8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h>
+#include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sync.h>
+
+using namespace proton;
+using namespace search;
+using namespace vespalib;
+
+using ReadGuard = ExclusiveAttributeReadAccessor::Guard;
+
+AttributeVector::SP
+createAttribute()
+{
+    attribute::Config cfg(attribute::BasicType::INT32, attribute::CollectionType::SINGLE);
+    return search::AttributeFactory::createAttribute("myattr", cfg);
+}
+
+struct Fixture
+{
+    AttributeVector::SP attribute;
+    SequencedTaskExecutor writer;
+    ExclusiveAttributeReadAccessor accessor;
+
+    Fixture()
+        : attribute(createAttribute()),
+          writer(1),
+          accessor(attribute, writer)
+    {}
+};
+
+TEST_F("require that attribute write thread is blocked while guard is held", Fixture)
+{
+    ReadGuard::UP guard = f.accessor.takeGuard();
+    Gate gate;
+    f.writer.execute("myattr", [&gate]() { gate.countDown(); });
+    bool reachedZero = gate.await(100);
+    EXPECT_FALSE(reachedZero);
+    EXPECT_EQUAL(1u, gate.getCount());
+
+    guard.reset();
+    gate.await();
+    EXPECT_EQUAL(0u, gate.getCount());
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore b/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore