Publish

1235 files changed, 109395 insertions, 0 deletions

diff --git a/searchcore/src/.gitignore b/searchcore/src/.gitignore
new file mode 100644
index 00000000000..78bed104e89
--- /dev/null
+++ b/searchcore/src/.gitignore
@@ -0,0 +1,17 @@
+*.dsp
+*.dsw
+*.ncb
+*.opt
+*.plg
+ID
+Makefile.ini
+bin
+config_command.bat
+config_command.sh
+distrib
+doxygen.err.log
+lib
+make.bootstrap
+output
+project.ncb
+project.opt
diff --git a/searchcore/src/apps/.gitignore b/searchcore/src/apps/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/apps/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/apps/fdispatch/.gitignore b/searchcore/src/apps/fdispatch/.gitignore
new file mode 100644
index 00000000000..7ef8af69759
--- /dev/null
+++ b/searchcore/src/apps/fdispatch/.gitignore
@@ -0,0 +1,2 @@
+/fdispatch
+fdispatch-bin
diff --git a/searchcore/src/apps/fdispatch/CMakeLists.txt b/searchcore/src/apps/fdispatch/CMakeLists.txt
new file mode 100644
index 00000000000..142826ec4ee
--- /dev/null
+++ b/searchcore/src/apps/fdispatch/CMakeLists.txt
@@ -0,0 +1,14 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_fdispatch_app
+    SOURCES
+    fdispatch.cpp
+    OUTPUT_NAME fdispatch-bin
+    INSTALL sbin
+    DEPENDS
+    searchcore_fdispatch_program
+    searchcore_fdispatch_search
+    searchcore_grouping
+    searchcore_fdcommon
+    searchcore_util
+    searchcore_fconfig
+)
diff --git a/searchcore/src/apps/fdispatch/fdispatch.cpp b/searchcore/src/apps/fdispatch/fdispatch.cpp
new file mode 100644
index 00000000000..58af00accf1
--- /dev/null
+++ b/searchcore/src/apps/fdispatch/fdispatch.cpp
@@ -0,0 +1,258 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("fdispatch");
+
+#include <vespa/searchcore/fdispatch/program/fdispatch.h>
+
+#include <vespa/searchcore/fdispatch/common/perftask.h>
+
+#include <vespa/vespalib/net/state_server.h>
+#include <vespa/vespalib/net/simple_health_producer.h>
+#include <vespa/vespalib/net/simple_metrics_producer.h>
+
+#include <vespa/searchlib/expression/forcelink.hpp>
+#include <vespa/searchlib/aggregation/forcelink.hpp>
+
+#include <vespa/vespalib/util/signalhandler.h>
+
+
+using fdispatch::Fdispatch;
+using vespa::config::search::core::FdispatchrcConfig;
+
+extern char FastS_VersionTag[];
+
+class FastS_FDispatchApp : public FastOS_Application
+{
+private:
+    FastS_FDispatchApp(const FastS_FDispatchApp &);
+    FastS_FDispatchApp& operator=(const FastS_FDispatchApp &);
+
+protected:
+    vespalib::string _configId;
+
+    bool CheckShutdownFlags ()
+    {
+        return (vespalib::SignalHandler::INT.check() || vespalib::SignalHandler::TERM.check());
+    }
+
+    void Usage(void);
+    bool GetOptions(int *exitCode);
+
+public:
+    int Main();
+    FastS_FDispatchApp(void);
+    ~FastS_FDispatchApp(void);
+};
+
+
+FastS_FDispatchApp::FastS_FDispatchApp(void)
+{
+}
+
+
+/**
+ * Main program for a dispatcher node.
+ */
+int
+FastS_FDispatchApp::Main()
+{
+    int exitCode;
+
+    forcelink_searchlib_expression();
+    forcelink_searchlib_aggregation();
+
+    if (!GetOptions(&exitCode)) {
+        EV_STOPPING("fdispatch",
+                    (exitCode == 0) ? "clean shutdown" : "error");
+        return exitCode;
+    }
+
+    exitCode = 0;
+
+    EV_STARTED("fdispatch");
+
+    vespalib::SignalHandler::INT.hook();
+    vespalib::SignalHandler::TERM.hook();
+    vespalib::SignalHandler::PIPE.ignore();
+
+    std::unique_ptr<Fdispatch> myfdispatch;
+    try {
+        myfdispatch.reset(new Fdispatch(_configId));
+    } catch (std::exception &ex) {
+        LOG(error, "getting config: %s", (const char *) ex.what());
+        exitCode = 1;
+        EV_STOPPING("fdispatch", "error getting config");
+        return exitCode;
+    }
+
+    try {
+#ifdef RLIMIT_NOFILE
+    struct rlimit  curlim;
+    getrlimit(RLIMIT_NOFILE, &curlim);
+    if (curlim.rlim_cur != (rlim_t)RLIM_INFINITY)
+        LOG(debug, "Max number of open files = %d", (int) curlim.rlim_cur);
+    else
+        LOG(debug, "Max number of open files = unlimited");
+    if (curlim.rlim_cur >= 64) {
+    } else {
+        LOG(error,
+            "CRITICAL: Too few file descriptors available: %d",
+            (int)curlim.rlim_cur);
+        throw std::runtime_error("CRITICAL: Too few file descriptors available");
+    }
+#endif
+#ifdef RLIMIT_DATA
+    getrlimit(RLIMIT_DATA, &curlim);
+    if (curlim.rlim_cur != (rlim_t)RLIM_INFINITY &&
+        curlim.rlim_cur < (rlim_t) (400 * 1024 * 1024)) {
+        if (curlim.rlim_max == (rlim_t)RLIM_INFINITY)
+            curlim.rlim_cur = (rlim_t) (400 * 1024 * 1024);
+        else
+            curlim.rlim_cur = curlim.rlim_max;
+        setrlimit(RLIMIT_DATA, &curlim);
+        getrlimit(RLIMIT_DATA, &curlim);
+    }
+    if (curlim.rlim_cur != (rlim_t)RLIM_INFINITY)
+        LOG(debug,
+            "VERBOSE: Max data segment size = %dM",
+            (int) ((curlim.rlim_cur + 512 * 1024) / (1024 * 1024)));
+    else
+        LOG(debug, "VERBOSE: Max data segment size = unlimited");
+#endif
+
+        if (!myfdispatch->Init()) {
+            throw std::runtime_error("myfdispatch->Init(" + _configId + ") failed");
+        }
+        if (myfdispatch->Failed()) {
+            throw std::runtime_error("myfdispatch->Failed()");
+        }
+        { // main loop scope
+            vespalib::SimpleHealthProducer health;
+            vespalib::SimpleMetricsProducer metrics;
+            vespalib::StateServer stateServer(myfdispatch->getHealthPort(), health, metrics, myfdispatch->getComponentConfig());
+            FastS_PerfTask perfTask(*myfdispatch, 300.0);
+            while (!CheckShutdownFlags()) {
+                if (myfdispatch->Failed()) {
+                    throw std::runtime_error("myfdispatch->Failed()");
+                }
+                FastOS_Thread::Sleep(1000);
+#ifndef NO_MONITOR_LATENCY_CHECK
+                if (!myfdispatch->CheckTempFail())
+                    break;
+#endif
+            }
+        } // main loop scope
+        if (myfdispatch->Failed()) {
+            throw std::runtime_error("myfdispatch->Failed()");
+        }
+    } catch (std::exception &e) {
+        LOG(error, "got exception during init: %s", e.what());
+        exitCode = 1;
+    } catch (...) {
+        LOG(error, "got exception during init");
+        exitCode = 1;
+    }
+
+    LOG(debug, "Deleting fdispatch");
+    myfdispatch.reset();
+    LOG(debug, "COMPLETION: Exiting");
+    EV_STOPPING("fdispatch",
+                (exitCode == 0) ? "clean shutdown" : "error");
+    return exitCode;
+}
+
+
+bool
+FastS_FDispatchApp::GetOptions(int *exitCode)
+{
+    int errflg = 0;
+    int c;
+    const char *optArgument;
+    int longopt_index;	/* Shows which long option was used */
+    static struct option longopts[] = {
+        { "config-id", 1, NULL, 0 },
+        { NULL, 0, NULL, 0 }
+    };
+    enum longopts_enum {
+        LONGOPT_CONFIGID
+    };
+    int optIndex = 1;    // Start with argument 1
+    while ((c = GetOptLong("c:",
+                           optArgument,
+                           optIndex,
+                           longopts,
+                           &longopt_index)) != -1) {
+        switch (c) {
+        case 0:
+            switch (longopt_index) {
+            case LONGOPT_CONFIGID:
+                break;
+            default:
+                if (optArgument != NULL)
+                    LOG(info,
+                        "longopt %s with arg %s",
+                        longopts[longopt_index].name, optArgument);
+                else
+                    LOG(info,
+                        "longopt %s",
+                        longopts[longopt_index].name);
+                break;
+            }
+            break;
+        case 'c':
+            _configId = optArgument;
+            break;
+        case '?':
+        default:
+            errflg++;
+        }
+    }
+    if (errflg) {
+        Usage();
+        *exitCode = 1;
+        return false;
+    }
+    return true;
+}
+
+void
+FastS_FDispatchApp::Usage(void)
+{
+    printf("FAST Search - fdispatch %s\n", FastS_VersionTag);
+    printf("\n"
+           "USAGE:\n"
+           "\n"
+           "fdispatch [-C fsroot] [-c rcFile] [-P preHttPort] [-V] [-w FFF]\n"
+           "\n"
+           "  -C fsroot      Fast Search's root directory\n"
+           "                 (default /usr/fastsearch/fastserver4)\n"
+           "  -c rcFile      fdispatchrc file (default FASTSEARCHROOT/etc/fdispatchrc)\n"
+           "  -P preHttPort  pre-allocated socket number for http service\n"
+           "  -V             show version and exit\n"
+           "  -w FFF         hex value (max 32 bit) for the Verbose mask\n"
+           "\n");
+}
+
+
+FastS_FDispatchApp::~FastS_FDispatchApp(void)
+{
+}
+
+
+int
+main(int argc, char **argv)
+{
+    FastS_FDispatchApp app;
+    int retval;
+
+    // Maybe this should be handeled by FastOS
+    setlocale(LC_ALL, "C");
+
+    retval = app.Entry(argc, argv);
+
+    return retval;
+}
diff --git a/searchcore/src/apps/proton/.gitignore b/searchcore/src/apps/proton/.gitignore
new file mode 100644
index 00000000000..4c244afd919
--- /dev/null
+++ b/searchcore/src/apps/proton/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+proton
+proton-bin
diff --git a/searchcore/src/apps/proton/CMakeLists.txt b/searchcore/src/apps/proton/CMakeLists.txt
new file mode 100644
index 00000000000..bcdc4fb95fb
--- /dev/null
+++ b/searchcore/src/apps/proton/CMakeLists.txt
@@ -0,0 +1,28 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_proton_app
+    SOURCES
+    downpersistence.cpp
+    proton.cpp
+    OUTPUT_NAME proton-bin
+    INSTALL sbin
+    DEPENDS
+    searchcore_server
+    searchcore_initializer
+    searchcore_reprocessing
+    searchcore_index
+    searchcore_persistenceengine
+    searchcore_feedoperation
+    searchcore_matchengine
+    searchcore_summaryengine
+    searchcore_matching
+    searchcore_docsummary
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_flushengine
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_fconfig
+    searchcore_util
+)
diff --git a/searchcore/src/apps/proton/downpersistence.cpp b/searchcore/src/apps/proton/downpersistence.cpp
new file mode 100644
index 00000000000..2217121c491
--- /dev/null
+++ b/searchcore/src/apps/proton/downpersistence.cpp
@@ -0,0 +1,200 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".downpersistence");
+#include <vespa/persistence/spi/persistenceprovider.h>
+#include <vespa/searchlib/util/statefile.h>
+#include "downpersistence.h"
+
+namespace storage
+{
+
+namespace spi
+{
+
+namespace
+{
+
+Result errorResult(Result::FATAL_ERROR, "Node is down");
+
+}
+
+
+DownPersistence::DownPersistence(const vespalib::string &downReason)
+    : _downReason(downReason)
+{
+}
+
+
+DownPersistence::~DownPersistence()
+{
+}
+
+
+Result
+DownPersistence::initialize()
+{
+    return Result();
+}
+
+PartitionStateListResult
+DownPersistence::getPartitionStates() const
+{
+    PartitionStateList list(1);
+    list[0] = PartitionState(PartitionState::DOWN, _downReason);
+    return PartitionStateListResult(list);
+}
+
+BucketIdListResult
+DownPersistence::listBuckets(PartitionId) const
+{
+    return BucketIdListResult(errorResult.getErrorCode(),
+                              errorResult.getErrorMessage());
+}
+
+Result
+DownPersistence:: setClusterState(const ClusterState&)
+{
+    return Result();
+}
+
+Result
+DownPersistence:: setActiveState(const Bucket&, BucketInfo::ActiveState)
+{
+    return errorResult;
+}
+
+
+BucketInfoResult
+DownPersistence:: getBucketInfo(const Bucket&) const
+{
+    return BucketInfoResult(errorResult.getErrorCode(),
+                            errorResult.getErrorMessage());
+}
+
+Result
+DownPersistence::put(const Bucket&, Timestamp, const Document::SP&, Context&)
+{
+    return errorResult;
+}
+
+
+RemoveResult
+DownPersistence:: remove(const Bucket&, Timestamp,
+                         const DocumentId&, Context&)
+{
+    return RemoveResult(errorResult.getErrorCode(),
+                        errorResult.getErrorMessage());
+}
+
+
+RemoveResult
+DownPersistence::removeIfFound(const Bucket&, Timestamp,
+                               const DocumentId&, Context&)
+{
+    return RemoveResult(errorResult.getErrorCode(),
+                        errorResult.getErrorMessage());
+}
+
+
+Result
+DownPersistence::removeEntry(const Bucket&, Timestamp, Context&)
+{
+    return errorResult;
+}
+
+
+UpdateResult DownPersistence::update(const Bucket&, Timestamp,
+                                     const DocumentUpdate::SP&, Context&)
+{
+    return UpdateResult(errorResult.getErrorCode(),
+                        errorResult.getErrorMessage());
+}
+
+
+Result
+DownPersistence::flush(const Bucket&, Context&)
+{
+    return errorResult;
+}
+
+GetResult
+DownPersistence::get(const Bucket&, const document::FieldSet&,
+                     const DocumentId&, Context&) const
+{
+    return GetResult(errorResult.getErrorCode(),
+                     errorResult.getErrorMessage());
+}
+
+CreateIteratorResult
+DownPersistence::createIterator(const Bucket&, const document::FieldSet&,
+                                const Selection&, IncludedVersions,
+                                Context&)
+{
+    return CreateIteratorResult(errorResult.getErrorCode(),
+                                errorResult.getErrorMessage());
+}
+
+IterateResult
+DownPersistence::iterate(IteratorId, uint64_t, Context&) const
+{
+    return IterateResult(errorResult.getErrorCode(),
+                         errorResult.getErrorMessage());
+}
+
+Result
+DownPersistence::destroyIterator(IteratorId, Context&)
+{
+    return errorResult;
+}
+
+Result
+DownPersistence::createBucket(const Bucket&, Context&)
+{
+    return errorResult;
+}
+
+Result
+DownPersistence::deleteBucket(const Bucket&, Context&)
+{
+    return errorResult;
+}
+
+
+BucketIdListResult
+DownPersistence::getModifiedBuckets() const
+{
+    return BucketIdListResult(errorResult.getErrorCode(),
+                              errorResult.getErrorMessage());
+}
+
+
+Result
+DownPersistence::maintain(const Bucket&, MaintenanceLevel)
+{
+    return errorResult;
+}
+
+Result
+DownPersistence::split(const Bucket&, const Bucket&, const Bucket&, Context&)
+{
+    return errorResult;
+}
+
+
+Result
+DownPersistence::join(const Bucket&, const Bucket&, const Bucket&, Context&)
+{
+    return errorResult;
+}
+
+
+Result
+DownPersistence::move(const Bucket&, PartitionId, Context&)
+{
+    return errorResult;
+}
+
+}
+
+}
diff --git a/searchcore/src/apps/proton/downpersistence.h b/searchcore/src/apps/proton/downpersistence.h
new file mode 100644
index 00000000000..ee49416b6dc
--- /dev/null
+++ b/searchcore/src/apps/proton/downpersistence.h
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+namespace storage
+{
+
+namespace spi
+{
+
+/*
+ * Persistence provider that returns error code for all operations
+ * except initialize(), getPartitionStates() and setClusterState().
+ *
+ * getPartitionStates() reports one partition, which is down with
+ * reason "proton state string is " + stateString.
+ *
+ * This class is used when proton is supposed to be down except for
+ * reporting state to cluster controller.
+ */
+class DownPersistence : public PersistenceProvider
+{
+    const vespalib::string _downReason;
+
+public:
+    DownPersistence(const vespalib::string &downReason);
+
+    typedef std::unique_ptr<PersistenceProvider> UP;
+
+    virtual ~DownPersistence();
+
+    virtual Result initialize() override;
+
+    virtual PartitionStateListResult getPartitionStates() const override;
+
+    virtual BucketIdListResult listBuckets(PartitionId) const override;
+
+    virtual Result setClusterState(const ClusterState&) override;
+
+    virtual Result setActiveState(const Bucket&,
+                                  BucketInfo::ActiveState) override;
+
+    virtual BucketInfoResult getBucketInfo(const Bucket&) const override;
+
+    virtual Result put(const Bucket&, Timestamp, const Document::SP&, Context&) override;
+
+    virtual RemoveResult remove(const Bucket&,
+                                Timestamp timestamp,
+                                const DocumentId& id,
+                                Context&) override;
+
+    virtual RemoveResult removeIfFound(const Bucket&,
+                                       Timestamp timestamp,
+                                       const DocumentId& id,
+                                       Context&) override;
+
+    virtual Result removeEntry(const Bucket&, Timestamp, Context&) override;
+
+    virtual UpdateResult update(const Bucket&,
+                                Timestamp timestamp,
+                                const DocumentUpdate::SP& update,
+                                Context&) override;
+
+    virtual Result flush(const Bucket&, Context&) override;
+
+    virtual GetResult get(const Bucket&,
+                          const document::FieldSet& fieldSet,
+                          const DocumentId& id,
+                          Context&) const override;
+
+    virtual CreateIteratorResult createIterator(
+            const Bucket&,
+            const document::FieldSet& fieldSet,
+            const Selection& selection, //TODO: Make AST
+            IncludedVersions versions,
+            Context&) override;
+
+    virtual IterateResult iterate(IteratorId id,
+                                  uint64_t maxByteSize,
+                                  Context&) const override;
+
+    virtual Result destroyIterator(IteratorId id, Context&) override;
+
+    virtual Result createBucket(const Bucket&, Context&) override;
+
+    virtual Result deleteBucket(const Bucket&, Context&) override;
+
+    virtual BucketIdListResult getModifiedBuckets() const override;
+
+    virtual Result maintain(const Bucket&,
+                            MaintenanceLevel level) override;
+
+    virtual Result split(const Bucket& source,
+                         const Bucket& target1,
+                         const Bucket& target2,
+                         Context&) override;
+
+    virtual Result join(const Bucket& source1,
+                        const Bucket& source2,
+                        const Bucket& target,
+                        Context&) override;
+
+    virtual Result move(const Bucket&, PartitionId target, Context&) override;
+
+};
+
+}
+
+}
diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp
new file mode 100644
index 00000000000..3cea3020805
--- /dev/null
+++ b/searchcore/src/apps/proton/proton.cpp
@@ -0,0 +1,276 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("proton");
+#include <vespa/vespalib/util/signalhandler.h>
+#include <vespa/vespalib/util/programoptions.h>
+#include <string>
+#include <iostream>
+#include <vespa/searchcore/proton/server/proton.h>
+#include <vespa/searchlib/util/statefile.h>
+#include <vespa/searchlib/util/sigbushandler.h>
+#include <vespa/searchlib/util/ioerrorhandler.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+typedef vespalib::SignalHandler SIG;
+
+struct Params
+{
+    std::string identity;
+    std::string serviceidentity;
+    uint64_t subscribeTimeout;
+};
+
+class App : public FastOS_Application
+{
+private:
+    void setupSignals();
+    Params parseParams();
+public:
+    int Main();
+};
+
+void
+App::setupSignals()
+{
+    SIG::PIPE.ignore();
+    SIG::INT.hook();
+    SIG::TERM.hook();
+}
+
+namespace
+{
+
+vespalib::string
+getStateString(search::StateFile &stateFile)
+{
+    std::vector<char> buf;
+    stateFile.readState(buf);
+    if (!buf.empty() && buf[buf.size() - 1] == '\n')
+        buf.resize(buf.size() - 1);
+    return vespalib::string(buf.begin(), buf.end());
+}
+
+bool stateIsDown(const vespalib::string &stateString)
+{
+    return strstr(stateString.c_str(), "state=down") != nullptr;
+}
+
+
+}
+
+Params
+App::parseParams()
+{
+    Params params;
+    vespalib::ProgramOptions parser(_argc, _argv);
+    parser.setSyntaxMessage("proton -- the nextgen search core");
+    parser.addOption("identity", params.identity, "Node identity and config id");
+    std::string empty("");
+    parser.addOption("serviceidentity", params.serviceidentity, empty, "Service node identity and config id");
+    parser.addOption("subscribeTimeout", params.subscribeTimeout, 600000UL, "Initial config subscribe timeout");
+    try {
+        parser.parse();
+    } catch (vespalib::InvalidCommandLineArgumentsException &e) {
+        parser.writeSyntaxPage(std::cerr);
+        throw;
+    }
+    return params;
+}
+
+
+#include "downpersistence.h"
+
+using storage::spi::PersistenceProvider;
+using storage::spi::DownPersistence;
+
+#include <vespa/storageserver/app/servicelayerprocess.h>
+
+class ProtonServiceLayerProcess : public storage::ServiceLayerProcess {
+    proton::Proton & _proton;
+    metrics::MetricManager* _metricManager;
+    PersistenceProvider *_downPersistence;
+
+public:
+    ProtonServiceLayerProcess(const config::ConfigUri & configUri,
+                              proton::Proton & proton,
+                              PersistenceProvider *downPersistence);
+    ~ProtonServiceLayerProcess() { shutdown(); }
+
+    virtual void shutdown();
+    virtual void setupProvider();
+    virtual storage::spi::PersistenceProvider& getProvider();
+
+    void setMetricManager(metrics::MetricManager& mm) {
+        // The service layer will call init(...) and stop() on the metric
+        // manager provided. Current design is that rather than depending
+        // on every component properly unregistering metrics and update
+        // hooks, the service layer stops metric manager ahead of shutting
+        // down component.
+        _metricManager = &mm;
+    }
+};
+
+ProtonServiceLayerProcess::ProtonServiceLayerProcess(const config::ConfigUri &
+                                                     configUri,
+                                                     proton::Proton & proton,
+                                                     PersistenceProvider *
+                                                     downPersistence)
+    : ServiceLayerProcess(configUri),
+      _proton(proton),
+      _metricManager(0),
+      _downPersistence(downPersistence)
+{
+    if (!downPersistence) {
+        setMetricManager(_proton.getMetricManager());
+    }
+}
+
+void
+ProtonServiceLayerProcess::shutdown()
+{
+    ServiceLayerProcess::shutdown();
+}
+
+void
+ProtonServiceLayerProcess::setupProvider()
+{
+    if (_metricManager != 0) {
+        _context.getComponentRegister().setMetricManager(*_metricManager);
+    }
+}
+
+storage::spi::PersistenceProvider &
+ProtonServiceLayerProcess::getProvider()
+{
+    return _downPersistence ? *_downPersistence : _proton.getPersistence();
+}
+
+int
+App::Main()
+{
+    proton::Proton::UP protonUP;
+    try {
+        setupSignals();
+        Params params = parseParams();
+        LOG(debug, "identity: '%s'", params.identity.c_str());
+        LOG(debug, "serviceidentity: '%s'", params.serviceidentity.c_str());
+        LOG(debug, "subscribeTimeout: '%" PRIu64 "'", params.subscribeTimeout);
+        std::unique_ptr<search::StateFile> stateFile;
+        std::unique_ptr<search::SigBusHandler> sigBusHandler;
+        std::unique_ptr<search::IOErrorHandler> ioErrorHandler;
+        protonUP = std::make_unique<proton::Proton>(params.identity, _argc > 0 ? _argv[0] : "proton", params.subscribeTimeout);
+        proton::Proton & proton = *protonUP;
+        proton::BootstrapConfig::SP configSnapshot = proton.init();
+        if (proton.hasAbortedInit()) {
+            EV_STOPPING("proton", "shutdown after aborted init");
+        } else {
+            const ProtonConfig &protonConfig =
+                configSnapshot->getProtonConfig();
+            vespalib::string basedir = protonConfig.basedir;
+            bool stopOnIOErrors = protonConfig.stoponioerrors;
+            vespalib::mkdir(basedir, true);
+            // TODO: Test that we can write to new file in directory
+            stateFile.reset(new search::StateFile(basedir + "/state"));
+            int stateGen = stateFile->getGen();
+            vespalib::string stateString = getStateString(*stateFile);
+            std::unique_ptr<PersistenceProvider> downPersistence;
+            if (stateIsDown(stateString)) {
+                LOG(error, "proton state string is %s", stateString.c_str());
+                if (stopOnIOErrors) {
+                    if ( !params.serviceidentity.empty()) {
+                        downPersistence.reset(
+                                new DownPersistence("proton state string is " +
+                                                    stateString));
+                    } else {
+                        LOG(info, "Sleeping 900 seconds due to proton state");
+                        int sleepLeft = 900;
+                        while (!(SIG::INT.check() || SIG::TERM.check()) &&
+                               sleepLeft > 0) {
+                            FastOS_Thread::Sleep(1000);
+                            --sleepLeft;
+                        }
+                        EV_STOPPING("proton",
+                                    "shutdown after stop on io errors");
+                        return 1;
+                    }
+                }
+            }
+            sigBusHandler.reset(new search::SigBusHandler(stateFile.get()));
+            ioErrorHandler.reset(new search::IOErrorHandler(stateFile.get()));
+            if (!downPersistence) {
+                proton.init(configSnapshot);
+            }
+            configSnapshot.reset();
+            std::unique_ptr<ProtonServiceLayerProcess> spiProton;
+            if ( ! params.serviceidentity.empty()) {
+                spiProton.reset(new ProtonServiceLayerProcess(params.serviceidentity, proton, downPersistence.get()));
+                spiProton->setupConfig(params.subscribeTimeout);
+                spiProton->createNode();
+                EV_STARTED("servicelayer");
+            } else {
+                proton.getMetricManager().init(params.identity, proton.getThreadPool());
+            }
+            EV_STARTED("proton");
+            while (!(SIG::INT.check() || SIG::TERM.check())) {
+                FastOS_Thread::Sleep(1000);
+                if (spiProton.get() && spiProton->configUpdated()) {
+                    storage::ResumeGuard guard(spiProton->getNode().pause());
+                    spiProton->updateConfig();
+                }
+                if (stateGen != stateFile->getGen()) {
+                    stateGen = stateFile->getGen();
+                    stateString = getStateString(*stateFile);
+                    if (stateIsDown(stateString)) {
+                        LOG(error, "proton state string is %s",
+                            stateString.c_str());
+                        if (stopOnIOErrors) {
+                            if (spiProton) {
+                                // report down state to cluster controller.
+                                spiProton->getNode().
+                                    notifyPartitionDown(0,
+                                                        "proton state "
+                                                        "string is " +
+                                                        stateString);
+                                FastOS_Thread::Sleep(1000);
+                            }
+                            EV_STOPPING("proton",
+                                        "shutdown after new stop on io errors");
+                            return 1;
+                        }
+                    }
+                }
+            }
+            if (spiProton.get()) {
+                spiProton->getNode().requestShutdown("controlled shutdown");
+                spiProton->shutdown();
+                EV_STOPPING("servicelayer", "clean shutdown");
+            }
+            EV_STOPPING("proton", "clean shutdown");
+        }
+    } catch (const vespalib::InvalidCommandLineArgumentsException &e) {
+        LOG(warning, "Invalid commandline arguments: '%s'", e.what());
+        return 1;
+    } catch (const config::ConfigTimeoutException &e) {
+        LOG(warning, "Error subscribing to initial config: '%s'", e.what());
+        return 1;
+    } catch (const vespalib::PortListenException &e) {
+        LOG(warning, "Failed listening to a network port(%d) with protocol(%s): '%s'",
+                   e.get_port(), e.get_protocol().c_str(), e.what());
+        return 1;
+    } catch (const vespalib::NetworkSetupFailureException & e) {
+        LOG(warning, "Network failure: '%s'", e.what());
+        return 1;
+    } catch (const vespalib::IllegalStateException & e) {
+        LOG(error, "Unknown IllegalStateException: '%s'", e.what());
+        throw;
+    }
+    protonUP.reset();
+    LOG(debug, "Fully stopped, all destructors run.)");
+    return 0;
+}
+
+int main(int argc, char **argv) {
+    App app;
+    return app.Entry(argc, argv);
+}
diff --git a/searchcore/src/apps/tests/.gitignore b/searchcore/src/apps/tests/.gitignore
new file mode 100644
index 00000000000..f32aa1d49cb
--- /dev/null
+++ b/searchcore/src/apps/tests/.gitignore
@@ -0,0 +1,2 @@
+/persistenceconformance_test
+searchcore_persistenceconformance_test_app
diff --git a/searchcore/src/apps/tests/CMakeLists.txt b/searchcore/src/apps/tests/CMakeLists.txt
new file mode 100644
index 00000000000..307ae51f7f6
--- /dev/null
+++ b/searchcore/src/apps/tests/CMakeLists.txt
@@ -0,0 +1,25 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_persistenceconformance_test_app
+    SOURCES
+    persistenceconformance_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_initializer
+    searchcore_reprocessing
+    searchcore_index
+    searchcore_persistenceengine
+    searchcore_docsummary
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_flushengine
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_fconfig
+    searchcore_util
+    vdstestlib
+)
+vespa_add_test(NAME searchcore_persistenceconformance_test_app COMMAND searchcore_persistenceconformance_test_app)
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp
new file mode 100644
index 00000000000..576b283572b
--- /dev/null
+++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp
@@ -0,0 +1,588 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("persistenceconformance_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/document/base/testdocman.h>
+#include <vespa/persistence/conformancetest/conformancetest.h>
+#include <vespa/searchcommon/common/schemaconfigurer.h>
+#include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h>
+#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h>
+#include <vespa/searchcore/proton/server/document_db_maintenance_config.h>
+#include <vespa/searchcore/proton/server/documentdb.h>
+#include <vespa/searchcore/proton/server/documentdbconfigmanager.h>
+#include <vespa/searchcore/proton/server/memoryconfigstore.h>
+#include <vespa/searchcore/proton/metrics/metricswireservice.h>
+#include <vespa/searchcore/proton/server/persistencehandlerproxy.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/transactionlog/translogserver.h>
+#include <tests/proton/common/dummydbowner.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+using namespace config;
+using namespace proton;
+using namespace vespa::config::search;
+using namespace vespa::config::search::core;
+using namespace vespa::config::search::summary;
+
+using std::shared_ptr;
+using document::DocumentType;
+using document::DocumentTypeRepo;
+using document::DocumenttypesConfig;
+using document::TestDocMan;
+using search::TuneFileDocumentDB;
+using search::index::DummyFileHeaderContext;
+using search::index::Schema;
+using search::index::SchemaBuilder;
+using search::transactionlog::TransLogServer;
+using storage::spi::ConformanceTest;
+using storage::spi::PersistenceProvider;
+
+typedef ConformanceTest::PersistenceFactory         PersistenceFactory;
+typedef DocumentDBConfig::DocumenttypesConfigSP     DocumenttypesConfigSP;
+typedef std::map<DocTypeName, DocumentDB::SP>       DocumentDBMap;
+typedef std::vector<DocTypeName>                    DocTypeVector;
+
+void
+storeDocType(DocTypeVector *types, const DocumentType &type)
+{
+    types->push_back(DocTypeName(type.getName()));
+}
+
+
+struct SchemaConfigFactory {
+    typedef DocumentDBConfig CS;
+    typedef std::shared_ptr<SchemaConfigFactory> SP;
+    virtual ~SchemaConfigFactory() {}
+    static SchemaConfigFactory::SP get() { return SchemaConfigFactory::SP(new SchemaConfigFactory()); }
+    virtual CS::IndexschemaConfigSP createIndexSchema(const DocumentType &docType) {
+        (void) docType;
+        return CS::IndexschemaConfigSP(new IndexschemaConfig());
+    }
+    virtual CS::AttributesConfigSP createAttributes(const DocumentType &docType) {
+        (void) docType;
+        return CS::AttributesConfigSP(new AttributesConfig());
+    }
+    virtual CS::SummaryConfigSP createSummary(const DocumentType &docType) {
+        (void) docType;
+        return CS::SummaryConfigSP(new SummaryConfig());
+    }
+};
+
+class ConfigFactory {
+private:
+    DocumentTypeRepo::SP    _repo;
+    DocumenttypesConfigSP   _typeCfg;
+    SchemaConfigFactory::SP _schemaFactory;
+
+public:
+    ConfigFactory(const DocumentTypeRepo::SP &repo,
+                  const DocumenttypesConfigSP &typeCfg,
+                  const SchemaConfigFactory::SP &schemaFactory) :
+        _repo(repo),
+        _typeCfg(typeCfg),
+        _schemaFactory(schemaFactory)
+    {
+    }
+    const DocumentTypeRepo::SP getTypeRepo() const { return _repo; }
+    const DocumenttypesConfigSP getTypeCfg() const { return _typeCfg; }
+    DocTypeVector getDocTypes() const {
+        DocTypeVector types;
+        _repo->forEachDocumentType(*makeClosure(storeDocType, &types));
+        return types;
+    }
+    DocumentDBConfig::SP create(const DocTypeName &docTypeName) const {
+        const DocumentType *docType =
+            _repo->getDocumentType(docTypeName.getName());
+        if (docType == NULL) {
+            return DocumentDBConfig::SP();
+        }
+        typedef DocumentDBConfig CS;
+        CS::RankProfilesConfigSP rankProfiles(new RankProfilesConfig());
+        CS::IndexschemaConfigSP indexschema = _schemaFactory->createIndexSchema(*docType);
+        CS::AttributesConfigSP attributes = _schemaFactory->createAttributes(*docType);
+        CS::SummaryConfigSP summary = _schemaFactory->createSummary(*docType);
+        CS::SummarymapConfigSP summarymap(new SummarymapConfig());
+        CS::JuniperrcConfigSP juniperrc(new JuniperrcConfig());
+        CS::MaintenanceConfigSP maintenance(new DocumentDBMaintenanceConfig);
+        TuneFileDocumentDB::SP tuneFileDocDB(new TuneFileDocumentDB());
+        Schema::SP schema(new Schema());
+        SchemaBuilder::build(*indexschema, *schema);
+        SchemaBuilder::build(*attributes, *schema);
+        SchemaBuilder::build(*summary, *schema);
+        return DocumentDBConfig::SP(new DocumentDBConfig(
+                        1,
+                        rankProfiles,
+                        indexschema,
+                        attributes,
+                        summary,
+                        summarymap,
+                        juniperrc,
+                        _typeCfg,
+                        _repo,
+                        tuneFileDocDB,
+                        schema,
+                        maintenance,
+                        "client",
+                        docTypeName.getName()));
+    }
+};
+
+class DocumentDBFactory : public DummyDBOwner {
+private:
+    vespalib::string          _baseDir;
+    DummyFileHeaderContext    _fileHeaderContext;
+    TransLogServer            _tls;
+    vespalib::string          _tlsSpec;
+    matching::QueryLimiter    _queryLimiter;
+    vespalib::Clock           _clock;
+    mutable DummyWireService  _metricsWireService;
+    mutable MemoryConfigStores _config_stores;
+    vespalib::ThreadStackExecutor _summaryExecutor;
+
+public:
+    DocumentDBFactory(const vespalib::string &baseDir, int tlsListenPort) :
+        _baseDir(baseDir),
+        _fileHeaderContext(),
+        _tls("tls", tlsListenPort, baseDir, _fileHeaderContext),
+        _tlsSpec(vespalib::make_string("tcp/localhost:%d", tlsListenPort)),
+        _queryLimiter(),
+        _clock(),
+        _metricsWireService(),
+        _summaryExecutor(8, 128 * 1024)
+    {
+    }
+    DocumentDB::SP create(const DocTypeName &docType,
+                          const ConfigFactory &factory) {
+        DocumentDBConfig::SP snapshot = factory.create(docType);
+        vespalib::mkdir(_baseDir, false);
+        vespalib::mkdir(_baseDir + "/" + docType.toString(), false);
+        vespalib::string inputCfg = _baseDir + "/" + docType.toString() + "/baseconfig";
+        {
+            FileConfigManager fileCfg(inputCfg, "", docType.getName());
+            fileCfg.saveConfig(*snapshot, Schema(), 1);
+        }
+        config::DirSpec spec(inputCfg + "/config-1");
+        TuneFileDocumentDB::SP tuneFileDocDB(new TuneFileDocumentDB());
+        DocumentDBConfigHelper mgr(spec, docType.getName());
+        BootstrapConfig::SP b(new BootstrapConfig(1,
+                                                  factory.getTypeCfg(),
+                                                  factory.getTypeRepo(),
+                                                  BootstrapConfig::ProtonConfigSP(new ProtonConfig()),
+                                                  tuneFileDocDB));
+        mgr.forwardConfig(b);
+        mgr.nextGeneration(0);
+        return DocumentDB::SP(
+                new DocumentDB(_baseDir,
+                               mgr.getConfig(),
+                               _tlsSpec,
+                               _queryLimiter,
+                               _clock,
+                               docType,
+                               ProtonConfig(),
+                               const_cast<DocumentDBFactory &>(*this),
+                               _summaryExecutor,
+                               _summaryExecutor,
+                               NULL,
+                               _metricsWireService,
+                               _fileHeaderContext,
+                               _config_stores.getConfigStore(
+                                       docType.toString()),
+                               std::make_shared<vespalib::ThreadStackExecutor>
+                               (16, 128 * 1024)));
+    }
+};
+
+
+class DocumentDBRepo {
+private:
+    DocumentDBMap _docDbs;
+public:
+    typedef std::unique_ptr<DocumentDBRepo> UP;
+    DocumentDBRepo(const ConfigFactory &cfgFactory,
+                   DocumentDBFactory &docDbFactory) :
+        _docDbs()
+    {
+        DocTypeVector types = cfgFactory.getDocTypes();
+        for (size_t i = 0; i < types.size(); ++i) {
+            DocumentDB::SP docDb = docDbFactory.create(types[i],
+                                                       cfgFactory);
+            docDb->start();
+            docDb->waitForOnlineState();
+            _docDbs[types[i]] = docDb;
+        }
+    }
+
+    void
+    close(void)
+    {
+        for (DocumentDBMap::iterator itr = _docDbs.begin();
+             itr != _docDbs.end(); ++itr) {
+            itr->second->close();
+        }
+    }
+
+    ~DocumentDBRepo()
+    {
+        close();
+    }
+    const DocumentDBMap &getDocDbs() const { return _docDbs; }
+};
+
+
+class DocDBRepoHolder
+{
+protected:
+    DocumentDBRepo::UP      _docDbRepo;
+
+    DocDBRepoHolder(DocumentDBRepo::UP docDbRepo)
+        : _docDbRepo(std::move(docDbRepo))
+    {
+    }
+
+    virtual
+    ~DocDBRepoHolder(void)
+    {
+    }
+
+    void
+    close(void)
+    {
+        if (_docDbRepo.get() != NULL)
+            _docDbRepo->close();
+    }
+};
+
+
+class MyPersistenceEngineOwner : public IPersistenceEngineOwner
+{
+    virtual void
+    setClusterState(const storage::spi::ClusterState &calc)
+    {
+        (void) calc;
+    }
+};
+
+struct MyResourceWriteFilter : public IResourceWriteFilter
+{
+    virtual bool acceptWriteOperation() const override { return true; }
+    virtual State getAcceptState() const override { return IResourceWriteFilter::State(); }
+};
+
+class MyPersistenceEngine : public DocDBRepoHolder,
+                            public PersistenceEngine
+{
+public:
+    MyPersistenceEngine(MyPersistenceEngineOwner &owner,
+                        MyResourceWriteFilter &writeFilter,
+                        DocumentDBRepo::UP docDbRepo,
+                        const vespalib::string &docType = "")
+        : DocDBRepoHolder(std::move(docDbRepo)),
+          PersistenceEngine(owner, writeFilter, -1, false)
+    {
+        addHandlers(docType);
+    }
+
+    void
+    addHandlers(const vespalib::string &docType)
+    {
+        if (_docDbRepo.get() == NULL)
+            return;
+        const DocumentDBMap &docDbs = _docDbRepo->getDocDbs();
+        for (DocumentDBMap::const_iterator itr = docDbs.begin();
+             itr != docDbs.end(); ++itr) {
+            if (!docType.empty() && docType != itr->first.getName()) {
+                continue;
+            }
+            LOG(info, "putHandler(%s)", itr->first.toString().c_str());
+            IPersistenceHandler::SP proxy(
+                    new PersistenceHandlerProxy(itr->second));
+            putHandler(itr->first, proxy);
+        }
+    }
+
+    void
+    removeHandlers(void)
+    {
+        if (_docDbRepo.get() == NULL)
+            return;
+        const DocumentDBMap &docDbs = _docDbRepo->getDocDbs();
+        for (DocumentDBMap::const_iterator itr = docDbs.begin();
+             itr != docDbs.end(); ++itr) {
+            IPersistenceHandler::SP proxy(removeHandler(itr->first));
+        }
+    }
+
+    virtual
+    ~MyPersistenceEngine(void)
+    {
+        destroyIterators();
+        removeHandlers(); // Block calls to document db from engine
+        close();	  // Block upcalls to engine from document db
+    }
+};
+
+class MyPersistenceFactory : public PersistenceFactory {
+private:
+    vespalib::string          _baseDir;
+    DocumentDBFactory       _docDbFactory;
+    SchemaConfigFactory::SP _schemaFactory;
+    DocumentDBRepo::UP      _docDbRepo;
+    vespalib::string        _docType;
+    MyPersistenceEngineOwner _engineOwner;
+    MyResourceWriteFilter    _writeFilter;
+public:
+    MyPersistenceFactory(const vespalib::string &baseDir, int tlsListenPort,
+                         const SchemaConfigFactory::SP &schemaFactory,
+                         const vespalib::string &docType = "")
+        : _baseDir(baseDir),
+          _docDbFactory(baseDir, tlsListenPort),
+          _schemaFactory(schemaFactory),
+          _docDbRepo(),
+          _docType(docType),
+          _engineOwner(),
+          _writeFilter()
+    {
+    }
+    virtual PersistenceProvider::UP getPersistenceImplementation(const DocumentTypeRepo::SP &repo,
+                                                                 const DocumenttypesConfig &typesCfg) {
+        ConfigFactory cfgFactory(repo, DocumenttypesConfigSP(new DocumenttypesConfig(typesCfg)), _schemaFactory);
+        _docDbRepo.reset(new DocumentDBRepo(cfgFactory, _docDbFactory));
+        PersistenceEngine::UP engine(new MyPersistenceEngine(_engineOwner,
+                                                             _writeFilter,
+                                                             std::move(_docDbRepo),
+                                                             _docType));
+        assert(_docDbRepo.get() == NULL); // Repo should be handed over
+        return PersistenceProvider::UP(engine.release());
+    }
+
+    virtual void clear() {
+        FastOS_FileInterface::EmptyAndRemoveDirectory(_baseDir.c_str());
+    }
+
+    virtual bool hasPersistence() const { return true; }
+    virtual bool supportsActiveState() const { return true; }
+};
+
+
+struct FixtureBaseBase
+{
+    FixtureBaseBase(void)
+    {
+        FastOS_FileInterface::EmptyAndRemoveDirectory("testdb");
+    }
+
+    ~FixtureBaseBase(void)
+    {
+        FastOS_FileInterface::EmptyAndRemoveDirectory("testdb");
+    }
+};
+
+
+struct FixtureBase : public FixtureBaseBase
+{
+    ConformanceTest test;
+    FixtureBase(const SchemaConfigFactory::SP &schemaFactory,
+                const vespalib::string &docType = "")
+        : FixtureBaseBase(),
+          test(PersistenceFactory::UP(new MyPersistenceFactory("testdb", 9017,
+                                                               schemaFactory,
+                                                               docType)))
+    {
+    }
+    ~FixtureBase(void)
+    {
+    }
+};
+
+
+struct TestFixture : public FixtureBase {
+    TestFixture() : FixtureBase(SchemaConfigFactory::get()) {}
+};
+
+
+struct SingleDocTypeFixture : public FixtureBase {
+    SingleDocTypeFixture() : FixtureBase(SchemaConfigFactory::get(), "testdoctype1") {}
+};
+
+TEST_F("require that testBucketActivation() works", TestFixture)
+{
+    f.test.testBucketActivation();
+}
+
+TEST_F("require that testListBuckets() works", TestFixture)
+{
+    f.test.testListBuckets();
+}
+
+TEST_F("require that testBucketInfo() works", TestFixture)
+{
+    f.test.testBucketInfo();
+}
+
+TEST_F("require that testPut() works", TestFixture)
+{
+    f.test.testPut();
+}
+
+TEST_F("require that testRemove() works", TestFixture)
+{
+    f.test.testRemove();
+}
+
+TEST_F("require that testRemoveMerge() works", TestFixture)
+{
+    f.test.testRemoveMerge();
+}
+
+TEST_F("require that testUpdate() works", TestFixture)
+{
+    f.test.testUpdate();
+}
+
+TEST_F("require that testGet() works", TestFixture)
+{
+    f.test.testGet();
+}
+
+TEST_F("require that testIterateCreateIterator() works", TestFixture)
+{
+    f.test.testIterateCreateIterator();
+}
+
+TEST_F("require that testIterateWithUnknownId() works", TestFixture)
+{
+    f.test.testIterateWithUnknownId();
+}
+
+TEST_F("require that testIterateDestroyIterator() works", TestFixture)
+{
+    f.test.testIterateDestroyIterator();
+}
+
+TEST_F("require that testIterateAllDocs() works", TestFixture)
+{
+    f.test.testIterateAllDocs();
+}
+
+TEST_F("require that testIterateChunked() works", TestFixture)
+{
+    f.test.testIterateChunked();
+}
+
+TEST_F("require that testMaxByteSize() works", TestFixture)
+{
+    f.test.testMaxByteSize();
+}
+
+TEST_F("require that testIterateMatchTimestampRange() works", TestFixture)
+{
+    f.test.testIterateMatchTimestampRange();
+}
+
+TEST_F("require that testIterateExplicitTimestampSubset() works", TestFixture)
+{
+    f.test.testIterateExplicitTimestampSubset();
+}
+
+TEST_F("require that testIterateRemoves() works", TestFixture)
+{
+    f.test.testIterateRemoves();
+}
+
+TEST_F("require that testIterateMatchSelection() works", TestFixture)
+{
+    f.test.testIterateMatchSelection();
+}
+
+TEST_F(
+  "require that testIterationRequiringDocumentIdOnlyMatching() works",
+  TestFixture)
+{
+    f.test.testIterationRequiringDocumentIdOnlyMatching();
+}
+
+TEST_F("require that testIterateBadDocumentSelection() works", TestFixture)
+{
+    f.test.testIterateBadDocumentSelection();
+}
+
+TEST_F("require that testIterateAlreadyCompleted() works", TestFixture)
+{
+    f.test.testIterateAlreadyCompleted();
+}
+
+TEST_F("require that testIterateEmptyBucket() works", TestFixture)
+{
+    f.test.testIterateEmptyBucket();
+}
+
+TEST_F("require that testDeleteBucket() works", TestFixture)
+{
+    f.test.testDeleteBucket();
+}
+
+TEST_F("require that testSplitNormalCase() works", TestFixture)
+{
+    f.test.testSplitNormalCase();
+}
+
+TEST_F("require that testSplitTargetExists() works", TestFixture)
+{
+    f.test.testSplitTargetExists();
+}
+
+TEST_F("require that testJoinNormalCase() works", TestFixture)
+{
+    f.test.testJoinNormalCase();
+}
+
+TEST_F("require that testJoinOneBucket() works", TestFixture)
+{
+    f.test.testJoinOneBucket();
+}
+
+TEST_F("require that testJoinTargetExists() works", TestFixture)
+{
+    f.test.testJoinTargetExists();
+}
+
+TEST_F("require that testBucketActivationSplitAndJoin() works", SingleDocTypeFixture)
+{
+    f.test.testBucketActivationSplitAndJoin();
+}
+
+TEST_F("require thant testJoinSameSourceBuckets() works", TestFixture)
+{
+    f.test.testJoinSameSourceBuckets();
+}
+
+TEST_F("require thant testJoinSameSourceBucketsTargetExists() works",
+       TestFixture)
+{
+    f.test.testJoinSameSourceBucketsTargetExists();
+}
+
+// *** Run all conformance tests, but ignore the results BEGIN ***
+
+#define CONVERT_TEST(testFunction)                                    \
+namespace ns_ ## testFunction {                                       \
+IGNORE_TEST_F(TEST_STR(testFunction) " (generated)", TestFixture()) { \
+    f.test.testFunction();                                            \
+}                                                                     \
+} // namespace testFunction
+
+#undef CPPUNIT_TEST
+#define CPPUNIT_TEST(testFunction) CONVERT_TEST(testFunction)
+#if 0
+DEFINE_CONFORMANCE_TESTS();
+#endif
+
+// *** Run all conformance tests, but ignore the results END ***
+
+TEST_MAIN()
+{
+    DummyFileHeaderContext::setCreator("persistenceconformance_test");
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/apps/verify_ranksetup/.gitignore b/searchcore/src/apps/verify_ranksetup/.gitignore
new file mode 100644
index 00000000000..269b313f1f0
--- /dev/null
+++ b/searchcore/src/apps/verify_ranksetup/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+verify_ranksetup
+verify_ranksetup-bin
diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
new file mode 100644
index 00000000000..c71d306db9a
--- /dev/null
+++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_verify_ranksetup_app
+    SOURCES
+    verify_ranksetup.cpp
+    OUTPUT_NAME verify_ranksetup-bin
+    INSTALL bin
+    DEPENDS
+    searchcore_matching
+    searchcore_documentmetastore
+)
diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
new file mode 100644
index 00000000000..0a02ac0832f
--- /dev/null
+++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("verify_ranksetup");
+#include <vespa/searchlib/fef/fef.h>
+#include <vespa/searchcommon/common/schemaconfigurer.h>
+#include <vespa/searchcore/proton/matching/indexenvironment.h>
+#include <vespa/searchlib/features/setup.h>
+#include <vespa/searchlib/fef/test/plugin/setup.h>
+#include <vespa/config/config.h>
+#include <vespa/config/helper/legacy.h>
+
+#include <vespa/config-rank-profiles.h>
+#include <vespa/config-indexschema.h>
+#include <vespa/config-attributes.h>
+
+using config::IConfigContext;
+using config::ConfigContext;
+using config::ConfigSubscriber;
+using config::ConfigHandle;
+using vespa::config::search::RankProfilesConfig;
+using vespa::config::search::IndexschemaConfig;
+using vespa::config::search::AttributesConfig;
+using config::ConfigRuntimeException;
+using config::InvalidConfigException;
+
+class App : public FastOS_Application
+{
+public:
+    bool verify(const search::index::Schema &schema,
+                const search::fef::Properties &props);
+
+    bool verifyConfig(const vespa::config::search::RankProfilesConfig &rankCfg,
+                      const vespa::config::search::IndexschemaConfig  &schemaCfg,
+                      const vespa::config::search::AttributesConfig   &attributeCfg);
+
+    int usage();
+    int Main();
+};
+
+bool
+App::verify(const search::index::Schema &schema,
+            const search::fef::Properties &props)
+{
+    proton::matching::IndexEnvironment indexEnv(schema, props);
+    search::fef::BlueprintFactory factory;
+    search::features::setup_search_features(factory);
+    search::fef::test::setup_fef_test_plugin(factory);
+
+    search::fef::RankSetup rankSetup(factory, indexEnv);
+    rankSetup.configure(); // reads config values from the property map
+    bool ok = true;
+    if (!rankSetup.getFirstPhaseRank().empty()) {
+        ok = verifyFeature(factory, indexEnv, rankSetup.getFirstPhaseRank(), "first phase ranking") && ok;
+    }
+    if (!rankSetup.getSecondPhaseRank().empty()) {
+        ok = verifyFeature(factory, indexEnv, rankSetup.getSecondPhaseRank(), "second phase ranking") && ok;
+    }
+    for (size_t i = 0; i < rankSetup.getSummaryFeatures().size(); ++i) {
+        ok = verifyFeature(factory, indexEnv, rankSetup.getSummaryFeatures()[i], "summary features") && ok;
+    }
+    for (size_t i = 0; i < rankSetup.getDumpFeatures().size(); ++i) {
+        ok = verifyFeature(factory, indexEnv, rankSetup.getDumpFeatures()[i], "dump features") && ok;
+    }
+    return ok;
+}
+
+bool
+App::verifyConfig(const vespa::config::search::RankProfilesConfig &rankCfg,
+                  const vespa::config::search::IndexschemaConfig  &schemaCfg,
+                  const vespa::config::search::AttributesConfig   &attributeCfg)
+{
+    bool ok = true;
+    search::index::Schema schema;
+    search::index::SchemaBuilder::build(schemaCfg, schema);
+    search::index::SchemaBuilder::build(attributeCfg, schema);
+
+    for(size_t i = 0; i < rankCfg.rankprofile.size(); i++) {
+        search::fef::Properties properties;
+        const vespa::config::search::RankProfilesConfig::Rankprofile &profile = rankCfg.rankprofile[i];
+        for(size_t j = 0; j < profile.fef.property.size(); j++) {
+            properties.add(profile.fef.property[j].name,
+                           profile.fef.property[j].value);
+        }
+        if (verify(schema, properties)) {
+            LOG(info, "rank profile '%s': pass", profile.name.c_str());
+        } else {
+            LOG(error, "rank profile '%s': FAIL", profile.name.c_str());
+            ok = false;
+        }
+    }
+    return ok;
+}
+
+int
+App::usage()
+{
+    fprintf(stderr, "Usage: verify_ranksetup <config-id>\n");
+    return 1;
+}
+
+int
+App::Main()
+{
+    if (_argc != 2) {
+        return usage();
+    }
+
+    const std::string configid = _argv[1];
+    LOG(debug, "verifying rank setup for config id '%s' ...",
+        configid.c_str());
+
+    bool ok = false;
+    try {
+        IConfigContext::SP ctx(new ConfigContext(*config::legacyConfigId2Spec(configid)));
+        vespalib::string cfgId(config::legacyConfigId2ConfigId(configid));
+        ConfigSubscriber subscriber(ctx);
+        ConfigHandle<RankProfilesConfig>::UP rankHandle = subscriber.subscribe<RankProfilesConfig>(cfgId);
+        ConfigHandle<AttributesConfig>::UP attributesHandle = subscriber.subscribe<AttributesConfig>(cfgId);
+        ConfigHandle<IndexschemaConfig>::UP schemaHandle = subscriber.subscribe<IndexschemaConfig>(cfgId);
+
+        subscriber.nextConfig();
+        ok = verifyConfig(*rankHandle->getConfig(), *schemaHandle->getConfig(), *attributesHandle->getConfig());
+    } catch (ConfigRuntimeException & e) {
+        LOG(error, "Unable to subscribe to config: %s", e.getMessage().c_str());
+    } catch (InvalidConfigException & e) {
+        LOG(error, "Error getting config: %s", e.getMessage().c_str());
+    }
+    if (!ok) {
+        return 1;
+    }
+    return 0;
+}
+
+int main(int argc, char **argv) {
+    App app;
+    return app.Entry(argc, argv);
+}
diff --git a/searchcore/src/apps/vespa-dump-feed/.gitignore b/searchcore/src/apps/vespa-dump-feed/.gitignore
new file mode 100644
index 00000000000..ea7ed317d25
--- /dev/null
+++ b/searchcore/src/apps/vespa-dump-feed/.gitignore
@@ -0,0 +1,4 @@
+/.depend
+/Makefile
+/vespa-dump-feed
+searchcore_vespa-dump-feed_app
diff --git a/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt b/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt
new file mode 100644
index 00000000000..85a704a7318
--- /dev/null
+++ b/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_vespa-dump-feed_app
+    SOURCES
+    vespa-dump-feed.cpp
+    INSTALL bin
+    DEPENDS
+)
+vespa_add_target_system_dependency(searchcore_vespa-dump-feed_app boost boost_system-mt-d)
+vespa_add_target_system_dependency(searchcore_vespa-dump-feed_app boost boost_filesystem-mt-d)
diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp
new file mode 100644
index 00000000000..00faadf709c
--- /dev/null
+++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp
@@ -0,0 +1,224 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/config/config.h>
+#include <vespa/config/print/fileconfigwriter.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/document.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/documentapi/documentapi.h>
+#include <vespa/messagebus/destinationsession.h>
+#include <vespa/messagebus/imessagehandler.h>
+#include <vespa/messagebus/iprotocol.h>
+#include <vespa/messagebus/message.h>
+#include <vespa/messagebus/protocolset.h>
+#include <vespa/messagebus/rpcmessagebus.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/signalhandler.h>
+#include <vespa/vespalib/util/slaveproc.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+#include <iostream>
+
+typedef vespalib::SignalHandler SIG;
+
+//-----------------------------------------------------------------------------
+
+class OutputFile
+{
+private:
+    FILE *file;
+
+public:
+    OutputFile(const std::string &name)
+        : file(fopen(name.c_str(), "w")) {}
+    bool valid() const { return (file != 0); }
+    void write(const char *data, size_t length) {
+        size_t res = fwrite(data, 1, length, file);
+        assert(res == length);
+        (void) res;
+    }
+    ~OutputFile() { fclose(file); }
+};
+
+//-----------------------------------------------------------------------------
+
+class FeedHandler : public mbus::IMessageHandler
+{
+private:
+    documentapi::LoadTypeSet     _loadTypes;
+    mbus::RPCMessageBus          _mbus;
+    mbus::DestinationSession::UP _session;
+    OutputFile                  &_idx;
+    OutputFile                  &_dat;
+    size_t                       _numDocs;
+
+    void handleDocumentPut(document::Document::SP doc);
+    virtual void handleMessage(mbus::Message::UP message);
+
+public:
+    FeedHandler(document::DocumentTypeRepo::SP repo, OutputFile &idx, OutputFile &dat);
+    std::string getRoute() { return _session->getConnectionSpec(); }
+    virtual ~FeedHandler();
+};
+
+void
+FeedHandler::handleDocumentPut(document::Document::SP doc)
+{
+    if (doc.get() != 0) {
+        vespalib::nbostream datStream(12345);
+        vespalib::nbostream idxStream(12);
+        doc->serialize(datStream);
+        idxStream << uint64_t(datStream.size());
+        _dat.write(datStream.peek(), datStream.size());
+        _idx.write(idxStream.peek(), idxStream.size());
+        ++_numDocs;
+    }
+}
+
+void
+FeedHandler::handleMessage(mbus::Message::UP message)
+{
+    mbus::Reply::UP reply;
+    documentapi::DocumentMessage::UP msg((documentapi::DocumentMessage*)message.release());
+    switch (msg->getType()) {
+    case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT:
+ 	handleDocumentPut(((documentapi::PutDocumentMessage&)(*msg)).getDocument());
+ 	break;
+    default:
+ 	break;
+    }
+    reply = msg->createReply(); // use default reply for all messages
+    msg->swapState(*reply);
+    _session->reply(std::move(reply)); // handle all messages synchronously
+}
+
+FeedHandler::FeedHandler(document::DocumentTypeRepo::SP repo, OutputFile &idx, OutputFile &dat)
+    : _loadTypes(),
+      _mbus(mbus::MessageBusParams().addProtocol(mbus::IProtocol::SP(new documentapi::DocumentProtocol(_loadTypes, repo))),
+            mbus::RPCNetworkParams()),
+      _session(_mbus.getMessageBus()
+               .createDestinationSession(mbus::DestinationSessionParams()
+                                         .setBroadcastName(false)
+                                         .setMessageHandler(*this)
+                                         .setName("dump-feed"))),
+      _idx(idx),
+      _dat(dat),
+      _numDocs()
+{
+}
+
+FeedHandler::~FeedHandler()
+{
+    _session.reset();
+    fprintf(stderr, "%zu document puts dumped to disk\n", _numDocs);
+}
+
+//-----------------------------------------------------------------------------
+
+class App : public FastOS_Application
+{
+public:
+    virtual bool useProcessStarter() const { return true; }
+    virtual int Main();
+};
+
+template <typename CFG>
+bool writeConfig(std::unique_ptr<CFG> cfg, const std::string &dirName) {
+    if (cfg.get() == 0) {
+        return false;
+    }
+    std::string fileName = dirName + "/" + CFG::CONFIG_DEF_NAME + ".cfg";
+    try {
+        config::FileConfigWriter w(fileName);
+        return w.write(*cfg);
+    } catch (config::ConfigWriteException & e) {
+        fprintf(stderr, "Unable to write config to disk: %s\n", e.what());
+    }
+    return false;
+}
+
+template <typename CFG>
+std::unique_ptr<CFG> getConfig() {
+    std::unique_ptr<CFG> ret(config::ConfigGetter<CFG>::getConfig("client"));
+    if (ret.get() == 0) {
+        fprintf(stderr, "error: could not obtain config (%s)\n", CFG::CONFIG_DEF_NAME.c_str());
+    }
+    return ret;
+}
+
+document::DocumentTypeRepo::SP getRepo() {
+    typedef document::DocumenttypesConfig DCFG;
+    std::unique_ptr<DCFG> dcfg = getConfig<DCFG>();
+    document::DocumentTypeRepo::SP ret;
+    if (dcfg.get() != 0) {
+        ret.reset(new document::DocumentTypeRepo(*dcfg));
+    }
+    return ret;
+}
+
+void setupSignals() {
+    SIG::PIPE.ignore();
+}
+
+int usage() {
+    fprintf(stderr, "Usage: vespa-dump-feed <input-feed> <output-directory>\n\n");
+    fprintf(stderr, "  Takes an XML vespa feed as input and dumps its contents as serialized documents.\n");
+    fprintf(stderr, "  In addition to the actual documents, an index file containing document sizes\n");
+    fprintf(stderr, "  and the appropriate config file(s) needed for deserialization are also stored.\n");
+    fprintf(stderr, "  This utility can be run anywhere vespafeeder can be run with default config id.\n");
+    return 1;
+}
+
+int
+App::Main()
+{
+    setupSignals();
+    if (_argc != 3) {
+        return usage();
+    }
+    std::string feedFile = _argv[1];
+    std::string dirName = _argv[2];
+    fprintf(stderr, "input feed: %s\n", feedFile.c_str());
+    fprintf(stderr, "output directory: %s\n", dirName.c_str());
+    vespalib::mkdir(dirName);
+    typedef document::DocumenttypesConfig DCFG;
+    if (!writeConfig(getConfig<DCFG>(), dirName)) {
+        fprintf(stderr, "error: could not save config to disk\n");
+        return 1;
+    }
+    document::DocumentTypeRepo::SP repo = getRepo();
+    if (repo.get() == 0) {
+        fprintf(stderr, "error: could not create document type repo\n");
+        return 1;
+    }
+    {
+        OutputFile idxFile(dirName + "/doc.idx");
+        OutputFile datFile(dirName + "/doc.dat");
+        if (!idxFile.valid() || !datFile.valid()) {
+            fprintf(stderr, "error: could not open output document files\n");
+            return 1;
+        }
+        FeedHandler feedHooks(repo, idxFile, datFile);
+        std::string route = feedHooks.getRoute();
+        fprintf(stderr, "route to self: %s\n", route.c_str());
+        std::string feedCmd(vespalib::make_string("vespafeeder --route \"%s\" %s",
+                                                  route.c_str(), feedFile.c_str()));
+        fprintf(stderr, "running feed command: %s\n", feedCmd.c_str());
+        std::string feederOutput;
+        bool feedingOk = vespalib::SlaveProc::run(feedCmd.c_str(), feederOutput);
+        if (!feedingOk) {
+            fprintf(stderr, "error: feed command failed\n");
+            fprintf(stderr, "feed command output:\n-----\n%s\n-----\n", feederOutput.c_str());
+            return 1;
+        }
+    }
+    return 0;
+}
+
+//-----------------------------------------------------------------------------
+
+int main(int argc, char **argv) {
+    App app;
+    return app.Entry(argc, argv);
+}
diff --git a/searchcore/src/apps/vespa-gen-testdocs/.gitignore b/searchcore/src/apps/vespa-gen-testdocs/.gitignore
new file mode 100644
index 00000000000..f24c9c78333
--- /dev/null
+++ b/searchcore/src/apps/vespa-gen-testdocs/.gitignore
@@ -0,0 +1,2 @@
+/vespa-gen-testdocs
+searchcore_vespa-gen-testdocs_app
diff --git a/searchcore/src/apps/vespa-gen-testdocs/CMakeLists.txt b/searchcore/src/apps/vespa-gen-testdocs/CMakeLists.txt
new file mode 100644
index 00000000000..2aafdd76ae9
--- /dev/null
+++ b/searchcore/src/apps/vespa-gen-testdocs/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_vespa-gen-testdocs_app
+    SOURCES
+    vespa-gen-testdocs.cpp
+    OUTPUT_NAME vespa-gen-testdocs
+    INSTALL bin
+    DEPENDS
+)
+vespa_add_target_system_dependency(searchcore_vespa-gen-testdocs_app crypto)
diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp
new file mode 100644
index 00000000000..7cbe90151ef
--- /dev/null
+++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp
@@ -0,0 +1,961 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("vespa-gen-testdocs");
+#include <algorithm>
+#include <string>
+#include <vespa/searchlib/util/rand48.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/fastlib/io/bufferedfile.h>
+#include <iostream>
+#include <openssl/sha.h>
+
+typedef vespalib::hash_set<vespalib::string> StringSet;
+typedef vespalib::hash_set<uint32_t> UIntSet;
+typedef std::vector<vespalib::string> StringArray;
+typedef std::shared_ptr<StringArray> StringArraySP;
+
+void
+usageHeader(void)
+{
+    using std::cerr;
+    cerr <<
+        "vespa-gen-testdocs version 0.0\n"
+        "\n"
+        "USAGE:\n";
+}
+
+vespalib::string
+prependBaseDir(const vespalib::string &baseDir,
+               const vespalib::string &file)
+{
+    if (baseDir.empty() || baseDir == ".")
+        return file;
+    return baseDir + "/" + file;
+}
+
+
+void
+shafile(const vespalib::string &baseDir,
+        const vespalib::string &file)
+{
+    unsigned char digest[SHA256_DIGEST_LENGTH];
+    SHA256_CTX c; 
+    vespalib::string fullFile(prependBaseDir(baseDir, file));
+    FastOS_File f;
+    std::ostringstream os;
+    vespalib::AlignedHeapAlloc buf(65536, 4096);
+    f.EnableDirectIO();
+    bool openres = f.OpenReadOnly(fullFile.c_str());
+    if (!openres) {
+        LOG(error, "Could not open %s for sha256 checksum", fullFile.c_str());
+        abort();
+    }
+    int64_t flen = f.GetSize();
+    int64_t remainder = flen;
+    SHA256_Init(&c);
+    while (remainder > 0) {
+        int64_t thistime =
+            std::min(remainder, static_cast<int64_t>(buf.size()));
+        f.ReadBuf(buf.get(), thistime);
+        SHA256_Update(&c, buf.get(), thistime);
+        remainder -= thistime;
+    }
+    f.Close();
+    SHA256_Final(digest, &c);
+    for (unsigned int i = 0; i < SHA256_DIGEST_LENGTH; ++i) {
+        os.width(2);
+        os.fill('0');
+        os << std::hex << static_cast<unsigned int>(digest[i]);
+    }
+    LOG(info,
+        "SHA256(%s)= %s",
+        file.c_str(),
+        os.str().c_str());
+}
+
+class StringGenerator
+{
+    search::Rand48 &_rnd;
+
+public:
+    StringGenerator(search::Rand48 &rnd);
+
+    void
+    rand_string(vespalib::string &res, uint32_t minLen, uint32_t maxLen);
+
+    void
+    rand_unique_array(StringArray &res,
+                      uint32_t minLen,
+                      uint32_t maxLen,
+                      uint32_t size);
+};
+
+
+StringGenerator::StringGenerator(search::Rand48 &rnd)
+    : _rnd(rnd)
+{
+}
+
+
+void
+StringGenerator::rand_string(vespalib::string &res,
+                             uint32_t minLen,
+                             uint32_t maxLen)
+{
+    uint32_t len = minLen + _rnd.lrand48() % (maxLen - minLen + 1);
+    
+    res.clear();
+    for (uint32_t i = 0; i < len; ++i) {
+        res.append('a' + _rnd.lrand48() % ('z' - 'a' + 1));
+    }
+}
+    
+
+void
+StringGenerator::rand_unique_array(StringArray &res,
+                                   uint32_t minLen,
+                                   uint32_t maxLen,
+                                   uint32_t size)
+{
+    StringSet set(size * 2);
+    vespalib::string s;
+    
+    res.reserve(size);
+    for (uint32_t i = 0; i < size; ++i) {
+        do {
+            rand_string(s, minLen, maxLen);
+        } while (!set.insert(s).second);
+        assert(s.size() > 0);
+        res.push_back(s);
+    }
+}
+
+
+class FieldGenerator
+{
+public:
+    typedef std::shared_ptr<FieldGenerator> SP;
+
+protected:
+    const vespalib::string _name;
+
+public:
+    FieldGenerator(const vespalib::string &name);
+
+    virtual
+    ~FieldGenerator(void);
+
+    virtual void
+    setup(void);
+
+    virtual void
+    clear(void);
+
+    virtual void
+    deleteHistogram(const vespalib::string &baseDir,
+                    const vespalib::string &name);
+
+    virtual void
+    writeHistogram(const vespalib::string &baseDir,
+                   const vespalib::string &name);
+
+    virtual void
+    generate(vespalib::asciistream &doc, uint32_t id) = 0;
+};
+
+
+
+FieldGenerator::FieldGenerator(const vespalib::string &name)
+    : _name(name)
+{
+}
+
+FieldGenerator::~FieldGenerator(void)
+{
+}
+
+
+void
+FieldGenerator::setup(void)
+{
+}
+
+
+void
+FieldGenerator::clear(void)
+{
+}
+
+
+void
+FieldGenerator::deleteHistogram(const vespalib::string &baseDir,
+                                const vespalib::string &name)
+{
+    (void) baseDir;
+    (void) name;
+}
+
+
+void
+FieldGenerator::writeHistogram(const vespalib::string &baseDir,
+                               const vespalib::string &name)
+{
+    (void) baseDir;
+    (void) name;
+}
+
+
+class RandTextFieldGenerator : public FieldGenerator
+{
+    search::Rand48 &_rnd;
+    uint32_t _numWords;
+    StringArray _strings;
+    std::vector<uint32_t> _histogram;
+    UIntSet _wnums;
+    uint32_t _colls;
+    uint32_t _minFill;
+    uint32_t _randFill;
+
+public:
+    RandTextFieldGenerator(const vespalib::string &name,
+                           search::Rand48 &rnd,
+                           uint32_t numWords,
+                           uint32_t minFill,
+                           uint32_t maxFill);
+
+    virtual
+    ~RandTextFieldGenerator(void);
+
+    virtual void
+    setup(void);
+
+    virtual void
+    clear(void);
+
+    virtual void
+    deleteHistogram(const vespalib::string &baseDir,
+                    const vespalib::string &name);
+
+    virtual void
+    writeHistogram(const vespalib::string &baseDir,
+                   const vespalib::string &name);
+
+    virtual void
+    generate(vespalib::asciistream &doc, uint32_t id);
+};
+
+
+RandTextFieldGenerator::RandTextFieldGenerator(const vespalib::string &name,
+                                               search::Rand48 &rnd,
+                                               uint32_t numWords,
+                                               uint32_t minFill,
+                                               uint32_t randFill)
+    : FieldGenerator(name),
+      _rnd(rnd),
+      _numWords(numWords),
+      _strings(),
+      _histogram(),
+      _wnums(),
+      _colls(0u),
+      _minFill(minFill),
+      _randFill(randFill)
+{
+}
+
+
+
+RandTextFieldGenerator::~RandTextFieldGenerator(void)
+{
+}
+
+
+void
+RandTextFieldGenerator::setup(void)
+{
+    LOG(info,
+        "generating dictionary for field %s (%u words)",
+        _name.c_str(), _numWords);
+    StringGenerator(_rnd).rand_unique_array(_strings, 5, 10, _numWords);
+    _histogram.resize(_numWords);
+}
+
+
+void
+RandTextFieldGenerator::clear(void)
+{
+    typedef std::vector<uint32_t>::iterator HI;
+    for (HI i(_histogram.begin()), ie(_histogram.end()); i != ie; ++i) {
+        *i = 0;
+    }
+    _colls = 0;
+}
+
+
+void
+RandTextFieldGenerator::deleteHistogram(const vespalib::string &baseDir,
+                                        const vespalib::string &name)
+{
+    vespalib::string fname(prependBaseDir(baseDir, name) + "-" + _name);
+    FastOS_File::Delete(fname.c_str());
+}
+
+
+void
+RandTextFieldGenerator::writeHistogram(const vespalib::string &baseDir,
+                                       const vespalib::string &name)
+{
+    LOG(info, "%u word collisions for field %s", _colls, _name.c_str());
+    vespalib::string fname(name + "-" + _name);
+    vespalib::string fullName(prependBaseDir(baseDir, fname));
+    LOG(info, "Writing histogram %s", fname.c_str());
+    Fast_BufferedFile f(new FastOS_File);
+    f.WriteOpen(fullName.c_str());
+    uint32_t numWords = _strings.size();
+    assert(numWords == _histogram.size());
+    for (uint32_t wNum = 0; wNum < numWords; ++wNum) {
+        f.WriteString(_strings[wNum].c_str());
+        f.WriteString(" ");
+        f.addNum(_histogram[wNum], 0, ' ');
+        f.WriteString("\n");
+    }
+    f.Flush();
+    f.Close();
+    shafile(baseDir, fname);
+}
+
+
+void
+RandTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id)
+{
+    (void) id;
+    doc << "  <" << _name << ">";
+    _wnums.clear();
+    uint32_t gLen = _minFill + _rnd.lrand48() % (_randFill + 1);
+    bool first = true;
+    for (uint32_t i = 0; i < gLen; ++i) {
+        if (!first)
+            doc << " ";
+        first = false;
+        uint32_t wNum = _rnd.lrand48() % _strings.size();
+        if (_wnums.insert(wNum).second)
+            _histogram[wNum]++;
+        else
+            ++_colls;
+        const vespalib::string &s(_strings[wNum]);
+        assert(s.size() > 0);
+        doc << s;
+    }
+    doc << "</" << _name << ">\n";
+}
+
+
+class ModTextFieldGenerator : public FieldGenerator
+{
+    search::Rand48 &_rnd;
+    std::vector<uint32_t> _mods;
+
+public:
+    ModTextFieldGenerator(const vespalib::string &name,
+                          search::Rand48 &rnd,
+                          const std::vector<uint32_t> &mods);
+    
+    virtual
+    ~ModTextFieldGenerator(void);
+
+    virtual void
+    clear(void);
+
+    virtual void
+    writeHistogram(const vespalib::string &name);
+
+    virtual void
+    generate(vespalib::asciistream &doc, uint32_t id);
+};
+
+
+ModTextFieldGenerator::ModTextFieldGenerator(const vespalib::string &name,
+                                             search::Rand48 &rnd,
+                                             const std::vector<uint32_t> &mods)
+    : FieldGenerator(name),
+      _rnd(rnd),
+      _mods(mods)
+{
+}
+    
+
+ModTextFieldGenerator::~ModTextFieldGenerator(void)
+{
+}
+
+
+void
+ModTextFieldGenerator::clear(void)
+{
+}
+
+
+void
+ModTextFieldGenerator::writeHistogram(const vespalib::string &name)
+{
+    (void) name;
+}
+
+
+void
+ModTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id)
+{
+    typedef std::vector<uint32_t>::const_iterator MI;
+    doc << "  <" << _name << ">";
+    bool first = true;
+    for (MI mi(_mods.begin()), me(_mods.end()); mi != me; ++mi) {
+        uint32_t m = *mi;
+        if (!first)
+            doc << " ";
+        first = false;
+        doc << "w" << m << "w" << (id % m);
+    }
+    doc << "</" << _name << ">\n";
+}
+
+
+class IdTextFieldGenerator : public FieldGenerator
+{
+public:
+    IdTextFieldGenerator(const vespalib::string &name);
+    
+    virtual
+    ~IdTextFieldGenerator(void);
+
+    virtual void
+    clear(void);
+
+    virtual void
+    writeHistogram(const vespalib::string &name);
+
+    virtual void
+    generate(vespalib::asciistream &doc, uint32_t id);
+};
+
+
+IdTextFieldGenerator::IdTextFieldGenerator(const vespalib::string &name)
+    : FieldGenerator(name)
+{
+}
+    
+
+IdTextFieldGenerator::~IdTextFieldGenerator(void)
+{
+}
+
+
+void
+IdTextFieldGenerator::clear(void)
+{
+}
+
+
+void
+IdTextFieldGenerator::writeHistogram(const vespalib::string &name)
+{
+    (void) name;
+}
+
+
+void
+IdTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id)
+{
+    doc << "  <" << _name << ">";
+    doc << id;
+    doc << "</" << _name << ">\n";
+}
+
+
+class RandIntFieldGenerator : public FieldGenerator
+{
+    search::Rand48 &_rnd;
+    uint32_t _low;
+    uint32_t _count;
+
+public:
+    RandIntFieldGenerator(const vespalib::string &name,
+                          search::Rand48 &rnd,
+                          uint32_t low,
+                          uint32_t count);
+    
+    virtual
+    ~RandIntFieldGenerator(void);
+
+    virtual void
+    clear(void);
+
+    virtual void
+    writeHistogram(const vespalib::string &name);
+
+    virtual void
+    generate(vespalib::asciistream &doc, uint32_t id);
+};
+
+
+
+RandIntFieldGenerator::RandIntFieldGenerator(const vespalib::string &name,
+                                             search::Rand48 &rnd,
+                                             uint32_t low,
+                                             uint32_t count)
+    : FieldGenerator(name),
+      _rnd(rnd),
+      _low(low),
+      _count(count)
+{
+}
+    
+
+RandIntFieldGenerator::~RandIntFieldGenerator(void)
+{
+}
+
+
+void
+RandIntFieldGenerator::clear(void)
+{
+}
+
+
+void
+RandIntFieldGenerator::writeHistogram(const vespalib::string &name)
+{
+    (void) name;
+}
+
+
+void
+RandIntFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id)
+{
+    (void) id;
+    doc << "  <" << _name << ">";
+    uint32_t r = _low + _rnd.lrand48() % _count;
+    doc << r;
+    doc << "</" << _name << ">\n";
+}
+
+class DocumentGenerator
+{
+    vespalib::string _docType;
+    vespalib::string _idPrefix;
+    vespalib::asciistream _doc;
+    typedef std::vector<FieldGenerator::SP> FieldVec;
+    const FieldVec _fields;
+ 
+    void
+    setup(void);
+public:   
+    DocumentGenerator(const vespalib::string &docType,
+                      const vespalib::string &idPrefix,
+                      const FieldVec &fields);
+
+    ~DocumentGenerator(void);
+
+    void
+    clear(void);
+
+    void
+    deleteHistogram(const vespalib::string &baseDir,
+                    const vespalib::string &name);
+
+    void
+    writeHistogram(const vespalib::string &baseDir,
+                   const vespalib::string &name);
+
+    void
+    generate(uint32_t id);
+
+    void
+    generate(uint32_t docMin, uint32_t docCount,
+             const vespalib::string &baseDir,
+             const vespalib::string &feedFileName,
+             bool headers);
+};
+
+
+DocumentGenerator::DocumentGenerator(const vespalib::string &docType,
+                                     const vespalib::string &idPrefix,
+                                     const FieldVec &fields)
+    : _docType(docType),
+      _idPrefix(idPrefix),
+      _doc(),
+      _fields(fields)
+{
+    setup();
+}
+
+
+DocumentGenerator::~DocumentGenerator(void)
+{
+}
+
+void
+DocumentGenerator::setup(void)
+{
+    typedef FieldVec::const_iterator FI;
+    for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) {
+        (*i)->setup();
+    }
+}
+
+
+void
+DocumentGenerator::clear(void)
+{
+    typedef FieldVec::const_iterator FI;
+    for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) {
+        (*i)->clear();
+    }
+}
+
+
+void
+DocumentGenerator::generate(uint32_t id)
+{
+    _doc.clear();
+    _doc << "<document documenttype=\"" << _docType << "\" documentid= \"" <<
+        _idPrefix << id << "\">\n";
+    typedef FieldVec::const_iterator FI;
+    for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) {
+        (*i)->generate(_doc, id);
+    }
+    _doc << "</document>\n";
+}
+
+
+void
+DocumentGenerator::deleteHistogram(const vespalib::string &baseDir,
+                                   const vespalib::string &name)
+{
+    typedef FieldVec::const_iterator FI;
+    for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) {
+        (*i)->deleteHistogram(baseDir, name);
+    }
+}
+                            
+void
+DocumentGenerator::writeHistogram(const vespalib::string &baseDir,
+                                  const vespalib::string &name)
+{
+    typedef FieldVec::const_iterator FI;
+    for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) {
+        (*i)->writeHistogram(baseDir, name);
+    }
+}
+                            
+void
+DocumentGenerator::generate(uint32_t docMin, uint32_t docCount,
+                            const vespalib::string &baseDir,
+                            const vespalib::string &feedFileName,
+                            bool headers)
+{
+    vespalib::string fullName(prependBaseDir(baseDir, feedFileName));
+    FastOS_File::Delete(fullName.c_str());
+    vespalib::string histname(feedFileName + ".histogram");
+    deleteHistogram(baseDir, histname);
+    Fast_BufferedFile f(new FastOS_File);
+    f.WriteOpen(fullName.c_str());
+    clear();
+    if (headers) {
+        f.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n");
+        f.WriteString("<vespafeed>\n");
+    }
+    uint32_t docLim = docMin + docCount;
+    for (uint32_t id = docMin; id < docLim; ++id) {
+        generate(id);
+        f.WriteBuf(_doc.c_str(), _doc.size());
+    }
+    if (headers) {
+        f.WriteString("</vespafeed>\n");
+    }
+    f.Flush();
+    f.Close();
+    LOG(info, "Calculating sha256 for %s", feedFileName.c_str());
+    shafile(baseDir, feedFileName);
+    writeHistogram(baseDir, histname);
+}
+
+
+class SubApp
+{
+protected:
+    FastOS_Application &_app;
+
+public:
+    SubApp(FastOS_Application &app)
+        : _app(app)
+    {
+    }
+
+    virtual
+    ~SubApp(void)
+    {
+    }
+
+    virtual void
+    usage(bool showHeader) = 0;
+
+    virtual bool
+    getOptions(void) = 0;
+
+    virtual int
+    run(void) = 0;
+};
+
+class GenTestDocsApp : public SubApp
+{
+    vespalib::string _baseDir;
+    vespalib::string _docType;
+    uint32_t _minDocId;
+    uint32_t _docIdLimit;
+    bool _verbose;
+    int _numWords;
+    int _optIndex;
+    std::vector<FieldGenerator::SP> _fields;
+    std::vector<uint32_t> _mods;
+    search::Rand48 _rnd;
+    vespalib::string _outFile;
+    bool _headers;
+    
+public:
+    GenTestDocsApp(FastOS_Application &app)
+        : SubApp(app),
+          _baseDir(""),
+          _docType("testdoc"),
+          _minDocId(0u),
+          _docIdLimit(5u),
+          _verbose(false),
+          _numWords(1000),
+          _optIndex(1),
+          _fields(),
+          _mods(),
+          _rnd(),
+          _outFile(),
+          _headers(false)
+    {
+        _mods.push_back(2);
+        _mods.push_back(3);
+        _mods.push_back(5);
+        _mods.push_back(7);
+        _mods.push_back(11);
+        _rnd.srand48(42);
+    }
+
+    virtual
+    ~GenTestDocsApp(void)
+    {
+    }
+
+    virtual void
+    usage(bool showHeader);
+
+    virtual bool
+    getOptions(void);
+
+    virtual int
+    run(void);
+};
+
+
+void
+GenTestDocsApp::usage(bool showHeader)
+{
+    using std::cerr;
+    if (showHeader)
+        usageHeader();
+    cerr <<
+        "vespa-gen-testdocs gentestdocs\n"
+        " [--basedir basedir]\n"
+        " [--randtextfield name]\n"
+        " [--modtextfield name]\n"
+        " [--idtextfield name]\n"
+        " [--randintfield name]\n"
+        " [--docidlimit docIdLimit]\n"
+        " [--mindocid mindocid]\n"
+        " [--numwords numWords]\n"
+        " [--doctype docType]\n"
+        " [--headers]\n"
+        " outFile\n";
+}
+
+bool
+GenTestDocsApp::getOptions(void)
+{
+    int c;
+    const char *optArgument = NULL;
+    int longopt_index = 0;
+    static struct option longopts[] = {
+        { "basedir", 1, NULL, 0 },
+        { "randtextfield", 1, NULL, 0 },
+        { "modtextfield", 1, NULL, 0 },
+        { "idtextfield", 1, NULL, 0 },
+        { "randintfield", 1, NULL, 0 },
+        { "docidlimit", 1, NULL, 0 },
+        { "mindocid", 1, NULL, 0 },
+        { "numwords", 1, NULL, 0 },
+        { "doctype", 1, NULL, 0 },
+        { "headers", 0, NULL, 0 }, 
+        { NULL, 0, NULL, 0 }
+    };
+    enum longopts_enum {
+        LONGOPT_BASEDIR,
+        LONGOPT_RANDTEXTFIELD,
+        LONGOPT_MODTEXTFIELD,
+        LONGOPT_IDTEXTFIELD,
+        LONGOPT_RANDINTFIELD,
+        LONGOPT_DOCIDLIMIT,
+        LONGOPT_MINDOCID,
+        LONGOPT_NUMWORDS,
+        LONGOPT_DOCTYPE,
+        LONGOPT_HEADERS
+    };
+    int optIndex = 2;
+    while ((c = _app.GetOptLong("v",
+                                optArgument,
+                                optIndex,
+                                longopts,
+                                &longopt_index)) != -1) {
+        FieldGenerator::SP g;
+        switch (c) {
+        case 0:
+            switch (longopt_index) {
+            case LONGOPT_BASEDIR:
+                _baseDir = optArgument;
+                break;
+            case LONGOPT_RANDTEXTFIELD:
+                g.reset(new RandTextFieldGenerator(optArgument,
+                                                   _rnd,
+                                                   _numWords,
+                                                   20,
+                                                   50));
+                _fields.push_back(g);
+                break;
+            case LONGOPT_MODTEXTFIELD:
+                g.reset(new ModTextFieldGenerator(optArgument,
+                                                  _rnd,
+                                                  _mods));
+                _fields.push_back(g);
+                break;
+            case LONGOPT_IDTEXTFIELD:
+                g.reset(new IdTextFieldGenerator(optArgument));
+                _fields.push_back(g);
+                break;
+            case LONGOPT_RANDINTFIELD:
+                g.reset(new RandIntFieldGenerator(optArgument,
+                                                  _rnd,
+                                                  0,
+                                                  100000));
+                _fields.push_back(g);
+                break;
+                break;
+            case LONGOPT_DOCIDLIMIT:
+                _docIdLimit = atoi(optArgument);
+                break;
+            case LONGOPT_MINDOCID:
+                _minDocId = atoi(optArgument);
+                break;
+            case LONGOPT_NUMWORDS:
+                _numWords = atoi(optArgument);
+                break;
+            case LONGOPT_DOCTYPE:
+                _docType = optArgument;
+                break;
+            case LONGOPT_HEADERS:
+                _headers = true;
+                break;
+            default:
+                if (optArgument != NULL) {
+                    LOG(error,
+                        "longopt %s with arg %s",
+                        longopts[longopt_index].name, optArgument);
+                } else {
+                    LOG(error,
+                        "longopt %s",
+                        longopts[longopt_index].name);
+                }
+            }
+            break;
+        case 'v':
+            _verbose = true;
+            break;
+        default:
+            return false;
+        }
+    }
+    _optIndex = optIndex;
+    if (_optIndex >= _app._argc) {
+        return false;
+    }
+    _outFile = _app._argv[optIndex];
+    return true;
+}
+
+
+int
+GenTestDocsApp::run(void)
+{
+    printf("Hello world\n");
+    vespalib::string idPrefix("id:test:");
+    idPrefix += _docType;
+    idPrefix += "::";
+    DocumentGenerator dg(_docType,
+                         idPrefix,
+                         _fields);
+    LOG(info, "generating %s", _outFile.c_str());
+    dg.generate(_minDocId, _docIdLimit, _baseDir, _outFile, _headers);
+    LOG(info, "done");
+    return 0;
+}
+
+
+class App : public FastOS_Application
+{
+public:
+    void
+    usage(void);
+
+    int
+    Main(void);
+};
+
+
+void
+App::usage(void)
+{
+    GenTestDocsApp(*this).usage(true);
+}
+
+int
+App::Main(void)
+{
+    if (_argc < 2) {
+        usage();
+        return 1;
+    }
+    std::unique_ptr<SubApp> subApp;
+    if (strcmp(_argv[1], "gentestdocs") == 0)
+        subApp.reset(new GenTestDocsApp(*this));
+    if (subApp.get() != NULL) {
+        if (!subApp->getOptions()) {
+            subApp->usage(true);
+            return 1;
+        }
+        return subApp->run();
+    }
+    usage();
+    return 1;
+}
+
+
+int
+main(int argc, char **argv)
+{
+    App app;
+    return app.Entry(argc, argv);
+}
diff --git a/searchcore/src/apps/vespa-proton-cmd/.gitignore b/searchcore/src/apps/vespa-proton-cmd/.gitignore
new file mode 100644
index 00000000000..5d106f9b232
--- /dev/null
+++ b/searchcore/src/apps/vespa-proton-cmd/.gitignore
@@ -0,0 +1,2 @@
+/vespa-proton-cmd
+vespa-proton-cmd-bin
diff --git a/searchcore/src/apps/vespa-proton-cmd/CMakeLists.txt b/searchcore/src/apps/vespa-proton-cmd/CMakeLists.txt
new file mode 100644
index 00000000000..36e62cc67e7
--- /dev/null
+++ b/searchcore/src/apps/vespa-proton-cmd/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_vespa-proton-cmd_app
+    SOURCES
+    vespa-proton-cmd.cpp
+    OUTPUT_NAME vespa-proton-cmd-bin
+    INSTALL bin
+    DEPENDS
+)
diff --git a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp
new file mode 100644
index 00000000000..36c8fc8a6c8
--- /dev/null
+++ b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp
@@ -0,0 +1,484 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("vespa-proton-cmd");
+#include <vespa/fnet/frt/frt.h>
+#include <algorithm>
+#include <string>
+#include <vespa/slobrok/sbmirror.h>
+#include <vespa/config-slobroks.h>
+#include <vespa/vespalib/util/host_name.h>
+
+
+namespace pandora {
+namespace rtc_cmd {
+
+class App : public FastOS_Application
+{
+private:
+    App(const App &);
+    App& operator=(const App &);
+
+    FRT_Supervisor *_supervisor;
+    FRT_Target     *_target;
+    FRT_RPCRequest *_req;
+
+public:
+    App() : _supervisor(NULL),
+            _target(NULL),
+            _req(NULL) {}
+    virtual ~App()
+    {
+        assert(_supervisor == NULL);
+        assert(_target == NULL);
+        assert(_req == NULL);
+    }
+
+    int usage()
+    {
+        fprintf(stderr, "usage: %s <port|spec|--local|--id=name> <cmd> [args]\n", _argv[0]);
+        fprintf(stderr, "die\n");
+        fprintf(stderr, "getConfigTime\n");
+        fprintf(stderr, "getProtonStatus\n");
+        fprintf(stderr, "getState\n");
+        fprintf(stderr, "listDocTypes\n");
+        fprintf(stderr, "listSchema documentType\n");
+        fprintf(stderr, "monitor\n");
+        fprintf(stderr, "enableSearching\n");
+        fprintf(stderr, "disableSearching\n");
+        fprintf(stderr, "triggerFlush\n");
+        fprintf(stderr, "prepareRestart\n");
+        fprintf(stderr, "wipeHistory\n");
+        return 1;
+    }
+
+    void initRPC()
+    {
+        _supervisor = new FRT_Supervisor();
+        _req = _supervisor->AllocRPCRequest();
+        _supervisor->Start();
+    }
+
+    void invokeRPC(bool print, double timeout=5.0)
+    {
+        if (_req == NULL)
+            return;
+
+        _target->InvokeSync(_req, timeout);
+        if (print || _req->IsError())
+            _req->Print(0);
+    }
+
+    void finiRPC()
+    {
+        if (_req != NULL) {
+            _req->SubRef();
+            _req = NULL;
+        }
+        if (_target != NULL) {
+            _target->SubRef();
+            _target = NULL;
+        }
+        if (_supervisor != NULL) {
+            _supervisor->ShutDown(true);
+            delete _supervisor;
+            _supervisor = NULL;
+        }
+    }
+
+    void
+    monitorLoop(void);
+
+    void
+    scanSpecs(slobrok::api::MirrorAPI::SpecList &specs,
+              const std::string &me,
+              std::string &service,
+              std::string &spec,
+              int &found)
+    {
+        for (size_t j = 0; j < specs.size(); ++j) {
+            if (specs[j].first == service)
+                continue;
+            if (specs[j].second.compare(0, me.length(), me) == 0) {
+                service = specs[j].first;
+                spec = specs[j].second;
+                printf("found local RTC '%s' with connection spec %s\n",
+                       specs[j].first.c_str(), spec.c_str());
+                ++found;
+            }
+        }
+    }
+
+    std::string findRTC() {
+        std::string me = "tcp/";
+        me += vespalib::HostName::get().c_str();
+        me += ":";
+
+        std::string rtcPattern = "search/cluster.*/c*/r*/realtimecontroller";
+        std::string rtcPattern2 = "*/search/cluster.*/*/realtimecontroller";
+        std::string rtcPattern3 = "*/search/*/realtimecontroller";
+
+        try {
+            slobrok::ConfiguratorFactory sbcfg("admin/slobrok.0");
+            slobrok::api::MirrorAPI sbmirror(*_supervisor, sbcfg);
+            for (int timeout = 1; timeout < 20; timeout++) {
+                if (!sbmirror.ready()) {
+                    FastOS_Thread::Sleep(50*timeout);
+                }
+            }
+            if (!sbmirror.ready()) {
+                fprintf(stderr,
+                        "ERROR: no data from service location broker\n");
+                exit(1);
+            }
+            slobrok::api::MirrorAPI::SpecList specs =
+                sbmirror.lookup(rtcPattern);
+            slobrok::api::MirrorAPI::SpecList specs2 =
+                sbmirror.lookup(rtcPattern2);
+            slobrok::api::MirrorAPI::SpecList specs3 =
+                sbmirror.lookup(rtcPattern3);
+
+            int found = 0;
+            std::string service;
+            std::string spec;
+
+            printf("looking for RTCs matching '%s' (length %d)\n",
+                   me.c_str(), (int)me.length());
+            scanSpecs(specs, me, service, spec, found);
+            scanSpecs(specs2, me, service, spec, found);
+            scanSpecs(specs3, me, service, spec, found);
+            if (found > 1) {
+                fprintf(stderr, "found more than one local RTC, you must use --id=<name>\n");
+                exit(1);
+            }
+            if (found < 1) {
+                fprintf(stderr, "found no local RTC, you must use --id=<name> (list follows):\n");
+                for (size_t j = 0; j < specs.size(); ++j) {
+                    printf("RTC name %s with connection spec %s\n",
+                           specs[j].first.c_str(), specs[j].second.c_str());
+                }
+                exit(1);
+            }
+            return spec;
+        } catch (config::InvalidConfigException& e) {
+            fprintf(stderr, "ERROR: failed to get service location broker configuration\n");
+            exit(1);
+        }
+        return "";
+    }
+
+    std::string findRTC(std::string id) {
+        std::string rtcPattern = "search/cluster.*/c*/r*/realtimecontroller";
+
+        try {
+            slobrok::ConfiguratorFactory sbcfg("admin/slobrok.0");
+            slobrok::api::MirrorAPI sbmirror(*_supervisor, sbcfg);
+            for (int timeout = 1; timeout < 20; timeout++) {
+                if (!sbmirror.ready()) {
+                    FastOS_Thread::Sleep(50*timeout);
+                }
+            }
+            if (!sbmirror.ready()) {
+                throw std::runtime_error("ERROR: no data from service location broker");
+            }
+            slobrok::api::MirrorAPI::SpecList specs = sbmirror.lookup(id);
+
+            int found = 0;
+            std::string spec;
+
+            for (size_t j = 0; j < specs.size(); ++j) {
+                std::string name = specs[j].first;
+                spec = specs[j].second;
+                printf("found RTC '%s' with connection spec %s\n",
+                       name.c_str(), spec.c_str());
+                ++found;
+            }
+            if (found > 1) {
+                throw std::runtime_error("found more than one RTC, use a more specific id");
+            }
+            if (found < 1) {
+                specs = sbmirror.lookup(rtcPattern);
+
+                std::string msg = vespalib::make_string("found no RTC named '%s' (list follows):\n", id.c_str());
+                for (size_t j = 0; j < specs.size(); ++j) {
+                    msg += vespalib::make_string("RTC name %s with connection spec %s\n", specs[j].first.c_str(), specs[j].second.c_str());
+                }
+                throw std::runtime_error(msg);
+            }
+            return spec;
+        } catch (config::InvalidConfigException& e) {
+            throw std::runtime_error("ERROR: failed to get service location broker configuration");
+        }
+        return "";
+    }
+
+
+    int Main()
+    {
+        if (_argc < 3) {
+            return usage();
+        }
+
+        initRPC();
+
+        int port = 0;
+        std::string spec = _argv[1];
+
+        try {
+            if (spec == "--local") {
+                spec = findRTC();
+            } else if (spec.compare(0, 5, "--id=") == 0) {
+                spec = findRTC(spec.substr(5));
+            } else {
+                port = atoi(_argv[1]);
+            }
+        } catch (const std::runtime_error & e) {
+            fprintf(stderr, "%s", e.what());
+            finiRPC();
+            return 1;
+        } catch (const config::ConfigTimeoutException & e) {
+            fprintf(stderr, "Getting config timed out: %s", e.what());
+            finiRPC();
+            return 2;
+        }
+
+        if (port == 0 && spec.compare(0, 4, "tcp/") != 0) {
+            finiRPC();
+            return usage();
+        }
+
+        if (port != 0) {
+            _target = _supervisor->GetTarget(port);
+        } else {
+            _target = _supervisor->GetTarget(spec.c_str());
+        }
+
+        bool invoked = false;
+
+        if (strcmp(_argv[2], "enableSearching") == 0) {
+            _req->SetMethodName("proton.enableSearching");
+        } else if (strcmp(_argv[2], "disableSearching") == 0) {
+            _req->SetMethodName("proton.disableSearching");
+        } else if (strcmp(_argv[2], "getState") == 0 &&
+                   _argc >= 3) {
+            _req->SetMethodName("pandora.rtc.getState");
+
+            FRT_Values &params = *_req->GetParams();
+
+            params.AddInt32(_argc > 3 ? atoi(_argv[3]) : 0);
+            params.AddInt32(_argc > 4 ? atoi(_argv[4]) : 0);
+            invokeRPC(false);
+            invoked = true;
+
+            FRT_Values &rvals = *_req->GetReturn();
+
+            if (!_req->IsError()) {
+                FRT_Value &names = rvals.GetValue(0);
+                FRT_Value &values = rvals.GetValue(1);
+                FRT_Value &gencnt = rvals.GetValue(2);
+
+                for (unsigned int i = 0;
+                     i < names._string_array._len &&
+                                      i < values._string_array._len;
+                     i++)
+                {
+                    printf("\"%s\", \"%s\"\n",
+                           names._string_array._pt[i]._str,
+                           values._string_array._pt[i]._str);
+                }
+                printf("gencnt=%u\n",
+                       static_cast<unsigned int>(gencnt._intval32));
+            }
+        } else if (strcmp(_argv[2], "getProtonStatus") == 0 &&
+                   _argc >= 3) {
+
+            _req->SetMethodName("proton.getStatus");
+            FRT_Values &params = *_req->GetParams();
+            params.AddString(_argc > 3 ? _argv[3] : "");
+            invokeRPC(false);
+            invoked = true;
+            FRT_Values &rvals = *_req->GetReturn();
+            if (!_req->IsError()) {
+                FRT_Value &components = rvals.GetValue(0);
+                FRT_Value &states = rvals.GetValue(1);
+                FRT_Value &internalStates = rvals.GetValue(2);
+                FRT_Value &messages = rvals.GetValue(3);
+                for (unsigned int i = 0; i < components._string_array._len &&
+                                         i < states._string_array._len &&
+                                         i < internalStates.
+                                               _string_array._len &&
+                                         i < messages._string_array._len;
+                                       i++) {
+                    printf("\"%s\",\"%s\",\"%s\",\"%s\"\n",
+                           components._string_array._pt[i]._str,
+                           states._string_array._pt[i]._str,
+                           internalStates._string_array._pt[i]._str,
+                           messages._string_array._pt[i]._str);
+                }
+
+            }
+        } else if (strcmp(_argv[2], "triggerFlush") == 0) {
+            _req->SetMethodName("proton.triggerFlush");
+            invokeRPC(false, 86400.0);
+            invoked = true;
+            if (! _req->IsError()) {
+                printf("OK: flush trigger enabled\n");
+            }
+        } else if (strcmp(_argv[2], "prepareRestart") == 0) {
+            _req->SetMethodName("proton.prepareRestart");
+            invokeRPC(false, 86400.0);
+            invoked = true;
+            if (! _req->IsError()) {
+                printf("OK: prepareRestart enabled\n");
+            }
+        } else if (strcmp(_argv[2], "listDocTypes") == 0) {
+            _req->SetMethodName("proton.listDocTypes");
+            invokeRPC(false, 86400.0);
+            invoked = true;
+            if (! _req->IsError()) {
+                FRT_Values &ret = *_req->GetReturn();
+                if (strcmp(ret.GetTypeString(), "S") == 0) {
+                    uint32_t dtLen = ret[0]._string_array._len;
+                    const FRT_StringValue *dt = ret[0]._string_array._pt;
+                    for (uint32_t i = 0; i < dtLen; ++i) {
+                        if (i > 0)
+                            printf(" ");
+                        printf("%s", dt[i]._str);
+                    }
+                    printf("\n");
+                } else {
+                    fprintf(stderr, "Unexpected return value\n");
+                }
+            }
+        } else if (strcmp(_argv[2], "listSchema") == 0 && _argc == 4) {
+            _req->SetMethodName("proton.listSchema");
+            FRT_Values &arg = *_req->GetParams();
+            arg.AddString(_argv[3]);
+            invokeRPC(false, 86400.0);
+            invoked = true;
+            if (! _req->IsError()) {
+                FRT_Values &ret = *_req->GetReturn();
+                if (strcmp(ret.GetTypeString(), "SSSS") == 0) {
+                    uint32_t fnLen = ret[0]._string_array._len;
+                    const FRT_StringValue *fn = ret[0]._string_array._pt;
+                    uint32_t fdtLen = ret[1]._string_array._len;
+                    const FRT_StringValue *fdt = ret[1]._string_array._pt;
+                    uint32_t fctLen = ret[2]._string_array._len;
+                    const FRT_StringValue *fct = ret[2]._string_array._pt;
+                    uint32_t flLen = ret[3]._string_array._len;
+                    const FRT_StringValue *fl = ret[3]._string_array._pt;
+                    for (uint32_t i = 0;
+                         i < fnLen && i < fdtLen && i < fctLen && i < flLen;
+                         ++i) {
+                        if (i > 0)
+                            printf(" ");
+                        printf("%s/%s/%s/%s",
+                               fn[i]._str,
+                               fdt[i]._str,
+                               fct[i]._str,
+                               fl[i]._str);
+                    }
+                    printf("\n");
+                } else {
+                    fprintf(stderr, "Unexpected return value\n");
+                }
+            }
+        } else if (strcmp(_argv[2], "getConfigTime") == 0) {
+            _req->SetMethodName("proton.getConfigTime");
+            invokeRPC(false, 86400.0);
+            invoked = true;
+            if (! _req->IsError()) {
+                FRT_Values &ret = *_req->GetReturn();
+                if (strcmp(ret.GetTypeString(), "l") == 0) {
+                    uint64_t configTime = ret[0]._intval64;
+                    printf("%" PRId64 "\n", configTime);
+                } else {
+                    fprintf(stderr, "Unexpected return value\n");
+                }
+            }
+        } else if (strcmp(_argv[2], "wipeHistory") == 0) {
+            _req->SetMethodName("proton.wipeHistory");
+            invokeRPC(false, 86400.0);
+            invoked = true;
+            if (! _req->IsError()) {
+                printf("OK: history wiped\n");
+            }
+        } else if (strcmp(_argv[2], "die") == 0) {
+            _req->SetMethodName("pandora.rtc.die");
+
+        } else if (strcmp(_argv[2], "monitor") == 0) {
+            invoked = true;
+            monitorLoop();
+        } else {
+            finiRPC();
+            return usage();
+        }
+        if (!invoked)
+            invokeRPC(true);
+        finiRPC();
+        return 0;
+    }
+};
+
+
+void
+App::monitorLoop(void)
+{
+    for (;;) {
+        FRT_RPCRequest *req = _supervisor->AllocRPCRequest();
+        req->SetMethodName("pandora.rtc.getIncrementalState");
+        FRT_Values &params = *req->GetParams();
+        params.AddInt32(2000);
+        _target->InvokeSync(req, 1200.0);
+
+        if (req->IsError()) {
+            req->Print(0);
+            req->SubRef();
+            break;
+        }
+        FRT_Values &rvals = *req->GetReturn();
+        FRT_Value &names = rvals.GetValue(0);
+        FRT_Value &values = rvals.GetValue(1);
+        struct timeval tnow;
+        gettimeofday(&tnow, NULL);
+
+        for (unsigned int i = 0;
+             i < names._string_array._len &&
+                              i < values._string_array._len;
+             i++)
+        {
+            time_t now;
+            struct tm *nowtm;
+
+            now = tnow.tv_sec;
+            nowtm = gmtime(&now);
+            fprintf(stdout,
+                    "%04d-%02d-%02dT%02d:%02d:%02d.%06dZ "
+                    "%010d.%06d ==> ",
+                    nowtm->tm_year + 1900,
+                    nowtm->tm_mon + 1,
+                    nowtm->tm_mday,
+                    nowtm->tm_hour,
+                    nowtm->tm_min,
+                    nowtm->tm_sec,
+                    (int)tnow.tv_usec,
+                    (int)tnow.tv_sec,
+                    (int) tnow.tv_usec);
+            printf("\"%s\", \"%s\"\n",
+                   names._string_array._pt[i]._str,
+                   values._string_array._pt[i]._str);
+        }
+        fflush(stdout);
+        req->SubRef();
+    }
+}
+
+} // namespace pandora::rtc_cmd
+} // namespace pandora
+
+
+int main(int argc, char **argv)
+{
+    pandora::rtc_cmd::App app;
+    return app.Entry(argc, argv);
+}
diff --git a/searchcore/src/apps/vespa-remove-indexes/vespa-remove-index.sh b/searchcore/src/apps/vespa-remove-indexes/vespa-remove-index.sh
new file mode 100755
index 00000000000..d66a0e79bb2
--- /dev/null
+++ b/searchcore/src/apps/vespa-remove-indexes/vespa-remove-index.sh
@@ -0,0 +1,255 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+    myname=${0}
+    mypath=${myname%/*}
+    myname=${myname##*/}
+    if [ "$mypath" ] && [ -d "$mypath" ]; then
+        return
+    fi
+    mypath=$(pwd)
+    if [ -f "${mypath}/${myname}" ]; then
+        return
+    fi
+    echo "FATAL: Could not figure out the path where $myname lives from $0"
+    exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+    if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+        # ensure it ends with "/" :
+        VESPA_HOME=${VESPA_HOME%/}/
+        export VESPA_HOME
+        common_env=$VESPA_HOME/$COMMON_ENV
+        if [ -f "$common_env" ]; then
+            . $common_env
+            return
+        fi
+    fi
+    return 1
+}
+
+findroot () {
+    source_common_env && return
+    if [ "$VESPA_HOME" ]; then
+        echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+        exit 1
+    fi
+    if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+        VESPA_HOME="$ROOT"
+        source_common_env && return
+    fi
+    findpath
+    while [ "$mypath" ]; do
+        VESPA_HOME=${mypath}
+        source_common_env && return
+        mypath=${mypath%/*}
+    done
+    echo "FATAL: missing VESPA_HOME environment variable"
+    echo "Could not locate $COMMON_ENV anywhere"
+    exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+ROOT=$VESPA_HOME
+cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; }
+
+usage() {
+    (
+	echo "This script will remove vespa indexes on this node."
+        echo "It will refuse to execute if Vespa is running."
+        echo "The following options are recognized:"
+        echo ""
+        echo "-cluster <cluster>      only remove data for given cluster"
+        echo "-key <distribution key> only remove data for given key"
+        echo "-row <row number>       only remove data for given row"
+        echo "-column <col number>    only remove data for given column"
+        echo "-old                    remove data from Vespa 4.2 or older"
+        echo "-force                  do not ask for confirmation before removal"
+    ) >&2
+}
+
+onlycluster=""
+onlykey=""
+onlyrow=""
+onlycolumn=""
+sudo="sudo"
+ask=true
+removeold=false
+confirmed=true
+olddir=var/db/vespa/index
+newdir=var/db/vespa/search
+
+if [ -w $newdir ] && [ -w $olddir ]; then
+    sudo=""
+fi
+
+while [ $# -gt 0 ]; do
+    case $1 in
+        -h|-help)        usage; exit 0;;
+        -C|-cluster)     shift; onlycluster="$1"; shift ;;
+        -k|-key)         shift; onlykey="$1"; shift ;;
+        -r|-row)         shift; onlyrow="$1"; shift ;;
+        -c|-column)      shift; onlycolumn="$1" ; shift ;;
+        -nosudo)         shift; sudo="" ;;
+        -sudo)           shift; sudo="sudo" ;;
+        -old)            shift; removeold=true ;;
+	-force)          shift; ask=false ;;
+        *)               echo "Unrecognized option '$1'" >&2; usage; exit 1;;
+    esac
+done
+# Will first check if vespa is running on this node
+P_CONFIGPROXY=var/run/configproxy.pid
+if [ -f $P_CONFIGPROXY ] && $sudo kill -0 `cat $P_CONFIGPROXY` 2>/dev/null; then
+  echo "[ERROR] Will not remove indexes while Vespa is running" 1>&2
+  echo "[ERROR] stop services and run 'ps xgauww' to check for Vespa processes" 1>&2
+  exit 1
+fi
+
+removedata() {
+    echo "[info] removing data: $sudo rm -rf $*"
+    $sudo rm -rf $*
+    echo "[info] removed."
+}
+
+confirm() {
+    confirmed=false
+    echo -n 'Really to remove this vespa index? Type "yes" if you are sure ==> ' 1>&2
+    answer=no
+    read answer
+    if [ "$answer" = "yes" ]; then
+	confirmed=true
+    else
+        confirmed=false
+        echo "[info] skipping removal ('$answer' != 'yes')"
+    fi
+}
+
+garbage_collect_dirs() {
+    find $olddir $newdir -type d -depth 2>/dev/null | while read dir; do
+	[ "$dir" = "$olddir" ] && continue
+	[ "$dir" = "$newdir" ] && continue
+	$sudo rmdir "$dir" 2>/dev/null
+    done
+}
+
+garbage_collect_dirs
+
+if $removeold && [ -d $olddir ]; then
+    if $ask; then
+	kb=$(du -sk $olddir | awk '{print $1}')
+	if [ $kb -gt 100 ]; then
+	    confirm
+	fi
+    fi
+    if $confirmed; then
+	removedata $olddir/*
+    fi
+fi
+
+dokey() {
+    key=$1
+    keydir=$clusterdir/n$key
+    if ! [ -e "$keydir" ]; then
+        echo "directory '$keydir' does not exist"
+        return
+    fi
+    kb=$(du -sk $keydir | awk '{print $1}')
+
+    echo "[info] For cluster $cluster distribution key $key you have:"
+    echo "[info] $kb kilobytes of data in $keydir"
+    if $ask && [ "$kb" -gt 2 ]; then
+	confirm
+    fi
+    if $confirmed; then
+	removedata $keydir
+    fi
+    echo ""
+}
+
+docol() {
+    column=$1
+    coldir=$rowdir/c$column
+    if ! [ -e "$coldir" ]; then
+        echo "directory '$coldir' does not exist"
+        return
+    fi
+    kb=$(du -sk $coldir | awk '{print $1}')
+
+    echo "[info] For cluster $cluster row $row column $column you have:"
+    echo "[info] $kb kilobytes of data in $coldir"
+    if $ask && [ "$kb" -gt 2 ]; then
+	confirm
+    fi
+    if $confirmed; then
+	removedata $coldir
+    fi
+    echo ""
+}
+
+dorow() {
+    row=$1
+    rowdir=$clusterdir/r$row
+    if [ "$onlycolumn" ]; then
+	docol $onlycolumn
+    else
+	for c in `ls $rowdir`; do
+	    col=${c#c}
+	    if [ "$col" ] && [ "$col" != "$c" ]; then
+		docol $col
+	    fi
+	done
+    fi
+}
+
+docluster() {
+    cluster=$1
+    clusterdir=$newdir/cluster.$cluster
+    if ! [ -e "$clusterdir" ]; then
+        echo "Skip $cluster cluster: directory '$clusterdir' does not exist"
+        return
+    fi
+    kb=$(du -sk $clusterdir | awk '{print $1}')
+    if [ $kb -gt 1000 ]; then
+	echo "[info] You have $kb kilobytes of data for cluster $cluster"
+    fi
+    if [ "$onlykey" ]; then
+        dokey $onlykey
+    elif [ "$onlyrow" ]; then
+	dorow $onlyrow
+    else
+	for dirname in `ls $clusterdir`; do
+	    key=${dirname#n}
+	    if [ "$key" ] && [ "$key" != "$dirname" ]; then
+		dokey $key
+	    fi
+	    row=${dirname#r}
+	    if [ "$row" ] && [ "$row" != "$dirname" ]; then
+		dorow $row
+	    fi
+	done
+    fi
+}
+
+if [ "$onlycluster" ]; then
+    docluster $onlycluster
+else
+    for dir in $newdir/cluster.* ; do
+	[ -d $dir ] || continue
+	cluster=${dir#*/cluster.}
+	docluster $cluster
+    done
+fi
+
+garbage_collect_dirs
+
+exit 0
diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/.gitignore b/searchcore/src/apps/vespa-transactionlog-inspect/.gitignore
new file mode 100644
index 00000000000..a2e7a39d83d
--- /dev/null
+++ b/searchcore/src/apps/vespa-transactionlog-inspect/.gitignore
@@ -0,0 +1,2 @@
+/vespa-transactionlog-inspect
+vespa-transactionlog-inspect-bin
diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/CMakeLists.txt b/searchcore/src/apps/vespa-transactionlog-inspect/CMakeLists.txt
new file mode 100644
index 00000000000..ac869b5afd0
--- /dev/null
+++ b/searchcore/src/apps/vespa-transactionlog-inspect/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_vespa-transactionlog-inspect_app
+    SOURCES
+    vespa-transactionlog-inspect.cpp
+    OUTPUT_NAME vespa-transactionlog-inspect-bin
+    INSTALL bin
+    DEPENDS
+    searchcore_server
+    searchcore_feedoperation
+    searchcore_pcommon
+)
diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp
new file mode 100644
index 00000000000..eb6b77cd1b3
--- /dev/null
+++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp
@@ -0,0 +1,634 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("vespa-transactionlog-inspect");
+
+#include <vespa/config/helper/configgetter.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/proton/server/replaypacketdispatcher.h>
+#include <vespa/searchlib/common/fileheadercontext.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
+#include <vespa/searchlib/transactionlog/translogserver.h>
+#include <vespa/vespalib/util/programoptions.h>
+#include <iostream>
+
+using namespace proton;
+using namespace search;
+using namespace search::common;
+using namespace search::transactionlog;
+
+using document::DocumenttypesConfig;
+using document::DocumentTypeRepo;
+
+typedef std::shared_ptr<DocumenttypesConfig> DocumenttypesConfigSP;
+typedef std::unique_ptr<IReplayPacketHandler> IReplayPacketHandlerUP;
+
+struct DummyFileHeaderContext : public FileHeaderContext
+{
+    typedef std::unique_ptr<DummyFileHeaderContext> UP;
+    virtual void addTags(vespalib::GenericHeader &, const vespalib::string &) const {}
+};
+
+
+namespace
+{
+
+class ConfigFile
+{
+    typedef std::shared_ptr<ConfigFile> SP;
+
+    vespalib::string _name;
+    time_t _modTime;
+    std::vector<char> _content;
+
+public:
+    ConfigFile(void);
+
+    const vespalib::string &
+    getName(void) const
+    {
+        return _name;
+    }
+
+    vespalib::nbostream &
+    deserialize(vespalib::nbostream &stream);
+
+    void
+    print(void) const;
+};
+
+
+ConfigFile::ConfigFile(void)
+    : _name(),
+      _modTime(0),
+      _content()
+{
+}
+
+
+vespalib::nbostream &
+ConfigFile::deserialize(vespalib::nbostream &stream)
+{
+    stream >> _name;
+    assert(strchr(_name.c_str(), '/') == NULL);
+    stream >> _modTime;
+    uint32_t sz;
+    stream >> sz;
+    _content.resize(sz);
+    assert(stream.size() >= sz);
+    memcpy(&_content[0], stream.peek(), sz);
+    stream.adjustReadPos(sz);
+    return stream;
+}
+
+void
+ConfigFile::print(void) const
+{
+    std::cout << "Name: " << _name << "\n" <<
+        "ModTime: " << _modTime << "\n" <<
+        "Content-Length: " << _content.size() << "\n\n";
+    std::cout.write(&_content[0], _content.size()); 
+    std::cout << "\n-----------------------------" << std::endl;
+}
+
+vespalib::nbostream &
+operator>>(vespalib::nbostream &stream, ConfigFile &configFile)
+{
+    return configFile.deserialize(stream);
+}
+
+
+}
+
+struct DummyStreamHandler : public NewConfigOperation::IStreamHandler {
+    std::map<std::string, ConfigFile> _cfs;
+
+    DummyStreamHandler(void)
+        : NewConfigOperation::IStreamHandler(),
+          _cfs()
+    {
+    }
+
+    virtual void
+    serializeConfig(SerialNum, vespalib::nbostream &)
+    {
+    }
+
+    virtual void
+    deserializeConfig(SerialNum, vespalib::nbostream &is)
+    {
+        _cfs.clear();
+        uint32_t numConfigs;
+        is >> numConfigs;
+        for (uint32_t i = 0; i < numConfigs; ++i) {
+            ConfigFile cf;
+            is >> cf;
+            _cfs[cf.getName()] = cf;
+        }
+        assert(is.size() == 0);
+    }
+};
+
+struct DocTypeRepo {
+    DocumenttypesConfigSP docTypeCfg;
+    DocumentTypeRepo docTypeRepo;
+    DocTypeRepo(const std::string &configDir)
+        : docTypeCfg(config::ConfigGetter<DocumenttypesConfig>::
+                     getConfig("", config::DirSpec(configDir)).release()),
+          docTypeRepo(*docTypeCfg)
+    {
+    }
+};
+
+
+/**
+ * Class the receives all concrete operations as part of a domain visit
+ * and prints the content of them to standard out.
+ */
+class OperationPrinter : public IReplayPacketHandler
+{
+private:
+    DocumentTypeRepo &_repo;
+    DummyStreamHandler _streamHandler;
+    size_t _counter;
+
+protected:
+    void print(const FeedOperation &op) {
+        std::cout << "OP[" << (_counter++) << "]: " << op.toString() << std::endl;
+    }
+
+public:
+    OperationPrinter(DocumentTypeRepo &repo)
+        : _repo(repo),
+          _streamHandler(),
+          _counter(0)
+    {
+    }
+    virtual void replay(const PutOperation &op) { print(op); }
+    virtual void replay(const RemoveOperation &op) { print(op); }
+    virtual void replay(const UpdateOperation &op) { print(op); }
+    virtual void replay(const NoopOperation &op) { print(op); }
+    virtual void replay(const NewConfigOperation &op)
+    {
+        print(op);
+        typedef std::map<std::string, ConfigFile>::const_iterator I;
+        for (I i(_streamHandler._cfs.begin()), ie(_streamHandler._cfs.end());
+             i != ie; ++i) {
+            i->second.print();
+        }
+    }
+
+    virtual void replay(const WipeHistoryOperation &op) { print(op); }
+    virtual void replay(const DeleteBucketOperation &op) { print(op); }
+    virtual void replay(const SplitBucketOperation &op) { print(op); }
+    virtual void replay(const JoinBucketsOperation &op) { print(op); }
+    virtual void replay(const PruneRemovedDocumentsOperation &op) { print(op); }
+    virtual void replay(const SpoolerReplayStartOperation &op) { print(op); }
+    virtual void replay(const SpoolerReplayCompleteOperation &op) { print(op); }
+    virtual void replay(const MoveOperation &op) { print(op); }
+    virtual void replay(const CreateBucketOperation &op) { print(op); }
+    virtual void replay(const CompactLidSpaceOperation &op) { print(op); }
+    virtual NewConfigOperation::IStreamHandler &getNewConfigStreamHandler() {
+        return _streamHandler;
+    }
+    virtual document::DocumentTypeRepo &getDeserializeRepo() {
+        return _repo;
+    }
+};
+
+
+/**
+ * Class the receives all concrete operations as part of a domain visit
+ * and prints all document operations to standard out.
+ */
+class DocumentPrinter : public OperationPrinter
+{
+private:
+    bool _printXml;
+    bool _verbose;
+
+    void printXml(const vespalib::xml::XmlSerializable &toPrint) {
+        vespalib::xml::XmlOutputStream out(std::cout);
+        toPrint.printXml(out);
+        std::cout << std::endl;
+    }
+
+    void printXml(const document::FieldValue &toPrint) {
+        vespalib::xml::XmlOutputStream out(std::cout);
+        toPrint.printXml(out);
+        std::cout << std::endl;
+    }
+
+    void printText(const document::Printable &toPrint) {
+        toPrint.print(std::cout, _verbose);
+        std::cout << std::endl;
+    }
+
+    void printText(const document::FieldValue &toPrint) {
+        toPrint.print(std::cout, _verbose);
+        std::cout << std::endl;
+    }
+
+public:
+    DocumentPrinter(DocumentTypeRepo &repo, bool printXml_, bool verbose)
+        : OperationPrinter(repo),
+          _printXml(printXml_),
+          _verbose(verbose)
+    {
+    }
+    virtual void replay(const PutOperation &op) {
+        print(op);
+        if (op.getDocument().get() != NULL) {
+            if (_printXml) {
+                printXml(*op.getDocument());
+            } else {
+                printText(*op.getDocument());
+            }
+        }
+    }
+    virtual void replay(const RemoveOperation &op) {
+        print(op);
+    }
+    virtual void replay(const UpdateOperation &op) {
+        print(op);
+        if (op.getUpdate().get() != NULL) {
+            if (_printXml) {
+                printXml(*op.getUpdate());
+            } else {
+                printText(*op.getUpdate());
+            }
+        }
+    }
+    virtual void replay(const NoopOperation &) { }
+    virtual void replay(const NewConfigOperation &) { }
+    virtual void replay(const WipeHistoryOperation &) { }
+    virtual void replay(const DeleteBucketOperation &) { }
+    virtual void replay(const SplitBucketOperation &) { }
+    virtual void replay(const JoinBucketsOperation &) { }
+    virtual void replay(const PruneRemovedDocumentsOperation &) { }
+    virtual void replay(const SpoolerReplayStartOperation &) { }
+    virtual void replay(const SpoolerReplayCompleteOperation &) { }
+    virtual void replay(const MoveOperation &) { }
+    virtual void replay(const CreateBucketOperation &) { }
+};
+
+
+/**
+ * Class that receives packets from the tls as part of a domain visit
+ * and dispatches each packet entry to the ReplayPacketDispatcher that
+ * transforms them into concrete operations.
+ */
+class VisitorCallback : public TransLogClient::Session::Callback
+{
+private:
+    ReplayPacketDispatcher _dispatcher;
+    bool _eof;
+
+public:
+    VisitorCallback(IReplayPacketHandler &handler)
+        : _dispatcher(handler),
+          _eof(false)
+    {
+    }
+    virtual RPC::Result receive(const Packet &packet) {
+        vespalib::nbostream handle(packet.getHandle().c_str(),
+                                   packet.getHandle().size(),
+                                   true);
+        try {
+            while (handle.size() > 0) {
+                Packet::Entry entry;
+                entry.deserialize(handle);
+                _dispatcher.replayEntry(entry);
+            }
+        } catch (const std::exception &e) {
+            std::cerr << "Error while handling transaction log packet: '"
+                << std::string(e.what()) << "'" << std::endl;
+            return RPC::ERROR;
+        }
+        return RPC::OK;
+    }
+    virtual void inSync() { }
+    virtual void eof() { _eof = true; }
+    bool isEof() const { return _eof; }
+};
+
+
+/**
+ * Interface for a utility.
+ */
+struct Utility
+{
+    virtual ~Utility() {}
+    typedef std::unique_ptr<Utility> UP;
+    virtual int run() = 0;
+};
+
+
+/**
+ * Base options used by a utility class.
+ */
+class BaseOptions
+{
+protected:
+    vespalib::ProgramOptions _opts;
+
+public:
+    std::string tlsDir;
+    std::string tlsName;
+    int         listenPort;
+    typedef std::unique_ptr<BaseOptions> UP;
+    BaseOptions(int argc, const char* const* argv)
+        : _opts(argc, argv)
+    {
+        _opts.addOption("tlsdir", tlsDir, "Tls directory");
+        _opts.addOption("tlsname", tlsName, std::string("tls"), "Name of the tls");
+        _opts.addOption("listenport", listenPort, 13701, "Tcp listen port");
+    }
+    virtual ~BaseOptions() {}
+    void usage() { _opts.writeSyntaxPage(std::cout); }
+    virtual void parse() { _opts.parse(); }
+    virtual std::string toString() const {
+        return vespalib::make_string("tlsdir=%s, tlsname=%s, listenport=%d",
+                                     tlsDir.c_str(), tlsName.c_str(), listenPort);
+    }
+    virtual Utility::UP createUtility() const = 0;
+};
+
+/**
+ * Base class for a utility with tls server and tls client.
+ */
+class BaseUtility : public Utility
+{
+protected:
+    const BaseOptions     &_bopts;
+    DummyFileHeaderContext _fileHeader;
+    TransLogServer         _server;
+    TransLogClient         _client;
+
+public:
+    BaseUtility(const BaseOptions &bopts)
+        : _bopts(bopts),
+          _fileHeader(),
+          _server(_bopts.tlsName, _bopts.listenPort, _bopts.tlsDir, _fileHeader),
+          _client(vespalib::make_string("tcp/localhost:%d", _bopts.listenPort))
+    {
+    }
+    virtual int run() = 0;
+};
+
+
+/**
+ * Program options used by ListDomainsUtility.
+ */
+struct ListDomainsOptions : public BaseOptions
+{
+    ListDomainsOptions(int argc, const char* const* argv)
+        : BaseOptions(argc, argv)
+    {
+        _opts.setSyntaxMessage("Utility to list all domains in a tls");
+    }
+    static std::string command() { return "listdomains"; }
+    virtual Utility::UP createUtility() const;
+};
+
+/**
+ * Utility to list all domains in a tls.
+ */
+class ListDomainsUtility : public BaseUtility
+{
+public:
+    ListDomainsUtility(const ListDomainsOptions &opts)
+        : BaseUtility(opts)
+    {
+    }
+    virtual int run() {
+        std::cout << ListDomainsOptions::command() << ": " << _bopts.toString() << std::endl;
+
+        std::vector<vespalib::string> domains;
+        _client.listDomains(domains);
+        std::cout << "Listing status for " << domains.size() << " domain(s):" << std::endl;
+        for (size_t i = 0; i < domains.size(); ++i) {
+            TransLogClient::Session::UP session = _client.open(domains[i]);
+            SerialNum first;
+            SerialNum last;
+            size_t count;
+            session->status(first, last, count);
+            std::cout << "Domain '" << domains[i] << "': first=" << first << ", last=" << last;
+            std::cout << ", count=" << count << std::endl;
+        }
+        return 0;
+    }
+};
+
+Utility::UP
+ListDomainsOptions::createUtility() const
+{
+    return Utility::UP(new ListDomainsUtility(*this));
+}
+
+
+/**
+ * Program options used by DumpOperationsUtility.
+ */
+struct DumpOperationsOptions : public BaseOptions
+{
+    std::string domainName;
+    SerialNum   firstSerialNum;
+    SerialNum   lastSerialNum;
+    std::string configDir;
+    DumpOperationsOptions(int argc, const char* const* argv)
+        : BaseOptions(argc, argv)
+    {
+        _opts.addOption("domain", domainName, "Name of the domain");
+        _opts.addOption("first", firstSerialNum, "Serial number of first operation");
+        _opts.addOption("last", lastSerialNum, "Serial number of last operation");
+        _opts.addOption("configdir", configDir, "Config directory (with documenttypes.cfg)");
+        _opts.setSyntaxMessage("Utility to dump a range of operations ([first,last]) in a tls domain");
+    }
+    static std::string command() { return "dumpoperations"; }
+    virtual std::string toString() const {
+        return vespalib::make_string("%s, domain=%s, first=%" PRIu64 ", last=%" PRIu64 ", configdir=%s",
+                                     BaseOptions::toString().c_str(), domainName.c_str(),
+                                     firstSerialNum, lastSerialNum,
+                                     configDir.c_str());
+    }
+    virtual Utility::UP createUtility() const;
+};
+
+/**
+ * Utility to dump a range of operations in a tls domain.
+ */
+class DumpOperationsUtility : public BaseUtility
+{
+protected:
+    const DumpOperationsOptions &_oopts;
+
+    virtual IReplayPacketHandlerUP createHandler(DocumentTypeRepo &repo) {
+        return IReplayPacketHandlerUP(new OperationPrinter(repo));
+    }
+
+    int doRun() {
+        DocTypeRepo repo(_oopts.configDir);
+        IReplayPacketHandlerUP handler = createHandler(repo.docTypeRepo);
+        VisitorCallback callback(*handler);
+        TransLogClient::Visitor::UP visitor = _client.createVisitor(_oopts.domainName, callback);
+        bool visitOk = visitor->visit(_oopts.firstSerialNum-1, _oopts.lastSerialNum);
+        if (!visitOk) {
+            std::cerr << "Visiting domain '" << _oopts.domainName << "' [" << _oopts.firstSerialNum << ","
+                << _oopts.lastSerialNum << "] failed" << std::endl;
+            return 1;
+        }
+        for (size_t i = 0; !callback.isEof() && (i < 60 * 60); i++ ) {
+            FastOS_Thread::Sleep(1000);
+        }
+        return 0;
+    }
+
+public:
+    DumpOperationsUtility(const DumpOperationsOptions &oopts)
+        : BaseUtility(oopts),
+        _oopts(oopts)
+    {
+    }
+    virtual int run() {
+        std::cout << DumpOperationsOptions::command() << ": " << _oopts.toString() << std::endl;
+        return doRun();
+    }
+};
+
+Utility::UP
+DumpOperationsOptions::createUtility() const
+{
+    return Utility::UP(new DumpOperationsUtility(*this));
+}
+
+
+/**
+ * Program options used by DumpDocumentsUtility.
+ */
+struct DumpDocumentsOptions : public DumpOperationsOptions
+{
+    std::string format;
+    bool verbose;
+    DumpDocumentsOptions(int argc, const char* const* argv)
+        : DumpOperationsOptions(argc, argv)
+    {
+        _opts.addOption("format", format, std::string("xml"), "Format in which the document operations should be dumped ('xml' or 'text')");
+        _opts.addOption("verbose", verbose, false, "Whether the document operations should be dumped verbosely");
+        _opts.setSyntaxMessage("Utility to dump a range of document operations ([first,last]) in a tls domain");
+    }
+    static std::string command() { return "dumpdocuments"; }
+    virtual void parse() {
+        DumpOperationsOptions::parse();
+        if (format != "xml" && format != "text") {
+            throw vespalib::InvalidCommandLineArgumentsException("Expected 'format' to be 'xml' or 'text'");
+        }
+    }
+    virtual std::string toString() const {
+        return vespalib::make_string("%s, format=%s, verbose=%s",
+                                     DumpOperationsOptions::toString().c_str(),
+                                     format.c_str(), (verbose ? "true" : "false"));
+    }
+    virtual Utility::UP createUtility() const;
+};
+
+/**
+ * Utility to dump a range of document operations in a tls domain.
+ */
+class DumpDocumentsUtility : public DumpOperationsUtility
+{
+protected:
+    const DumpDocumentsOptions &_dopts;
+    virtual IReplayPacketHandlerUP createHandler(DocumentTypeRepo &repo) {
+        return IReplayPacketHandlerUP(new DocumentPrinter(repo, _dopts.format == "xml", _dopts.verbose));
+    }
+
+public:
+    DumpDocumentsUtility(const DumpDocumentsOptions &dopts)
+        : DumpOperationsUtility(dopts),
+          _dopts(dopts)
+    {
+    }
+    virtual int run() {
+        std::cout << DumpDocumentsOptions::command() << ": " << _oopts.toString() << std::endl;
+        return doRun();
+    }
+};
+
+Utility::UP
+DumpDocumentsOptions::createUtility() const
+{
+    return Utility::UP(new DumpDocumentsUtility(*this));
+}
+
+
+/**
+ * Main application.
+ */
+class App : public FastOS_Application
+{
+private:
+    std::string _programName;
+    std::string _tmpArg;
+
+    void combineFirstArgs() {
+        _tmpArg = vespalib::make_string("%s %s", _argv[0], _argv[1]).c_str();
+        _argv[1] = &_tmpArg[0];
+    }
+    void replaceFirstArg(const std::string &replace) {
+        _tmpArg = vespalib::make_string("%s %s", _programName.c_str(), replace.c_str()).c_str();
+        _argv[0] = &_tmpArg[0];
+    }
+    void usageHeader() {
+        std::cout << _programName << " version 0.0\n";
+    }
+    void usage() {
+        usageHeader();
+        replaceFirstArg(ListDomainsOptions::command());
+        ListDomainsOptions(_argc, _argv).usage();
+        replaceFirstArg(DumpOperationsOptions::command());
+        DumpOperationsOptions(_argc, _argv).usage();
+        replaceFirstArg(DumpDocumentsOptions::command());
+        DumpDocumentsOptions(_argc, _argv).usage();
+    }
+
+public:
+    int Main() {
+        _programName = _argv[0];
+        if (_argc < 2) {
+            usage();
+            return 1;
+        }
+        BaseOptions::UP opts;
+        if (strcmp(_argv[1], ListDomainsOptions::command().c_str()) == 0) {
+            combineFirstArgs();
+            opts.reset(new ListDomainsOptions(_argc-1, _argv+1));
+        } else if (strcmp(_argv[1], DumpOperationsOptions::command().c_str()) == 0) {
+            combineFirstArgs();
+            opts.reset(new DumpOperationsOptions(_argc-1, _argv+1));
+        } else if (strcmp(_argv[1], DumpDocumentsOptions::command().c_str()) == 0) {
+            combineFirstArgs();
+            opts.reset(new DumpDocumentsOptions(_argc-1, _argv+1));
+        }
+        if (opts.get() != NULL) {
+            try {
+                opts->parse();
+            } catch (const vespalib::InvalidCommandLineArgumentsException &e) {
+                std::cerr << "Error parsing program options: " << e.getMessage() << std::endl;
+                usageHeader();
+                opts->usage();
+                return 1;
+            }
+            return opts->createUtility()->run();
+        }
+        usage();
+        return 1;
+    }
+};
+
+int
+main(int argc, char **argv)
+{
+    App app;
+    return app.Entry(argc, argv);
+}
diff --git a/searchcore/src/sample/.gitignore b/searchcore/src/sample/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/sample/.gitignore
diff --git a/searchcore/src/testlist.txt b/searchcore/src/testlist.txt
new file mode 100644
index 00000000000..b87a95c2cda
--- /dev/null
+++ b/searchcore/src/testlist.txt
@@ -0,0 +1,70 @@
+?tests/proton/proton
+tests/applyattrupdates
+tests/fdispatch/randomrow
+tests/fdispatch/search_path
+tests/grouping
+tests/proton/attribute
+tests/proton/attribute/attribute_manager
+tests/proton/attribute/attribute_populator
+tests/proton/attribute/attribute_usage_filter
+tests/proton/attribute/attributes_state_explorer
+tests/proton/attribute/document_field_populator
+tests/proton/bucketdb/bucketdb
+tests/proton/common
+tests/proton/common/document_type_inspector
+tests/proton/common/state_reporter_utils
+tests/proton/config
+tests/proton/docsummary
+tests/proton/document_iterator
+tests/proton/documentdb
+tests/proton/documentdb/buckethandler
+tests/proton/documentdb/clusterstatehandler
+tests/proton/documentdb/combiningfeedview
+tests/proton/documentdb/configurer
+tests/proton/documentdb/configvalidator
+tests/proton/documentdb/document_scan_iterator
+tests/proton/documentdb/document_subdbs
+tests/proton/documentdb/documentbucketmover
+tests/proton/documentdb/documentdbconfig
+tests/proton/documentdb/documentdbconfigscout
+tests/proton/documentdb/feedhandler
+tests/proton/documentdb/feedview
+tests/proton/documentdb/fileconfigmanager
+tests/proton/documentdb/job_tracked_maintenance_job
+tests/proton/documentdb/lid_space_compaction
+tests/proton/documentdb/maintenancecontroller
+tests/proton/documentdb/storeonlyfeedview
+tests/proton/documentmetastore
+tests/proton/documentmetastore/lidreusedelayer
+tests/proton/feed_and_search
+tests/proton/feedoperation
+tests/proton/feedtoken
+tests/proton/flushengine
+tests/proton/flushengine/prepare_restart_flush_strategy
+tests/proton/index
+tests/proton/index/index_writer
+tests/proton/initializer
+tests/proton/matchengine
+tests/proton/matching
+tests/proton/matching/docid_range_scheduler
+tests/proton/matching/match_loop_communicator
+tests/proton/matching/match_phase_limiter
+tests/proton/matching/partial_result
+tests/proton/metrics/documentdb_job_trackers
+tests/proton/metrics/job_load_sampler
+tests/proton/metrics/job_tracked_flush
+tests/proton/metrics/metrics_engine
+tests/proton/persistenceconformance
+tests/proton/persistenceengine
+tests/proton/reprocessing/attribute_reprocessing_initializer
+tests/proton/reprocessing/document_reprocessing_handler
+tests/proton/reprocessing/reprocessing_runner
+tests/proton/server
+tests/proton/server/data_directory_upgrader
+tests/proton/server/disk_mem_usage_filter
+tests/proton/server/health_adapter
+tests/proton/server/memoryflush
+tests/proton/server/visibility_handler
+tests/proton/statusreport
+tests/proton/summaryengine
+tests/proton/verify_ranksetup
diff --git a/searchcore/src/tests/.gitignore b/searchcore/src/tests/.gitignore
new file mode 100644
index 00000000000..a3e9c375723
--- /dev/null
+++ b/searchcore/src/tests/.gitignore
@@ -0,0 +1,3 @@
+.depend
+Makefile
+*_test
diff --git a/searchcore/src/tests/applyattrupdates/.gitignore b/searchcore/src/tests/applyattrupdates/.gitignore
new file mode 100644
index 00000000000..b7789427c09
--- /dev/null
+++ b/searchcore/src/tests/applyattrupdates/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+applyattrupdates_test
+searchcore_applyattrupdates_test_app
diff --git a/searchcore/src/tests/applyattrupdates/CMakeLists.txt b/searchcore/src/tests/applyattrupdates/CMakeLists.txt
new file mode 100644
index 00000000000..2778d0f62dc
--- /dev/null
+++ b/searchcore/src/tests/applyattrupdates/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_applyattrupdates_test_app
+    SOURCES
+    applyattrupdates.cpp
+    DEPENDS
+    searchcore_pcommon
+    searchcore_util
+)
+vespa_add_test(NAME searchcore_applyattrupdates_test_app COMMAND searchcore_applyattrupdates_test_app)
diff --git a/searchcore/src/tests/applyattrupdates/applyattrupdates.cpp b/searchcore/src/tests/applyattrupdates/applyattrupdates.cpp
new file mode 100644
index 00000000000..bc1f44740da
--- /dev/null
+++ b/searchcore/src/tests/applyattrupdates/applyattrupdates.cpp
@@ -0,0 +1,338 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/update/addvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/update/clearvalueupdate.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/update/removevalueupdate.h>
+#include <vespa/log/log.h>
+#include <vespa/searchcore/proton/common/attrupdate.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/vespalib/testkit/testapp.h>
+
+LOG_SETUP("applyattrupdates_test");
+
+using namespace document;
+using search::attribute::BasicType;
+using search::attribute::Config;
+using search::attribute::CollectionType;
+
+namespace search {
+
+//-----------------------------------------------------------------------------
+
+template <typename T>
+class Vector
+{
+private:
+    std::vector<T> _vec;
+public:
+    Vector() : _vec() {}
+    size_t size() const {
+        return _vec.size();
+    }
+    Vector & pb(const T & val) {
+        _vec.push_back(val);
+        return *this;
+    }
+    const T & operator [] (size_t idx) const {
+        return _vec[idx];
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+typedef AttributeVector::SP AttributePtr;
+typedef AttributeVector::WeightedInt WeightedInt;
+typedef AttributeVector::WeightedFloat WeightedFloat;
+typedef AttributeVector::WeightedString WeightedString;
+
+class Test : public vespalib::TestApp
+{
+private:
+    template <typename T, typename VectorType>
+    AttributePtr
+    create(uint32_t numDocs, T val, int32_t weight,
+           const std::string & baseName,
+           const Config &info)
+    {
+        LOG(info, "create attribute vector: %s", baseName.c_str());
+        AttributePtr vec = AttributeFactory::createAttribute(baseName, info);
+        VectorType * api = static_cast<VectorType *>(vec.get());
+        for (uint32_t i = 0; i < numDocs; ++i) {
+            if (!api->addDoc(i)) {
+                LOG(info, "failed adding doc: %u", i);
+                return AttributePtr();
+            }
+            if (api->hasMultiValue()) {
+                if (!api->append(i, val, weight)) {
+                    LOG(info, "failed append to doc: %u", i);
+                }
+            } else {
+                if (!api->update(i, val)) {
+                    LOG(info, "failed update doc: %u", i);
+                    return AttributePtr();
+                }
+            }
+        }
+        api->commit();
+        return vec;
+    }
+
+    template <typename T>
+    bool check(const AttributePtr & vec, uint32_t docId, const Vector<T> & values) {
+        uint32_t sz = vec->getValueCount(docId);
+        if (!EXPECT_EQUAL(sz, values.size())) return false;
+        std::vector<T> buf(sz);
+        uint32_t asz = vec->get(docId, &buf[0], sz);
+        if (!EXPECT_EQUAL(sz, asz)) return false;
+        for (uint32_t i = 0; i < values.size(); ++i) {
+            if (!EXPECT_EQUAL(buf[i].getValue(), values[i].getValue())) return false;
+            if (!EXPECT_EQUAL(buf[i].getWeight(), values[i].getWeight())) return false;
+        }
+        return true;
+    }
+
+    void applyValueUpdate(AttributeVector & vec, uint32_t docId, const ValueUpdate & upd) {
+        FieldUpdate fupd(_docType->getField(vec.getName()));
+        fupd.addUpdate(upd);
+        search::AttrUpdate::handleUpdate(vec, docId, fupd);
+        vec.commit();
+    }
+
+    void applyArrayUpdates(AttributeVector & vec, const FieldValue & assign,
+                           const FieldValue & first, const FieldValue & second) {
+        applyValueUpdate(vec, 0, AssignValueUpdate(assign));
+        applyValueUpdate(vec, 1, AddValueUpdate(second));
+        applyValueUpdate(vec, 2, RemoveValueUpdate(first));
+        applyValueUpdate(vec, 3, ClearValueUpdate());
+    }
+
+    void applyWeightedSetUpdates(AttributeVector & vec, const FieldValue & assign,
+                                 const FieldValue & first, const FieldValue & second) {
+        applyValueUpdate(vec, 0, AssignValueUpdate(assign));
+        applyValueUpdate(vec, 1, AddValueUpdate(second, 20));
+        applyValueUpdate(vec, 2, RemoveValueUpdate(first));
+        applyValueUpdate(vec, 3, ClearValueUpdate());
+        ArithmeticValueUpdate arithmetic(ArithmeticValueUpdate::Add, 10);
+        applyValueUpdate(vec, 4, MapValueUpdate(first, arithmetic));
+    }
+
+    void requireThatSingleAttributesAreUpdated();
+    void requireThatArrayAttributesAreUpdated();
+    void requireThatWeightedSetAttributesAreUpdated();
+
+    DocumentTypeRepo _repo;
+    const DocumentType* _docType;
+
+public:
+    Test();
+    int Main();
+};
+
+void
+Test::requireThatSingleAttributesAreUpdated()
+{
+    using search::attribute::getUndefined;
+    CollectionType ct(CollectionType::SINGLE);
+    {
+        BasicType bt(BasicType::INT32);
+        AttributePtr vec = create<int32_t, IntegerAttribute>(3, 32, 0,
+                                                             "in1/int",
+                                                             Config(bt, ct));
+        applyValueUpdate(*vec, 0, AssignValueUpdate(IntFieldValue(64)));
+        applyValueUpdate(*vec, 1, ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10));
+        applyValueUpdate(*vec, 2, ClearValueUpdate());
+        EXPECT_EQUAL(3u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedInt>().pb(WeightedInt(64))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedInt>().pb(WeightedInt(42))));
+        EXPECT_TRUE(check(vec, 2, Vector<WeightedInt>().pb(WeightedInt(getUndefined<int32_t>()))));
+    }
+    {
+        BasicType bt(BasicType::FLOAT);
+        AttributePtr vec = create<float, FloatingPointAttribute>(3, 55.5f, 0,
+                                                                 "in1/float",
+                                                                 Config(bt,
+                                                                        ct));
+        applyValueUpdate(*vec, 0, AssignValueUpdate(FloatFieldValue(77.7f)));
+        applyValueUpdate(*vec, 1, ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10));
+        applyValueUpdate(*vec, 2, ClearValueUpdate());
+        EXPECT_EQUAL(3u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedFloat>().pb(WeightedFloat(77.7f))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedFloat>().pb(WeightedFloat(65.5f))));
+        EXPECT_TRUE(std::isnan(vec->getFloat(2)));
+    }
+    {
+        BasicType bt(BasicType::STRING);
+        AttributePtr vec = create<std::string, StringAttribute>(3, "first", 0,
+                                                                "in1/string",
+                                                                Config(bt,
+                                                                       ct));
+        applyValueUpdate(*vec, 0, AssignValueUpdate(StringFieldValue("second")));
+        applyValueUpdate(*vec, 2, ClearValueUpdate());
+        EXPECT_EQUAL(3u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedString>().pb(WeightedString("second"))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedString>().pb(WeightedString("first"))));
+        EXPECT_TRUE(check(vec, 2, Vector<WeightedString>().pb(WeightedString(""))));
+    }
+}
+
+void
+Test::requireThatArrayAttributesAreUpdated()
+{
+    CollectionType ct(CollectionType::ARRAY);
+    {
+        BasicType bt(BasicType::INT32);
+        AttributePtr vec = create<int32_t, IntegerAttribute>(5, 32, 1,
+                                                             "in1/aint",
+                                                             Config(bt, ct));
+        IntFieldValue first(32);
+        IntFieldValue second(64);
+        ArrayFieldValue assign(_docType->getField("aint").getDataType());
+        assign.add(second);
+        applyArrayUpdates(*vec, assign, first, second);
+
+        EXPECT_EQUAL(5u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedInt>().pb(WeightedInt(64))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedInt>().pb(WeightedInt(32)).pb(WeightedInt(64))));
+        EXPECT_TRUE(check(vec, 2, Vector<WeightedInt>()));
+        EXPECT_TRUE(check(vec, 3, Vector<WeightedInt>()));
+        EXPECT_TRUE(check(vec, 4, Vector<WeightedInt>().pb(WeightedInt(32))));
+    }
+    {
+        BasicType bt(BasicType::FLOAT);
+        AttributePtr vec = create<float, FloatingPointAttribute>(5, 55.5f, 1,
+                                                                 "in1/afloat",
+                                                                 Config(bt,
+                                                                        ct));
+        FloatFieldValue first(55.5f);
+        FloatFieldValue second(77.7f);
+        ArrayFieldValue assign(_docType->getField("afloat").getDataType());
+        assign.add(second);
+        applyArrayUpdates(*vec, assign, first, second);
+
+        EXPECT_EQUAL(5u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedFloat>().pb(WeightedFloat(77.7f))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedFloat>().pb(WeightedFloat(55.5f)).pb(WeightedFloat(77.7f))));
+        EXPECT_TRUE(check(vec, 2, Vector<WeightedFloat>()));
+        EXPECT_TRUE(check(vec, 3, Vector<WeightedFloat>()));
+        EXPECT_TRUE(check(vec, 4, Vector<WeightedFloat>().pb(WeightedFloat(55.5f))));
+    }
+    {
+        BasicType bt(BasicType::STRING);
+        AttributePtr vec = create<std::string, StringAttribute>(5, "first", 1,
+                                                                "in1/astring",
+                                                                Config(bt, ct));
+        StringFieldValue first("first");
+        StringFieldValue second("second");
+        ArrayFieldValue assign(_docType->getField("astring").getDataType());
+        assign.add(second);
+        applyArrayUpdates(*vec, assign, first, second);
+
+        EXPECT_EQUAL(5u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedString>().pb(WeightedString("second"))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedString>().pb(WeightedString("first")).pb(WeightedString("second"))));
+        EXPECT_TRUE(check(vec, 2, Vector<WeightedString>()));
+        EXPECT_TRUE(check(vec, 3, Vector<WeightedString>()));
+        EXPECT_TRUE(check(vec, 4, Vector<WeightedString>().pb(WeightedString("first"))));
+    }
+}
+
+void
+Test::requireThatWeightedSetAttributesAreUpdated()
+{
+    CollectionType ct(CollectionType::WSET);
+    {
+        BasicType bt(BasicType::INT32);
+        AttributePtr vec = create<int32_t, IntegerAttribute>(5, 32, 100,
+                                                             "in1/wsint",
+                                                             Config(bt, ct));
+        IntFieldValue first(32);
+        IntFieldValue second(64);
+        WeightedSetFieldValue
+            assign(_docType->getField("wsint").getDataType());
+        assign.add(second, 20);
+        applyWeightedSetUpdates(*vec, assign, first, second);
+
+        EXPECT_EQUAL(5u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedInt>().pb(WeightedInt(64, 20))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedInt>().pb(WeightedInt(32, 100)).pb(WeightedInt(64, 20))));
+        EXPECT_TRUE(check(vec, 2, Vector<WeightedInt>()));
+        EXPECT_TRUE(check(vec, 3, Vector<WeightedInt>()));
+        EXPECT_TRUE(check(vec, 4, Vector<WeightedInt>().pb(WeightedInt(32, 110))));
+    }
+    {
+        BasicType bt(BasicType::FLOAT);
+        AttributePtr vec = create<float, FloatingPointAttribute>(5, 55.5f, 100,
+                                                                 "in1/wsfloat",
+                                                                 Config(bt,
+                                                                        ct));
+        FloatFieldValue first(55.5f);
+        FloatFieldValue second(77.7f);
+        WeightedSetFieldValue
+            assign(_docType->getField("wsfloat").getDataType());
+        assign.add(second, 20);
+        applyWeightedSetUpdates(*vec, assign, first, second);
+
+        EXPECT_EQUAL(5u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedFloat>().pb(WeightedFloat(77.7f, 20))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedFloat>().pb(WeightedFloat(55.5f, 100)).pb(WeightedFloat(77.7f, 20))));
+        EXPECT_TRUE(check(vec, 2, Vector<WeightedFloat>()));
+        EXPECT_TRUE(check(vec, 3, Vector<WeightedFloat>()));
+        EXPECT_TRUE(check(vec, 4, Vector<WeightedFloat>().pb(WeightedFloat(55.5f, 110))));
+    }
+    {
+        BasicType bt(BasicType::STRING);
+        AttributePtr vec = create<std::string, StringAttribute>(5, "first",
+                                                                100,
+                                                                "in1/wsstring",
+                                                                Config(bt,
+                                                                       ct));
+        StringFieldValue first("first");
+        StringFieldValue second("second");
+        WeightedSetFieldValue
+            assign(_docType->getField("wsstring").getDataType());
+        assign.add(second, 20);
+        applyWeightedSetUpdates(*vec, assign, first, second);
+
+        EXPECT_EQUAL(5u, vec->getNumDocs());
+        EXPECT_TRUE(check(vec, 0, Vector<WeightedString>().pb(WeightedString("second", 20))));
+        EXPECT_TRUE(check(vec, 1, Vector<WeightedString>().pb(WeightedString("first", 100)).pb(WeightedString("second", 20))));
+        EXPECT_TRUE(check(vec, 2, Vector<WeightedString>()));
+        EXPECT_TRUE(check(vec, 3, Vector<WeightedString>()));
+        EXPECT_TRUE(check(vec, 4, Vector<WeightedString>().pb(WeightedString("first", 110))));
+    }
+}
+
+Test::Test()
+    : _repo(readDocumenttypesConfig("doctypes.cfg")),
+      _docType(_repo.getDocumentType("testdoc"))
+{
+}
+
+int
+Test::Main()
+{
+    TEST_INIT("applyattrupdates_test");
+
+    TEST_DO(requireThatSingleAttributesAreUpdated());
+    TEST_DO(requireThatArrayAttributesAreUpdated());
+    TEST_DO(requireThatWeightedSetAttributesAreUpdated());
+
+    TEST_DONE();
+}
+
+} // namespace search
+
+TEST_APPHOOK(search::Test);
diff --git a/searchcore/src/tests/applyattrupdates/doctypes.cfg b/searchcore/src/tests/applyattrupdates/doctypes.cfg
new file mode 100644
index 00000000000..23cbf06629e
--- /dev/null
+++ b/searchcore/src/tests/applyattrupdates/doctypes.cfg
@@ -0,0 +1,174 @@
+enablecompression false
+documenttype[1]
+documenttype[0].id -1175657560
+documenttype[0].name "testdoc"
+documenttype[0].version 0
+documenttype[0].headerstruct -1636745577
+documenttype[0].bodystruct 1878320748
+documenttype[0].inherits[0]
+documenttype[0].datatype[8]
+documenttype[0].datatype[0].id 100
+documenttype[0].datatype[0].type ARRAY
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name ""
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 95
+documenttype[0].datatype[0].sstruct.compression.minsize 200
+documenttype[0].datatype[0].sstruct.field[0]
+documenttype[0].datatype[1].id 101
+documenttype[0].datatype[1].type ARRAY
+documenttype[0].datatype[1].array.element.id 1
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name ""
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 95
+documenttype[0].datatype[1].sstruct.compression.minsize 200
+documenttype[0].datatype[1].sstruct.field[0]
+documenttype[0].datatype[2].id 102
+documenttype[0].datatype[2].type ARRAY
+documenttype[0].datatype[2].array.element.id 2
+documenttype[0].datatype[2].map.key.id 0
+documenttype[0].datatype[2].map.value.id 0
+documenttype[0].datatype[2].wset.key.id 0
+documenttype[0].datatype[2].wset.createifnonexistent false
+documenttype[0].datatype[2].wset.removeifzero false
+documenttype[0].datatype[2].annotationref.annotation.id 0
+documenttype[0].datatype[2].sstruct.name ""
+documenttype[0].datatype[2].sstruct.version 0
+documenttype[0].datatype[2].sstruct.compression.type NONE
+documenttype[0].datatype[2].sstruct.compression.level 0
+documenttype[0].datatype[2].sstruct.compression.threshold 95
+documenttype[0].datatype[2].sstruct.compression.minsize 200
+documenttype[0].datatype[2].sstruct.field[0]
+documenttype[0].datatype[3].id 200
+documenttype[0].datatype[3].type WSET
+documenttype[0].datatype[3].array.element.id 0
+documenttype[0].datatype[3].map.key.id 0
+documenttype[0].datatype[3].map.value.id 0
+documenttype[0].datatype[3].wset.key.id 0
+documenttype[0].datatype[3].wset.createifnonexistent false
+documenttype[0].datatype[3].wset.removeifzero false
+documenttype[0].datatype[3].annotationref.annotation.id 0
+documenttype[0].datatype[3].sstruct.name ""
+documenttype[0].datatype[3].sstruct.version 0
+documenttype[0].datatype[3].sstruct.compression.type NONE
+documenttype[0].datatype[3].sstruct.compression.level 0
+documenttype[0].datatype[3].sstruct.compression.threshold 95
+documenttype[0].datatype[3].sstruct.compression.minsize 200
+documenttype[0].datatype[3].sstruct.field[0]
+documenttype[0].datatype[4].id 201
+documenttype[0].datatype[4].type WSET
+documenttype[0].datatype[4].array.element.id 0
+documenttype[0].datatype[4].map.key.id 0
+documenttype[0].datatype[4].map.value.id 0
+documenttype[0].datatype[4].wset.key.id 1
+documenttype[0].datatype[4].wset.createifnonexistent false
+documenttype[0].datatype[4].wset.removeifzero false
+documenttype[0].datatype[4].annotationref.annotation.id 0
+documenttype[0].datatype[4].sstruct.name ""
+documenttype[0].datatype[4].sstruct.version 0
+documenttype[0].datatype[4].sstruct.compression.type NONE
+documenttype[0].datatype[4].sstruct.compression.level 0
+documenttype[0].datatype[4].sstruct.compression.threshold 95
+documenttype[0].datatype[4].sstruct.compression.minsize 200
+documenttype[0].datatype[4].sstruct.field[0]
+documenttype[0].datatype[5].id 202
+documenttype[0].datatype[5].type WSET
+documenttype[0].datatype[5].array.element.id 0
+documenttype[0].datatype[5].map.key.id 0
+documenttype[0].datatype[5].map.value.id 0
+documenttype[0].datatype[5].wset.key.id 2
+documenttype[0].datatype[5].wset.createifnonexistent false
+documenttype[0].datatype[5].wset.removeifzero false
+documenttype[0].datatype[5].annotationref.annotation.id 0
+documenttype[0].datatype[5].sstruct.name ""
+documenttype[0].datatype[5].sstruct.version 0
+documenttype[0].datatype[5].sstruct.compression.type NONE
+documenttype[0].datatype[5].sstruct.compression.level 0
+documenttype[0].datatype[5].sstruct.compression.threshold 95
+documenttype[0].datatype[5].sstruct.compression.minsize 200
+documenttype[0].datatype[5].sstruct.field[0]
+documenttype[0].datatype[6].id -1636745577
+documenttype[0].datatype[6].type STRUCT
+documenttype[0].datatype[6].array.element.id 0
+documenttype[0].datatype[6].map.key.id 0
+documenttype[0].datatype[6].map.value.id 0
+documenttype[0].datatype[6].wset.key.id 0
+documenttype[0].datatype[6].wset.createifnonexistent false
+documenttype[0].datatype[6].wset.removeifzero false
+documenttype[0].datatype[6].annotationref.annotation.id 0
+documenttype[0].datatype[6].sstruct.name "testdoc.header"
+documenttype[0].datatype[6].sstruct.version 0
+documenttype[0].datatype[6].sstruct.compression.type NONE
+documenttype[0].datatype[6].sstruct.compression.level 0
+documenttype[0].datatype[6].sstruct.compression.threshold 90
+documenttype[0].datatype[6].sstruct.compression.minsize 0
+documenttype[0].datatype[6].sstruct.field[9]
+documenttype[0].datatype[6].sstruct.field[0].name "afloat"
+documenttype[0].datatype[6].sstruct.field[0].id 401182245
+documenttype[0].datatype[6].sstruct.field[0].id_v6 303812879
+documenttype[0].datatype[6].sstruct.field[0].datatype 101
+documenttype[0].datatype[6].sstruct.field[1].name "aint"
+documenttype[0].datatype[6].sstruct.field[1].id 19542829
+documenttype[0].datatype[6].sstruct.field[1].id_v6 764769238
+documenttype[0].datatype[6].sstruct.field[1].datatype 100
+documenttype[0].datatype[6].sstruct.field[2].name "astring"
+documenttype[0].datatype[6].sstruct.field[2].id 1494118564
+documenttype[0].datatype[6].sstruct.field[2].id_v6 1745177607
+documenttype[0].datatype[6].sstruct.field[2].datatype 102
+documenttype[0].datatype[6].sstruct.field[3].name "float"
+documenttype[0].datatype[6].sstruct.field[3].id 151686688
+documenttype[0].datatype[6].sstruct.field[3].id_v6 827904364
+documenttype[0].datatype[6].sstruct.field[3].datatype 1
+documenttype[0].datatype[6].sstruct.field[4].name "int"
+documenttype[0].datatype[6].sstruct.field[4].id 123383020
+documenttype[0].datatype[6].sstruct.field[4].id_v6 2014709351
+documenttype[0].datatype[6].sstruct.field[4].datatype 0
+documenttype[0].datatype[6].sstruct.field[5].name "string"
+documenttype[0].datatype[6].sstruct.field[5].id 1572342091
+documenttype[0].datatype[6].sstruct.field[5].id_v6 1847335717
+documenttype[0].datatype[6].sstruct.field[5].datatype 2
+documenttype[0].datatype[6].sstruct.field[6].name "wsfloat"
+documenttype[0].datatype[6].sstruct.field[6].id 821634779
+documenttype[0].datatype[6].sstruct.field[6].id_v6 1168403784
+documenttype[0].datatype[6].sstruct.field[6].datatype 201
+documenttype[0].datatype[6].sstruct.field[7].name "wsint"
+documenttype[0].datatype[6].sstruct.field[7].id 1160390473
+documenttype[0].datatype[6].sstruct.field[7].id_v6 1177062897
+documenttype[0].datatype[6].sstruct.field[7].datatype 200
+documenttype[0].datatype[6].sstruct.field[8].name "wsstring"
+documenttype[0].datatype[6].sstruct.field[8].id 981031285
+documenttype[0].datatype[6].sstruct.field[8].id_v6 682978193
+documenttype[0].datatype[6].sstruct.field[8].datatype 202
+documenttype[0].datatype[7].id 1878320748
+documenttype[0].datatype[7].type STRUCT
+documenttype[0].datatype[7].array.element.id 0
+documenttype[0].datatype[7].map.key.id 0
+documenttype[0].datatype[7].map.value.id 0
+documenttype[0].datatype[7].wset.key.id 0
+documenttype[0].datatype[7].wset.createifnonexistent false
+documenttype[0].datatype[7].wset.removeifzero false
+documenttype[0].datatype[7].annotationref.annotation.id 0
+documenttype[0].datatype[7].sstruct.name "testdoc.body"
+documenttype[0].datatype[7].sstruct.version 0
+documenttype[0].datatype[7].sstruct.compression.type NONE
+documenttype[0].datatype[7].sstruct.compression.level 0
+documenttype[0].datatype[7].sstruct.compression.threshold 90
+documenttype[0].datatype[7].sstruct.compression.minsize 0
+documenttype[0].datatype[7].sstruct.field[0]
+documenttype[0].annotationtype[0]
diff --git a/searchcore/src/tests/fdispatch/randomrow/.gitignore b/searchcore/src/tests/fdispatch/randomrow/.gitignore
new file mode 100644
index 00000000000..bfe075b287a
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/randomrow/.gitignore
@@ -0,0 +1 @@
+searchcore_randomrow_test_app
diff --git a/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt b/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt
new file mode 100644
index 00000000000..f3ad936ded2
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_randomrow_test_app
+    SOURCES
+    randomrow_test.cpp
+    DEPENDS
+    searchcore_fdispatch_search
+    searchcore_util
+    searchcore_fdcommon
+)
+vespa_add_test(NAME searchcore_randomrow_test_app COMMAND searchcore_randomrow_test_app)
diff --git a/searchcore/src/tests/fdispatch/randomrow/DESC b/searchcore/src/tests/fdispatch/randomrow/DESC
new file mode 100644
index 00000000000..86d8eecd44c
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/randomrow/DESC
@@ -0,0 +1 @@
+randomrow test. Take a look at randomrow_test.cpp for details.
diff --git a/searchcore/src/tests/fdispatch/randomrow/FILES b/searchcore/src/tests/fdispatch/randomrow/FILES
new file mode 100644
index 00000000000..2f15498219f
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/randomrow/FILES
@@ -0,0 +1 @@
+randomrow_test.cpp
diff --git a/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp b/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp
new file mode 100644
index 00000000000..375afee1777
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("randomrow_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/fdispatch/search/plain_dataset.h>
+
+using fdispatch::StateOfRows;
+
+TEST("requireThatEmpyStateReturnsRowZero")
+{
+    StateOfRows s(1, 1.0, 1000);
+    EXPECT_EQUAL(0u, s.getRandomWeightedRow());
+    EXPECT_EQUAL(1.0, s.getRowState(0).getAverageSearchTime());
+}
+
+TEST("requireThatDecayWorks")
+{
+    StateOfRows s(1, 1.0, 1000);
+    s.updateSearchTime(1.0, 0);
+    EXPECT_EQUAL(1.0, s.getRowState(0).getAverageSearchTime());
+    s.updateSearchTime(2.0, 0);
+    EXPECT_EQUAL(1.001, s.getRowState(0).getAverageSearchTime());
+    s.updateSearchTime(2.0, 0);
+    EXPECT_APPROX(1.002, s.getRowState(0).getAverageSearchTime(), 0.0001);
+    s.updateSearchTime(0.1, 0);
+    s.updateSearchTime(0.1, 0);
+    s.updateSearchTime(0.1, 0);
+    s.updateSearchTime(0.1, 0);
+    EXPECT_APPROX(0.998396, s.getRowState(0).getAverageSearchTime(), 0.000001);
+}
+
+TEST("requireWeightedSelectionWorks")
+{
+    StateOfRows s(5, 1.0, 1000);
+    EXPECT_EQUAL(0u, s.getWeightedNode(-0.1));
+    EXPECT_EQUAL(0u, s.getWeightedNode(0.0));
+    EXPECT_EQUAL(0u, s.getWeightedNode(0.1));
+    EXPECT_EQUAL(1u, s.getWeightedNode(0.2));
+    EXPECT_EQUAL(1u, s.getWeightedNode(0.39));
+    EXPECT_EQUAL(2u, s.getWeightedNode(0.4));
+    EXPECT_EQUAL(3u, s.getWeightedNode(0.6));
+    EXPECT_EQUAL(4u, s.getWeightedNode(0.8));
+    EXPECT_EQUAL(4u, s.getWeightedNode(2.0));
+}
+
+TEST("requireWeightedSelectionWorksFineWithDifferentWeights")
+{
+    StateOfRows s(5, 1.0, 1000);
+    s.getRowState(0).setAverageSearchTime(0.1);
+    s.getRowState(1).setAverageSearchTime(0.2);
+    s.getRowState(2).setAverageSearchTime(0.3);
+    s.getRowState(3).setAverageSearchTime(0.4);
+    s.getRowState(4).setAverageSearchTime(0.5);
+    EXPECT_EQUAL(0.1, s.getRowState(0).getAverageSearchTime());
+    EXPECT_EQUAL(0.2, s.getRowState(1).getAverageSearchTime());
+    EXPECT_EQUAL(0.3, s.getRowState(2).getAverageSearchTime());
+    EXPECT_EQUAL(0.4, s.getRowState(3).getAverageSearchTime());
+    EXPECT_EQUAL(0.5, s.getRowState(4).getAverageSearchTime());
+    EXPECT_EQUAL(0u, s.getWeightedNode(-0.1));
+    EXPECT_EQUAL(0u, s.getWeightedNode(0.0));
+    EXPECT_EQUAL(0u, s.getWeightedNode(0.4379));
+    EXPECT_EQUAL(1u, s.getWeightedNode(0.4380));
+    EXPECT_EQUAL(1u, s.getWeightedNode(0.6569));
+    EXPECT_EQUAL(2u, s.getWeightedNode(0.6570));
+    EXPECT_EQUAL(2u, s.getWeightedNode(0.8029));
+    EXPECT_EQUAL(3u, s.getWeightedNode(0.8030));
+    EXPECT_EQUAL(3u, s.getWeightedNode(0.9124));
+    EXPECT_EQUAL(4u, s.getWeightedNode(0.9125));
+    EXPECT_EQUAL(4u, s.getWeightedNode(2.0));
+}
+
+TEST("require randomness")
+{
+    StateOfRows s(3, 1.0, 1000);
+    s.getRowState(0).setAverageSearchTime(1.0);
+    s.getRowState(1).setAverageSearchTime(1.0);
+    s.getRowState(2).setAverageSearchTime(1.0);
+    size_t counts[3] = {0,0,0};
+    for (size_t i(0); i < 1000; i++) {
+        counts[s.getRandomWeightedRow()]++;
+    }
+    EXPECT_EQUAL(322ul, counts[0]);
+    EXPECT_EQUAL(345ul, counts[1]);
+    EXPECT_EQUAL(333ul, counts[2]);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/fdispatch/search_path/.gitignore b/searchcore/src/tests/fdispatch/search_path/.gitignore
new file mode 100644
index 00000000000..7452ecf3ecc
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/search_path/.gitignore
@@ -0,0 +1 @@
+searchcore_search_path_test_app
diff --git a/searchcore/src/tests/fdispatch/search_path/CMakeLists.txt b/searchcore/src/tests/fdispatch/search_path/CMakeLists.txt
new file mode 100644
index 00000000000..86067faa4cc
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/search_path/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_search_path_test_app
+    SOURCES
+    search_path_test.cpp
+    DEPENDS
+    searchcore_fdispatch_search
+)
+vespa_add_test(NAME searchcore_search_path_test_app COMMAND searchcore_search_path_test_app)
diff --git a/searchcore/src/tests/fdispatch/search_path/DESC b/searchcore/src/tests/fdispatch/search_path/DESC
new file mode 100644
index 00000000000..4bc24883896
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/search_path/DESC
@@ -0,0 +1 @@
+search_path test. Take a look at search_path_test.cpp for details.
diff --git a/searchcore/src/tests/fdispatch/search_path/FILES b/searchcore/src/tests/fdispatch/search_path/FILES
new file mode 100644
index 00000000000..a38e13c26fd
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/search_path/FILES
@@ -0,0 +1 @@
+search_path_test.cpp
diff --git a/searchcore/src/tests/fdispatch/search_path/search_path_test.cpp b/searchcore/src/tests/fdispatch/search_path/search_path_test.cpp
new file mode 100644
index 00000000000..8dd3ada4270
--- /dev/null
+++ b/searchcore/src/tests/fdispatch/search_path/search_path_test.cpp
@@ -0,0 +1,124 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("search_path_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/fdispatch/search/search_path.h>
+#include <vespa/searchcore/fdispatch/search/fnet_search.h>
+#include <iostream>
+
+using namespace fdispatch;
+
+template <typename T>
+vespalib::string
+toString(const T &val)
+{
+    std::ostringstream oss;
+    oss << "[";
+    bool first = true;
+    for (auto v : val) {
+        if (!first) oss << ",";
+        oss << v;
+        first = false;
+    }
+    oss << "]";
+    return oss.str();
+}
+
+void
+assertParts(const std::vector<size_t> &exp, const SearchPath::NodeList &act)
+{
+    std::string expStr = toString(exp);
+    std::string actStr = toString(act);
+    std::cout << "assertParts(" << expStr << "," << actStr << ")" << std::endl;
+    EXPECT_EQUAL(expStr, actStr);
+}
+
+void
+assertElement(const std::vector<size_t> &parts, size_t row, const SearchPath::Element &elem)
+{
+    assertParts(parts, elem.nodes());
+    EXPECT_TRUE(elem.hasRow());
+    EXPECT_EQUAL(row, elem.row());
+}
+
+void
+assertElement(const std::vector<size_t> &parts, const SearchPath::Element &elem)
+{
+    assertParts(parts, elem.nodes());
+    EXPECT_FALSE(elem.hasRow());
+}
+
+void
+assertSinglePath(const std::vector<size_t> &parts, const vespalib::string &spec, size_t numNodes=0)
+{
+    SearchPath p(spec, numNodes);
+    EXPECT_EQUAL(1u, p.elements().size());
+    assertElement(parts, p.elements().front());
+}
+
+void
+assertSinglePath(const std::vector<size_t> &parts, size_t row, const vespalib::string &spec, size_t numNodes=0)
+{
+    SearchPath p(spec, numNodes);
+    EXPECT_EQUAL(1u, p.elements().size());
+    assertElement(parts, row, p.elements().front());
+}
+
+TEST("requireThatSinglePartCanBeSpecified")
+{
+    assertSinglePath({0}, "0/");
+}
+
+TEST("requireThatMultiplePartsCanBeSpecified")
+{
+    assertSinglePath({1,3,5}, "1,3,5/");
+}
+
+TEST("requireThatRangePartsCanBeSpecified")
+{
+    assertSinglePath({1,2,3}, "[1,4>/", 6);
+}
+
+TEST("requireThatAllPartsCanBeSpecified")
+{
+    assertSinglePath({0,1,2,3}, "*/", 4);
+}
+
+TEST("requireThatRowCanBeSpecified")
+{
+    assertSinglePath({1}, 2, "1/2");
+}
+
+TEST("requireThatMultipleSimpleElementsCanBeSpecified")
+{
+    SearchPath p("0/1;2/3", 3);
+    EXPECT_EQUAL(2u, p.elements().size());
+    assertElement({0}, 1, p.elements()[0]);
+    assertElement({2}, 3, p.elements()[1]);
+}
+
+TEST("requireThatMultipleComplexElementsCanBeSpecified")
+{
+    SearchPath p("0,2,4/1;1,3,5/3", 6);
+    EXPECT_EQUAL(2u, p.elements().size());
+    assertElement({0,2,4}, 1, p.elements()[0]);
+    assertElement({1,3,5}, 3, p.elements()[1]);
+}
+
+TEST("requireThatMultipleElementsWithoutRowsCanBeSpecified")
+{
+    SearchPath p("0/;1/", 2);
+    EXPECT_EQUAL(2u, p.elements().size());
+    assertElement({0}, p.elements()[0]);
+    assertElement({1}, p.elements()[1]);
+}
+
+TEST("require that sizeof FastS_FNET_SearchNode is reasonable")
+{
+    EXPECT_EQUAL(240u, sizeof(FastS_FNET_SearchNode));
+    EXPECT_EQUAL(40u, sizeof(search::common::SortDataIterator));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/grouping/.gitignore b/searchcore/src/tests/grouping/.gitignore
new file mode 100644
index 00000000000..940fd49d20b
--- /dev/null
+++ b/searchcore/src/tests/grouping/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+grouping_test
+searchcore_grouping_test_app
diff --git a/searchcore/src/tests/grouping/CMakeLists.txt b/searchcore/src/tests/grouping/CMakeLists.txt
new file mode 100644
index 00000000000..245406187f6
--- /dev/null
+++ b/searchcore/src/tests/grouping/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_grouping_test_app
+    SOURCES
+    grouping.cpp
+    DEPENDS
+    searchcore_grouping
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_grouping_test_app COMMAND searchcore_grouping_test_app)
diff --git a/searchcore/src/tests/grouping/DESC b/searchcore/src/tests/grouping/DESC
new file mode 100644
index 00000000000..1aa6cb37e89
--- /dev/null
+++ b/searchcore/src/tests/grouping/DESC
@@ -0,0 +1 @@
+grouping test. Take a look at grouping.cpp for details.
diff --git a/searchcore/src/tests/grouping/FILES b/searchcore/src/tests/grouping/FILES
new file mode 100644
index 00000000000..a3a45cfb198
--- /dev/null
+++ b/searchcore/src/tests/grouping/FILES
@@ -0,0 +1 @@
+grouping.cpp
diff --git a/searchcore/src/tests/grouping/grouping.cpp b/searchcore/src/tests/grouping/grouping.cpp
new file mode 100644
index 00000000000..740e1aeb285
--- /dev/null
+++ b/searchcore/src/tests/grouping/grouping.cpp
@@ -0,0 +1,604 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("grouping_test");
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchlib/aggregation/grouping.h>
+#include <vespa/searchlib/aggregation/sumaggregationresult.h>
+#include <vespa/searchcommon/attribute/iattributevector.h>
+#include <vespa/searchlib/expression/attributenode.h>
+#include <vespa/searchlib/attribute/extendableattributes.h>
+#include <vespa/searchcore/grouping/groupingcontext.h>
+#include <vespa/searchcore/grouping/groupingmanager.h>
+#include <vespa/searchcore/grouping/groupingsession.h>
+#include <vespa/searchcore/grouping/sessionid.h>
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
+
+using namespace search::attribute;
+using namespace search::aggregation;
+using namespace search::expression;
+using namespace search::grouping;
+using namespace search;
+
+using proton::matching::SessionManager;
+
+
+//-----------------------------------------------------------------------------
+
+const uint32_t NUM_DOCS = 1000;
+
+//-----------------------------------------------------------------------------
+
+class MyAttributeContext : public IAttributeContext
+{
+private:
+    typedef std::map<string, IAttributeVector *> Map;
+    Map _vectors;
+
+public:
+    const IAttributeVector *get(const string &name) const {
+        if (_vectors.find(name) == _vectors.end()) {
+            return 0;
+        }
+        return _vectors.find(name)->second;
+    }
+    virtual const IAttributeVector *
+    getAttribute(const string &name) const {
+        return get(name);
+    }
+    virtual const IAttributeVector *
+    getAttributeStableEnum(const string &name) const {
+        return get(name);
+    }
+    virtual void
+    getAttributeList(std::vector<const IAttributeVector *> & list) const {
+        Map::const_iterator pos = _vectors.begin();
+        Map::const_iterator end = _vectors.end();
+        for (; pos != end; ++pos) {
+            list.push_back(pos->second);
+        }
+    }
+    ~MyAttributeContext() {
+        Map::iterator pos = _vectors.begin();
+        Map::iterator end = _vectors.end();
+        for (; pos != end; ++pos) {
+            delete pos->second;
+        }
+    }
+
+    //-------------------------------------------------------------------------
+
+    void add(IAttributeVector *attr) {
+        _vectors[attr->getName()] = attr;
+    }
+};
+
+
+//-----------------------------------------------------------------------------
+
+struct MyWorld {
+    MyAttributeContext attributeContext;
+
+    void basicSetup() {
+        // attribute context
+        {
+            SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr0");
+            AttributeVector::DocId docid;
+            for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+                attr->addDoc(docid);
+                attr->add(i, docid); // value = docid
+            }
+            assert(docid + 1 == NUM_DOCS);
+            attributeContext.add(attr);
+        }
+        {
+            SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr1");
+            AttributeVector::DocId docid;
+            for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+                attr->addDoc(docid);
+                attr->add(i * 2, docid); // value = docid * 2
+            }
+            assert(docid + 1 == NUM_DOCS);
+            attributeContext.add(attr);
+        }
+        {
+            SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr2");
+            AttributeVector::DocId docid;
+            for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+                attr->addDoc(docid);
+                attr->add(i * 3, docid); // value = docid * 3
+            }
+            assert(docid + 1 == NUM_DOCS);
+            attributeContext.add(attr);
+        }
+        {
+            SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr3");
+            AttributeVector::DocId docid;
+            for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+                attr->addDoc(docid);
+                attr->add(i * 4, docid); // value = docid * 4
+            }
+            assert(docid + 1 == NUM_DOCS);
+            attributeContext.add(attr);
+        }
+
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+typedef GroupingContext::GroupingList GroupingList;
+
+SessionId createSessionId(const std::string & s) {
+    std::vector<char> vec;
+    for (size_t i = 0; i < s.size(); i++) {
+        vec.push_back(s[i]);
+    }
+    return SessionId(&vec[0], vec.size());
+}
+
+class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate
+{
+public:
+    CheckAttributeReferences() : _numrefs(0) { }
+    int _numrefs;
+private:
+    virtual void execute(vespalib::Identifiable &obj) {
+        if (static_cast<AttributeNode &>(obj).getAttribute() != NULL) {
+            _numrefs++;
+        }
+    }
+    virtual bool check(const vespalib::Identifiable &obj) const { return obj.inherits(AttributeNode::classId); }
+};
+
+struct DoomFixture {
+    vespalib::Clock clock;
+    fastos::TimeStamp timeOfDoom;
+    DoomFixture() : clock(), timeOfDoom(fastos::TimeStamp::FUTURE) {}
+};
+
+//-----------------------------------------------------------------------------
+
+TEST("testSessionId") {
+    SessionId id1;
+    ASSERT_TRUE(id1.empty());
+
+    SessionId id2(createSessionId("foo"));
+    SessionId id3(createSessionId("bar"));
+
+    ASSERT_TRUE(!id2.empty());
+    ASSERT_TRUE(!id3.empty());
+    ASSERT_TRUE(id3 < id2);
+    EXPECT_EQUAL(id2, id2);
+}
+
+TEST_F("testGroupingContextInitialization", DoomFixture()) {
+    vespalib::nbostream os;
+    Grouping baseRequest = Grouping()
+                           .setRoot(Group()
+                                    .addResult(SumAggregationResult()
+                                            .setExpression(AttributeNode("attr0"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr2"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr3"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr3"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr1"))));
+
+    vespalib::NBOSerializer nos(os);
+    nos << (uint32_t)1;
+    baseRequest.serialize(nos);
+
+    GroupingContext context(f1.clock, f1.timeOfDoom, os.c_str(), os.size());
+    ASSERT_TRUE(!context.empty());
+    GroupingContext::GroupingList list = context.getGroupingList();
+    ASSERT_TRUE(list.size() == 1);
+    EXPECT_EQUAL(list[0]->asString(), baseRequest.asString());
+    context.reset();
+    ASSERT_TRUE(context.empty());
+}
+
+TEST_F("testGroupingContextUsage", DoomFixture()) {
+    vespalib::nbostream os;
+    Grouping request1 = Grouping()
+                           .setFirstLevel(0)
+                           .setLastLevel(0)
+                           .setRoot(Group()
+                                    .addResult(SumAggregationResult()
+                                            .setExpression(AttributeNode("attr0"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr2"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr3"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr3"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr1"))));
+
+    Grouping request2 = Grouping()
+                           .setFirstLevel(0)
+                           .setLastLevel(3)
+                           .setRoot(Group()
+                                    .addResult(SumAggregationResult()
+                                            .setExpression(AttributeNode("attr0"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr2"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr3"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr3"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr1"))));
+
+
+    GroupingContext::GroupingPtr r1(new Grouping(request1));
+    GroupingContext::GroupingPtr r2(new Grouping(request2));
+    GroupingContext context(f1.clock, f1.timeOfDoom);
+    ASSERT_TRUE(context.empty());
+    context.addGrouping(r1);
+    ASSERT_TRUE(context.getGroupingList().size() == 1);
+    context.addGrouping(r2);
+    ASSERT_TRUE(context.getGroupingList().size() == 2);
+    context.reset();
+    ASSERT_TRUE(context.empty());
+}
+
+TEST_F("testGroupingContextSerializing", DoomFixture()) {
+    Grouping baseRequest = Grouping()
+                           .setRoot(Group()
+                                    .addResult(SumAggregationResult()
+                                            .setExpression(AttributeNode("attr0"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr2"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr3"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr3"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr1"))));
+
+    vespalib::nbostream os;
+    vespalib::NBOSerializer nos(os);
+    nos << (uint32_t)1;
+    baseRequest.serialize(nos);
+
+    GroupingContext context(f1.clock, f1.timeOfDoom);
+    GroupingContext::GroupingPtr bp(new Grouping(baseRequest));
+    context.addGrouping(bp);
+    context.serialize();
+    vespalib::nbostream & res(context.getResult());
+    EXPECT_EQUAL(res.size(), os.size());
+    ASSERT_TRUE(memcmp(res.c_str(), os.c_str(), res.size()) == 0);
+}
+
+TEST_F("testGroupingManager", DoomFixture()) {
+    vespalib::nbostream os;
+    Grouping request1 = Grouping()
+                           .setFirstLevel(0)
+                           .setLastLevel(0)
+                           .setRoot(Group()
+                                    .addResult(SumAggregationResult()
+                                            .setExpression(AttributeNode("attr0"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr2"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                             .setExpression(AttributeNode("attr3"))));
+
+    GroupingContext context(f1.clock, f1.timeOfDoom);
+    GroupingContext::GroupingPtr bp(new Grouping(request1));
+    context.addGrouping(bp);
+    GroupingManager manager(context);
+    ASSERT_TRUE(!manager.empty());
+}
+
+TEST_F("testGroupingSession", DoomFixture()) {
+    MyWorld world;
+    world.basicSetup();
+    vespalib::nbostream os;
+    Grouping request1 = Grouping()
+                           .setId(0)
+                           .setFirstLevel(0)
+                           .setLastLevel(0)
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr2"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr3"))));
+
+    Grouping request2 = Grouping()
+                           .setId(1)
+                           .setFirstLevel(0)
+                           .setLastLevel(3)
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr2"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr3"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr3"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr1"))));
+
+
+    CheckAttributeReferences attrCheck;
+    request1.select(attrCheck, attrCheck);
+    EXPECT_EQUAL(attrCheck._numrefs, 0);
+    request2.select(attrCheck, attrCheck);
+    EXPECT_EQUAL(attrCheck._numrefs, 0);
+
+    GroupingContext::GroupingPtr r1(new Grouping(request1));
+    GroupingContext::GroupingPtr r2(new Grouping(request2));
+    GroupingContext initContext(f1.clock, f1.timeOfDoom);
+    initContext.addGrouping(r1);
+    initContext.addGrouping(r2);
+    SessionId id("foo");
+
+    // Test initialization phase
+    GroupingSession session(id, initContext, world.attributeContext);
+    CheckAttributeReferences attrCheck2;
+    GroupingList &gl2(initContext.getGroupingList());
+    for (unsigned int i = 0; i < gl2.size(); i++) {
+        gl2[i]->select(attrCheck2, attrCheck2);
+    }
+    EXPECT_EQUAL(attrCheck2._numrefs, 10);
+    RankedHit hit;
+    hit._docId = 0;
+    GroupingManager &manager(session.getGroupingManager());
+    manager.groupInRelevanceOrder(&hit, 1);
+    CheckAttributeReferences attrCheck_after;
+    GroupingList &gl3(initContext.getGroupingList());
+    for (unsigned int i = 0; i < gl3.size(); i++) {
+        gl3[i]->select(attrCheck_after, attrCheck_after);
+    }
+    EXPECT_EQUAL(attrCheck_after._numrefs, 0);
+    {
+        EXPECT_EQUAL(id, session.getSessionId());
+        ASSERT_TRUE(!session.getGroupingManager().empty());
+        ASSERT_TRUE(!session.finished());
+        session.continueExecution(initContext);
+        ASSERT_TRUE(!session.finished());
+    }
+    // Test second pass
+    {
+        GroupingContext context(f1.clock, f1.timeOfDoom);
+        GroupingContext::GroupingPtr r(new Grouping(request1));
+        r->setFirstLevel(1);
+        r->setLastLevel(1);
+        context.addGrouping(r);
+
+        session.continueExecution(context);
+        ASSERT_TRUE(!session.finished());
+    }
+    // Test last pass. Session should be marked as finished
+    {
+        GroupingContext context(f1.clock, f1.timeOfDoom);
+        GroupingContext::GroupingPtr r(new Grouping(request1));
+        r->setFirstLevel(2);
+        r->setLastLevel(2);
+        context.addGrouping(r);
+
+        session.continueExecution(context);
+        ASSERT_TRUE(session.finished());
+    }
+
+}
+
+TEST_F("testEmptySessionId", DoomFixture()) {
+    MyWorld world;
+    world.basicSetup();
+    vespalib::nbostream os;
+    Grouping request1 = Grouping()
+                           .setId(0)
+                           .setFirstLevel(0)
+                           .setLastLevel(0)
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr2"))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr3"))));
+
+    GroupingContext::GroupingPtr r1(new Grouping(request1));
+    GroupingContext initContext(f1.clock, f1.timeOfDoom);
+    initContext.addGrouping(r1);
+    SessionId id;
+
+    // Test initialization phase
+    GroupingSession session(id, initContext, world.attributeContext);
+    RankedHit hit;
+    hit._docId = 0;
+    GroupingManager &manager(session.getGroupingManager());
+    manager.groupInRelevanceOrder(&hit, 1);
+    EXPECT_EQUAL(id, session.getSessionId());
+    ASSERT_TRUE(!session.getGroupingManager().empty());
+    ASSERT_TRUE(session.finished() && session.getSessionId().empty());
+    session.continueExecution(initContext);
+    ASSERT_TRUE(session.finished());
+    ASSERT_TRUE(r1->getRoot().getChildrenSize() > 0);
+}
+
+TEST_F("testSessionManager", DoomFixture()) {
+    MyWorld world;
+    world.basicSetup();
+    vespalib::nbostream os;
+    Grouping request1 = Grouping()
+                           .setId(0)
+                           .setFirstLevel(0)
+                           .setLastLevel(0)
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr1"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr2"))
+                                                .setResult(Int64ResultNode(0))))
+                           .addLevel(GroupingLevel()
+                                     .setExpression(AttributeNode("attr2"))
+                                     .addResult(SumAggregationResult()
+                                                .setExpression(AttributeNode("attr3"))
+                                                .setResult(Int64ResultNode(0))))
+                           .setRoot(Group()
+                                    .addResult(SumAggregationResult()
+                                               .setExpression(AttributeNode("attr0"))
+                                               .setResult(Int64ResultNode(0))));
+
+    GroupingContext::GroupingPtr r1(new Grouping(request1));
+    GroupingContext initContext(f1.clock, f1.timeOfDoom);
+    initContext.addGrouping(r1);
+
+    SessionManager mgr(2);
+    SessionId id1("foo");
+    SessionId id2("bar");
+    SessionId id3("baz");
+    GroupingSession::UP s1(new GroupingSession(id1, initContext, world.attributeContext));
+    GroupingSession::UP s2(new GroupingSession(id2, initContext, world.attributeContext));
+    GroupingSession::UP s3(new GroupingSession(id3, initContext, world.attributeContext));
+    ASSERT_EQUAL(f1.timeOfDoom, s1->getTimeOfDoom());
+    mgr.insert(std::move(s1));
+    s1 = mgr.pickGrouping(id1);
+    ASSERT_TRUE(s1.get());
+    EXPECT_EQUAL(id1, s1->getSessionId());
+
+    mgr.insert(std::move(s1));
+    mgr.insert(std::move(s2));
+    mgr.insert(std::move(s3));
+    s1 = mgr.pickGrouping(id1);
+    s2 = mgr.pickGrouping(id2);
+    s3 = mgr.pickGrouping(id3);
+    ASSERT_TRUE(s1.get() == NULL);
+    ASSERT_TRUE(s2.get() != NULL);
+    ASSERT_TRUE(s3.get() != NULL);
+    EXPECT_EQUAL(id2, s2->getSessionId());
+    EXPECT_EQUAL(id3, s3->getSessionId());
+    SessionManager::Stats stats = mgr.getGroupingStats();
+    EXPECT_EQUAL(4u, stats.numInsert);
+    EXPECT_EQUAL(3u, stats.numPick);
+    EXPECT_EQUAL(1u, stats.numDropped);
+}
+
+void doGrouping(GroupingContext &ctx,
+                uint32_t doc1, double rank1,
+                uint32_t doc2, double rank2,
+                uint32_t doc3, double rank3)
+{
+    GroupingManager man(ctx);
+    std::vector<RankedHit> hits;
+    hits.push_back(RankedHit(doc1, rank1));
+    hits.push_back(RankedHit(doc2, rank2));
+    hits.push_back(RankedHit(doc3, rank3));
+    man.groupInRelevanceOrder(&hits[0], 3);
+}
+
+TEST_F("test grouping fork/join", DoomFixture()) {
+    MyWorld world;
+    world.basicSetup();
+
+    Grouping request = Grouping()
+                       .setRoot(Group()
+                                .addResult(SumAggregationResult()
+                                           .setExpression(AttributeNode("attr0"))))
+                       .addLevel(GroupingLevel()
+                                 .setMaxGroups(3)
+                                 .setExpression(AttributeNode("attr0")))
+                       .setFirstLevel(0)
+                       .setLastLevel(1);
+
+    GroupingContext::GroupingPtr g1(new Grouping(request));
+    GroupingContext context(f1.clock, f1.timeOfDoom);
+    context.addGrouping(g1);
+    GroupingSession session(SessionId(), context, world.attributeContext);
+    session.prepareThreadContextCreation(4);
+
+    GroupingContext::UP ctx0 = session.createThreadContext(0, world.attributeContext);
+    GroupingContext::UP ctx1 = session.createThreadContext(1, world.attributeContext);
+    GroupingContext::UP ctx2 = session.createThreadContext(2, world.attributeContext);
+    GroupingContext::UP ctx3 = session.createThreadContext(3, world.attributeContext);
+    doGrouping(*ctx0, 12, 30.0, 11, 20.0, 10, 10.0);
+    doGrouping(*ctx1, 22, 150.0, 21, 40.0, 20, 25.0);
+    doGrouping(*ctx2, 32, 100.0, 31, 15.0, 30, 5.0);
+    doGrouping(*ctx3, 42, 4.0, 41, 3.0, 40, 2.0); // not merged (verify independent contexts)
+    {
+        GroupingManager man(*ctx0);
+        man.merge(*ctx1);
+        man.merge(*ctx2);
+        man.prune();
+    }
+
+    Grouping expect = Grouping()
+                      .setRoot(Group()
+                               .addResult(SumAggregationResult()
+                                          .setExpression(AttributeNode("attr0"))
+                                          .setResult(Int64ResultNode(189)))
+                               .addChild(Group().setId(Int64ResultNode(21)).setRank(40.0))
+                               .addChild(Group().setId(Int64ResultNode(22)).setRank(150.0))
+                               .addChild(Group().setId(Int64ResultNode(32)).setRank(100.0)))
+                      .addLevel(GroupingLevel()
+                                .setMaxGroups(3)
+                                .setExpression(AttributeNode("attr0")))
+                      .setFirstLevel(0)
+                      .setLastLevel(1);
+
+    session.continueExecution(context);
+    GroupingContext::GroupingList list = context.getGroupingList();
+    ASSERT_TRUE(list.size() == 1);
+    EXPECT_EQUAL(expect.asString(), list[0]->asString());
+}
+
+TEST_F("test session timeout", DoomFixture()) {
+    MyWorld world;
+    world.basicSetup();
+    SessionManager mgr(2);
+    SessionId id1("foo");
+    SessionId id2("bar");
+
+    GroupingContext initContext1(f1.clock, 10);
+    GroupingContext initContext2(f1.clock, 20);
+    GroupingSession::UP s1(new GroupingSession(id1, initContext1, world.attributeContext));
+    GroupingSession::UP s2(new GroupingSession(id2, initContext2, world.attributeContext));
+    mgr.insert(std::move(s1));
+    mgr.insert(std::move(s2));
+    mgr.pruneTimedOutSessions(5);
+    SessionManager::Stats stats(mgr.getGroupingStats());
+    ASSERT_EQUAL(2u, stats.numCached);
+    mgr.pruneTimedOutSessions(10);
+    stats = mgr.getGroupingStats();
+    ASSERT_EQUAL(2u, stats.numCached);
+
+    mgr.pruneTimedOutSessions(11);
+    stats = mgr.getGroupingStats();
+    ASSERT_EQUAL(1u, stats.numCached);
+
+    mgr.pruneTimedOutSessions(21);
+    stats = mgr.getGroupingStats();
+    ASSERT_EQUAL(0u, stats.numCached);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/attribute/.gitignore b/searchcore/src/tests/proton/attribute/.gitignore
new file mode 100644
index 00000000000..794f5f454f8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/.gitignore
@@ -0,0 +1,9 @@
+.depend
+Makefile
+*_test
+test
+test_output
+flush
+
+searchcore_attribute_test_app
+searchcore_attributeflush_test_app
diff --git a/searchcore/src/tests/proton/attribute/CMakeLists.txt b/searchcore/src/tests/proton/attribute/CMakeLists.txt
new file mode 100644
index 00000000000..1439c2b2646
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/CMakeLists.txt
@@ -0,0 +1,21 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_test_app
+    SOURCES
+    attribute_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_attribute
+    searchcore_flushengine
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attribute_test_app COMMAND sh attribute_test.sh)
+vespa_add_executable(searchcore_attributeflush_test_app
+    SOURCES
+    attributeflush_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_attribute
+    searchcore_flushengine
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attributeflush_test_app COMMAND sh attributeflush_test.sh)
diff --git a/searchcore/src/tests/proton/attribute/DESC b/searchcore/src/tests/proton/attribute/DESC
new file mode 100644
index 00000000000..bd71a808c51
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/DESC
@@ -0,0 +1 @@
+attribute test. Take a look at attribute.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/FILES b/searchcore/src/tests/proton/attribute/FILES
new file mode 100644
index 00000000000..84bc710d58b
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/FILES
@@ -0,0 +1 @@
+attribute.cpp
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore b/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore
new file mode 100644
index 00000000000..3e77da66466
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore
@@ -0,0 +1 @@
+searchcore_attribute_manager_test_app
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt
new file mode 100644
index 00000000000..7e8ab14a13b
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt
@@ -0,0 +1,14 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_manager_test_app
+    SOURCES
+    attribute_manager_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_initializer
+    searchcore_flushengine
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attribute_manager_test_app COMMAND searchcore_attribute_manager_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/DESC b/searchcore/src/tests/proton/attribute/attribute_manager/DESC
new file mode 100644
index 00000000000..f1cdc01fd47
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/DESC
@@ -0,0 +1 @@
+attribute manager test. Take a look at attribute_manager_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/FILES b/searchcore/src/tests/proton/attribute/attribute_manager/FILES
new file mode 100644
index 00000000000..8e4fbdcb888
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/FILES
@@ -0,0 +1 @@
+attribute_manager_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
new file mode 100644
index 00000000000..34c67da4ac8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -0,0 +1,686 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_manager_test");
+
+#include <vespa/fastos/file.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h>
+#include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_functor.h>
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+#include <vespa/searchcore/proton/initializer/task_runner.h>
+#include <vespa/searchcore/proton/test/attribute_utils.h>
+#include <vespa/searchcore/proton/test/attribute_vectors.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/util/filekit.h>
+
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/predicate_attribute.h>
+#include <vespa/searchlib/predicate/predicate_index.h>
+#include <vespa/searchlib/predicate/predicate_tree_annotator.h>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/config-attributes.h>
+
+namespace vespa { namespace config { namespace search {}}}
+
+using std::string;
+using namespace vespa::config::search;
+using namespace config;
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+using proton::initializer::InitializerTask;
+using proton::test::AttributeUtils;
+using proton::test::Int32Attribute;
+using search::TuneFileAttributes;
+using search::index::DummyFileHeaderContext;
+using search::ForegroundTaskExecutor;
+using search::predicate::PredicateIndex;
+using search::predicate::PredicateTreeAnnotations;
+using vespa::config::search::AttributesConfig;
+using vespa::config::search::AttributesConfigBuilder;
+
+typedef search::attribute::Config AVConfig;
+typedef proton::AttributeCollectionSpec::Attribute AttrSpec;
+typedef proton::AttributeCollectionSpec::AttributeList AttrSpecList;
+typedef proton::AttributeCollectionSpec AttrMgrSpec;
+
+namespace {
+
+const uint64_t createSerialNum = 42u;
+
+class MyAttributeFunctor : public proton::IAttributeFunctor
+{
+    std::vector<vespalib::string> _names;
+
+public:
+    virtual void
+    operator()(const search::AttributeVector &attributeVector) override {
+        _names.push_back(attributeVector.getName());
+    }
+
+    std::string getSortedNames() {
+        std::ostringstream os;
+        std::sort(_names.begin(), _names.end());
+        for (const vespalib::string &name : _names) {
+            if (!os.str().empty())
+                os << ",";
+            os << name;
+        }
+        return os.str();
+    }
+};
+
+}
+
+const string test_dir = "test_output";
+const AVConfig INT32_SINGLE = AttributeUtils::getInt32Config();
+const AVConfig INT32_ARRAY = AttributeUtils::getInt32ArrayConfig();
+
+void
+fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken)
+{
+    test::AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken);
+}
+
+void
+fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken)
+{
+    test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken);
+}
+
+struct BaseFixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext   _fileHeaderContext;
+    ForegroundTaskExecutor   _attributeFieldWriter;
+    BaseFixture()
+        : _dirHandler(test_dir),
+          _fileHeaderContext(),
+          _attributeFieldWriter()
+    {
+    }
+};
+
+
+struct AttributeManagerFixture
+{
+    proton::AttributeManager::SP _msp;
+    proton::AttributeManager &_m;
+    AttributeWriter _aw;
+    AttributeManagerFixture(BaseFixture &bf)
+        : _msp(std::make_shared<proton::AttributeManager>
+               (test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext,
+                bf._attributeFieldWriter)),
+          _m(*_msp),
+          _aw(_msp)
+    {
+    }
+    AttributeVector::SP addAttribute(const vespalib::string &name) {
+        return _m.addAttribute(name, INT32_SINGLE, createSerialNum);
+    }
+};
+
+struct Fixture : public BaseFixture, public AttributeManagerFixture
+{
+    Fixture()
+        : BaseFixture(),
+          AttributeManagerFixture(*static_cast<BaseFixture *>(this))
+    {
+    }
+};
+
+struct SequentialAttributeManager
+{
+    SequentialAttributesInitializer initializer;
+    proton::AttributeManager mgr;
+    SequentialAttributeManager(const AttributeManager &currMgr,
+                               const AttrMgrSpec &newSpec)
+        : initializer(newSpec.getDocIdLimit()),
+          mgr(currMgr, newSpec, initializer)
+    {
+        mgr.addInitializedAttributes(initializer.getInitializedAttributes());
+    }
+};
+
+struct DummyInitializerTask : public InitializerTask
+{
+    virtual void run() override {}
+};
+
+struct ParallelAttributeManager
+{
+    InitializerTask::SP documentMetaStoreInitTask;
+    BucketDBOwner::SP bucketDbOwner;
+    DocumentMetaStore::SP documentMetaStore;
+    search::GrowStrategy attributeGrow;
+    size_t attributeGrowNumDocs;
+    bool fastAccessAttributesOnly;
+    std::shared_ptr<AttributeManager::SP> mgr;
+    AttributeManagerInitializer::SP initializer;
+
+    ParallelAttributeManager(search::SerialNum configSerialNum,
+                             AttributeManager::SP baseAttrMgr,
+                             const AttributesConfig &attrCfg,
+                             uint32_t docIdLimit)
+        : documentMetaStoreInitTask(std::make_shared<DummyInitializerTask>()),
+          bucketDbOwner(std::make_shared<BucketDBOwner>()),
+          documentMetaStore(std::make_shared<DocumentMetaStore>(bucketDbOwner)),
+          attributeGrow(),
+          attributeGrowNumDocs(1),
+          fastAccessAttributesOnly(false),
+          mgr(std::make_shared<AttributeManager::SP>()),
+          initializer(std::make_shared<AttributeManagerInitializer>
+                              (configSerialNum, documentMetaStoreInitTask, documentMetaStore, baseAttrMgr, attrCfg,
+                               attributeGrow, attributeGrowNumDocs, fastAccessAttributesOnly, mgr))
+    {
+        documentMetaStore->setCommittedDocIdLimit(docIdLimit);
+        vespalib::ThreadStackExecutor executor(3, 128 * 1024);
+        initializer::TaskRunner taskRunner(executor);
+        taskRunner.runTask(initializer);
+    }
+};
+
+
+TEST_F("require that attributes are added", Fixture)
+{
+    EXPECT_TRUE(f.addAttribute("a1").get() != NULL);
+    EXPECT_TRUE(f.addAttribute("a2").get() != NULL);
+    EXPECT_EQUAL("a1", (*f._m.getAttribute("a1"))->getName());
+    EXPECT_EQUAL("a1", (*f._m.getAttributeStableEnum("a1"))->getName());
+    EXPECT_EQUAL("a2", (*f._m.getAttribute("a2"))->getName());
+    EXPECT_EQUAL("a2", (*f._m.getAttributeStableEnum("a2"))->getName());
+    EXPECT_TRUE(!f._m.getAttribute("not")->valid());
+}
+
+TEST_F("require that predicate attributes are added", Fixture)
+{
+    EXPECT_TRUE(f._m.addAttribute("p1", AttributeUtils::getPredicateConfig(),
+                                  createSerialNum).get() != NULL);
+    EXPECT_EQUAL("p1", (*f._m.getAttribute("p1"))->getName());
+    EXPECT_EQUAL("p1", (*f._m.getAttributeStableEnum("p1"))->getName());
+}
+
+TEST_F("require that attributes are flushed and loaded", BaseFixture)
+{
+    IndexMetaInfo ia1(test_dir + "/a1");
+    IndexMetaInfo ia2(test_dir + "/a2");
+    IndexMetaInfo ia3(test_dir + "/a3");
+    {
+        AttributeManagerFixture amf(f);
+        proton::AttributeManager &am = amf._m;
+        AttributeVector::SP a1 = amf.addAttribute("a1");
+        EXPECT_EQUAL(1u, a1->getNumDocs());	// Resized to size of attributemanager
+        fillAttribute(a1, 1, 3, 2, 10);
+        EXPECT_EQUAL(3u, a1->getNumDocs());	// Resized to size of attributemanager
+        AttributeVector::SP a2 = amf.addAttribute("a2");
+        EXPECT_EQUAL(1u, a2->getNumDocs());	// Not resized to size of attributemanager
+        fillAttribute(a2, 1, 5, 4, 10);
+        EXPECT_EQUAL(5u, a2->getNumDocs());	// Increased
+        EXPECT_TRUE(ia1.load());
+        EXPECT_TRUE(!ia1.getBestSnapshot().valid);
+        EXPECT_TRUE(ia2.load());
+        EXPECT_TRUE(!ia2.getBestSnapshot().valid);
+        EXPECT_TRUE(!ia3.load());
+        am.flushAll(0);
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia2.load());
+        EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken);
+    }
+    {
+        AttributeManagerFixture amf(f);
+        proton::AttributeManager &am = amf._m;
+        AttributeVector::SP a1 = amf.addAttribute("a1"); // loaded
+
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+        fillAttribute(a1, 1, 2, 20);
+        EXPECT_EQUAL(4u, a1->getNumDocs());
+        AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded
+        EXPECT_EQUAL(5u, a2->getNumDocs());
+        EXPECT_EQUAL(4u, a1->getNumDocs());
+        amf._aw.onReplayDone(5u);
+        EXPECT_EQUAL(5u, a2->getNumDocs());
+        EXPECT_EQUAL(5u, a1->getNumDocs());
+        fillAttribute(a2, 1, 4, 20);
+        EXPECT_EQUAL(6u, a2->getNumDocs());
+        AttributeVector::SP a3 = amf.addAttribute("a3"); // not-loaded
+        EXPECT_EQUAL(1u, a3->getNumDocs());
+        amf._aw.onReplayDone(6);
+        EXPECT_EQUAL(6u, a3->getNumDocs());
+        fillAttribute(a3, 1, 7, 6, 20);
+        EXPECT_EQUAL(7u, a3->getNumDocs());
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia2.load());
+        EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia3.load());
+        EXPECT_TRUE(!ia3.getBestSnapshot().valid);
+        am.flushAll(0);
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia2.load());
+        EXPECT_EQUAL(20u, ia2.getBestSnapshot().syncToken);
+        EXPECT_TRUE(ia3.load());
+        EXPECT_EQUAL(20u, ia3.getBestSnapshot().syncToken);
+    }
+    {
+        AttributeManagerFixture amf(f);
+        AttributeVector::SP a1 = amf.addAttribute("a1"); // loaded
+        EXPECT_EQUAL(6u, a1->getNumDocs());
+        AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded
+        EXPECT_EQUAL(6u, a1->getNumDocs());
+        EXPECT_EQUAL(6u, a2->getNumDocs());
+        AttributeVector::SP a3 = amf.addAttribute("a3"); // loaded
+        EXPECT_EQUAL(6u, a1->getNumDocs());
+        EXPECT_EQUAL(6u, a2->getNumDocs());
+        EXPECT_EQUAL(7u, a3->getNumDocs());
+        amf._aw.onReplayDone(7);
+        EXPECT_EQUAL(7u, a1->getNumDocs());
+        EXPECT_EQUAL(7u, a2->getNumDocs());
+        EXPECT_EQUAL(7u, a3->getNumDocs());
+    }
+}
+
+TEST_F("require that predicate attributes are flushed and loaded", BaseFixture)
+{
+    IndexMetaInfo ia1(test_dir + "/a1");
+    {
+        AttributeManagerFixture amf(f);
+        proton::AttributeManager &am = amf._m;
+        AttributeVector::SP a1 =
+            am.addAttribute("a1",
+                            AttributeUtils::getPredicateConfig(),
+                            createSerialNum);
+        EXPECT_EQUAL(1u, a1->getNumDocs());
+
+        PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1);
+        PredicateIndex &index = pa.getIndex();
+        uint32_t doc_id;
+        a1->addDoc(doc_id);
+        index.indexEmptyDocument(doc_id);
+        pa.commit(10, 10);
+
+        EXPECT_EQUAL(2u, a1->getNumDocs());
+
+        EXPECT_TRUE(ia1.load());
+        EXPECT_TRUE(!ia1.getBestSnapshot().valid);
+        am.flushAll(0);
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+    }
+    {
+        AttributeManagerFixture amf(f);
+        proton::AttributeManager &am = amf._m;
+        AttributeVector::SP a1 =
+            am.addAttribute("a1", AttributeUtils::getPredicateConfig(),
+                            createSerialNum); // loaded
+        EXPECT_EQUAL(2u, a1->getNumDocs());
+
+        PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1);
+        PredicateIndex &index = pa.getIndex();
+        uint32_t doc_id;
+        a1->addDoc(doc_id);
+        PredicateTreeAnnotations annotations(3);
+        annotations.interval_map[123] = {{ 0x0001ffff }};
+        index.indexDocument(1, annotations);
+        pa.commit(20, 20);
+
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken);
+        am.flushAll(0);
+        EXPECT_TRUE(ia1.load());
+        EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken);
+    }
+}
+
+TEST_F("require that extra attribute is added", Fixture)
+{
+    AttributeVector::SP extra(new Int32Attribute("extra"));
+    f._m.addExtraAttribute(extra);
+    AttributeGuard::UP exguard(f._m.getAttribute("extra"));
+    EXPECT_TRUE(dynamic_cast<Int32Attribute *>(exguard->operator->()) !=
+                NULL);
+}
+
+TEST_F("require that reconfig can add attributes", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+
+    AttrSpecList newSpec;
+    newSpec.push_back(AttrSpec("a1", INT32_SINGLE));
+    newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+    newSpec.push_back(AttrSpec("a3", INT32_SINGLE));
+
+    SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, f._m.getNumDocs(), 0));
+    std::vector<AttributeGuard> list;
+    sam.mgr.getAttributeList(list);
+    std::sort(list.begin(), list.end(), [](const AttributeGuard & a, const AttributeGuard & b) {
+        return a->getName() < b->getName();
+    });
+    EXPECT_EQUAL(3u, list.size());
+    EXPECT_EQUAL("a1", list[0]->getName());
+    EXPECT_TRUE(list[0].operator->() == a1.get()); // reuse
+    EXPECT_EQUAL("a2", list[1]->getName());
+    EXPECT_EQUAL("a3", list[2]->getName());
+    EXPECT_TRUE(sam.mgr.getAttribute("ex")->operator->() == ex.get()); // reuse
+}
+
+TEST_F("require that reconfig can remove attributes", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    AttributeVector::SP a3 = f.addAttribute("a3");
+
+    AttrSpecList newSpec;
+    newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+
+    SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 1, 0));
+    std::vector<AttributeGuard> list;
+    sam.mgr.getAttributeList(list);
+    EXPECT_EQUAL(1u, list.size());
+    EXPECT_EQUAL("a2", list[0]->getName());
+    EXPECT_TRUE(list[0].operator->() == a2.get()); // reuse
+}
+
+TEST_F("require that new attributes after reconfig are initialized", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    uint32_t docId(0);
+    a1->addDoc(docId);
+    EXPECT_EQUAL(1u, docId);
+    a1->addDoc(docId);
+    EXPECT_EQUAL(2u, docId);
+    EXPECT_EQUAL(3u, a1->getNumDocs());
+
+    AttrSpecList newSpec;
+    newSpec.push_back(AttrSpec("a1", INT32_SINGLE));
+    newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+    newSpec.push_back(AttrSpec("a3", INT32_ARRAY));
+
+    SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 3, 4));
+    AttributeGuard::UP a2ap = sam.mgr.getAttribute("a2");
+    AttributeGuard &a2(*a2ap);
+    EXPECT_EQUAL(3u, a2->getNumDocs());
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(1)));
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(2)));
+    EXPECT_EQUAL(0u, a2->getStatus().getLastSyncToken());
+    AttributeGuard::UP a3ap = sam.mgr.getAttribute("a3");
+    AttributeGuard &a3(*a3ap);
+    AttributeVector::largeint_t buf[1];
+    EXPECT_EQUAL(3u, a3->getNumDocs());
+    EXPECT_EQUAL(0u, a3->get(1, buf, 1));
+    EXPECT_EQUAL(0u, a3->get(2, buf, 1));
+    EXPECT_EQUAL(0u, a3->getStatus().getLastSyncToken());
+}
+
+TEST_F("require that removed attributes can resurrect", BaseFixture)
+{
+    proton::AttributeManager::SP am1(
+            new proton::AttributeManager(test_dir, "test.subdb",
+                                         TuneFileAttributes(),
+                                         f._fileHeaderContext,
+                                         f._attributeFieldWriter));
+    {
+        AttributeVector::SP a1 =
+            am1->addAttribute("a1", INT32_SINGLE,
+                              0);
+        fillAttribute(a1, 2, 10, 15);
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+    }
+
+    AttrSpecList ns1;
+    SequentialAttributeManager am2(*am1, AttrMgrSpec(ns1, 3, 16));
+    am1.reset();
+
+    AttrSpecList ns2;
+    ns2.push_back(AttrSpec("a1", INT32_SINGLE));
+    // 2 new documents added since a1 was removed
+    SequentialAttributeManager am3(am2.mgr, AttrMgrSpec(ns2, 5, 20));
+
+    AttributeGuard::UP ag1ap = am3.mgr.getAttribute("a1");
+    AttributeGuard &ag1(*ag1ap);
+    ASSERT_TRUE(ag1.valid());
+    EXPECT_EQUAL(5u, ag1->getNumDocs());
+    EXPECT_EQUAL(10, ag1->getInt(1));
+    EXPECT_EQUAL(10, ag1->getInt(2));
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ag1->getInt(3)));
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ag1->getInt(4)));
+    EXPECT_EQUAL(16u, ag1->getStatus().getLastSyncToken());
+}
+
+TEST_F("require that extra attribute is not treated as removed", Fixture)
+{
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+    ex->commit(1,1);
+
+    AttrSpecList ns;
+    SequentialAttributeManager am2(f._m, AttrMgrSpec(ns, 2, 1));
+    EXPECT_TRUE(am2.mgr.getAttribute("ex")->operator->() == ex.get()); // reuse
+}
+
+TEST_F("require that history can be wiped", Fixture)
+{
+    f.addAttribute("a1");
+    f.addAttribute("a2");
+    f.addAttribute("a3");
+    f._m.flushAll(10);
+    Schema hs;
+    hs.addAttributeField(Schema::AttributeField("a1", Schema::INT32));
+    hs.addAttributeField(Schema::AttributeField("a3", Schema::INT32));
+    f._m.wipeHistory(hs);
+    FastOS_StatInfo si;
+    EXPECT_TRUE(!FastOS_File::Stat(vespalib::string(test_dir + "/a1").c_str(), &si));
+    EXPECT_TRUE(FastOS_File::Stat(vespalib::string(test_dir + "/a2").c_str(), &si));
+    EXPECT_TRUE(!FastOS_File::Stat(vespalib::string(test_dir + "/a3").c_str(), &si));
+}
+
+TEST_F("require that lid space can be compacted", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+    const int64_t attrValue = 33;
+    fillAttribute(a1, 20, attrValue, 100);
+    fillAttribute(a2, 20, attrValue, 100);
+    fillAttribute(ex, 20, attrValue, 100);
+
+    EXPECT_EQUAL(21u, a1->getNumDocs());
+    EXPECT_EQUAL(21u, a2->getNumDocs());
+    EXPECT_EQUAL(20u, ex->getNumDocs());
+    EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit());
+    EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit());
+    EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit());
+
+    f._aw.compactLidSpace(10, 101);
+
+    EXPECT_EQUAL(21u, a1->getNumDocs());
+    EXPECT_EQUAL(21u, a2->getNumDocs());
+    EXPECT_EQUAL(20u, ex->getNumDocs());
+    EXPECT_EQUAL(10u, a1->getCommittedDocIdLimit());
+    EXPECT_EQUAL(10u, a2->getCommittedDocIdLimit());
+    EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit());
+}
+
+TEST_F("require that lid space compaction op can be ignored", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+    const int64_t attrValue = 33;
+    fillAttribute(a1, 20, attrValue, 200);
+    fillAttribute(a2, 20, attrValue, 100);
+    fillAttribute(ex, 20, attrValue, 100);
+
+    EXPECT_EQUAL(21u, a1->getNumDocs());
+    EXPECT_EQUAL(21u, a2->getNumDocs());
+    EXPECT_EQUAL(20u, ex->getNumDocs());
+    EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit());
+    EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit());
+    EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit());
+
+    f._aw.compactLidSpace(10, 101);
+
+    EXPECT_EQUAL(21u, a1->getNumDocs());
+    EXPECT_EQUAL(21u, a2->getNumDocs());
+    EXPECT_EQUAL(20u, ex->getNumDocs());
+    EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit());
+    EXPECT_EQUAL(10u, a2->getCommittedDocIdLimit());
+    EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit());
+}
+
+TEST_F("require that flushed serial number can be retrieved", Fixture)
+{
+    f.addAttribute("a1");
+    EXPECT_EQUAL(0u, f._m.getFlushedSerialNum("a1"));
+    f._m.flushAll(100);
+    EXPECT_EQUAL(100u, f._m.getFlushedSerialNum("a1"));
+    EXPECT_EQUAL(0u, f._m.getFlushedSerialNum("a2"));
+}
+
+
+TEST_F("require that writable attributes can be retrieved", Fixture)
+{
+    auto a1 = f.addAttribute("a1");
+    auto a2 = f.addAttribute("a2");
+    AttributeVector::SP ex(new Int32Attribute("ex"));
+    f._m.addExtraAttribute(ex);
+    auto &vec = f._m.getWritableAttributes();
+    EXPECT_EQUAL(2u, vec.size());
+    EXPECT_EQUAL(a1.get(), vec[0]);
+    EXPECT_EQUAL(a2.get(), vec[1]);
+    EXPECT_EQUAL(a1.get(), f._m.getWritableAttribute("a1"));
+    EXPECT_EQUAL(a2.get(), f._m.getWritableAttribute("a2"));
+    AttributeVector *noAttr = nullptr;
+    EXPECT_EQUAL(noAttr, f._m.getWritableAttribute("a3"));
+    EXPECT_EQUAL(noAttr, f._m.getWritableAttribute("ex"));
+}
+
+
+void
+populateAndFlushAttributes(AttributeManagerFixture &f)
+{
+    const int64_t attrValue = 7;
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    fillAttribute(a1, 1, 10, attrValue, createSerialNum);
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    fillAttribute(a2, 1, 10, attrValue, createSerialNum);
+    AttributeVector::SP a3 = f.addAttribute("a3");
+    fillAttribute(a3, 1, 10, attrValue, createSerialNum);
+    f._m.flushAll(createSerialNum + 3);
+}
+
+void
+validateAttribute(const AttributeVector &attr)
+{
+    ASSERT_EQUAL(10u, attr.getNumDocs());
+    EXPECT_EQUAL(createSerialNum + 3, attr.getStatus().getLastSyncToken());
+    for (uint32_t docId = 1; docId < 10; ++docId) {
+        EXPECT_EQUAL(7, attr.getInt(docId));
+    }
+}
+
+TEST_F("require that attributes can be initialized and loaded in sequence", BaseFixture)
+{
+    {
+        AttributeManagerFixture amf(f);
+        populateAndFlushAttributes(amf);
+    }
+    {
+        AttributeManagerFixture amf(f);
+
+        AttrSpecList newSpec;
+        newSpec.push_back(AttrSpec("a1", INT32_SINGLE));
+        newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+        newSpec.push_back(AttrSpec("a3", INT32_SINGLE));
+
+        SequentialAttributeManager newMgr(amf._m, AttrMgrSpec(newSpec, 10, createSerialNum + 5));
+
+        AttributeGuard::UP a1 = newMgr.mgr.getAttribute("a1");
+        TEST_DO(validateAttribute(a1->get()));
+        AttributeGuard::UP a2 = newMgr.mgr.getAttribute("a2");
+        TEST_DO(validateAttribute(a2->get()));
+        AttributeGuard::UP a3 = newMgr.mgr.getAttribute("a3");
+        TEST_DO(validateAttribute(a3->get()));
+    }
+}
+
+AttributesConfigBuilder::Attribute
+createAttributeConfig(const vespalib::string &name)
+{
+    AttributesConfigBuilder::Attribute result;
+    result.name = name;
+    result.datatype = AttributesConfigBuilder::Attribute::Datatype::INT32;
+    result.collectiontype = AttributesConfigBuilder::Attribute::Collectiontype::SINGLE;
+    return result;
+}
+
+TEST_F("require that attributes can be initialized and loaded in parallel", BaseFixture)
+{
+    {
+        AttributeManagerFixture amf(f);
+        populateAndFlushAttributes(amf);
+    }
+    {
+        AttributeManagerFixture amf(f);
+
+        AttributesConfigBuilder attrCfg;
+        attrCfg.attribute.push_back(createAttributeConfig("a1"));
+        attrCfg.attribute.push_back(createAttributeConfig("a2"));
+        attrCfg.attribute.push_back(createAttributeConfig("a3"));
+
+        ParallelAttributeManager newMgr(createSerialNum + 5, amf._msp, attrCfg, 10);
+
+        AttributeGuard::UP a1 = newMgr.mgr->get()->getAttribute("a1");
+        TEST_DO(validateAttribute(a1->get()));
+        AttributeGuard::UP a2 = newMgr.mgr->get()->getAttribute("a2");
+        TEST_DO(validateAttribute(a2->get()));
+        AttributeGuard::UP a3 = newMgr.mgr->get()->getAttribute("a3");
+        TEST_DO(validateAttribute(a3->get()));
+    }
+}
+
+TEST_F("require that we can call functions on all attributes via functor",
+       Fixture)
+{
+    f.addAttribute("a1");
+    f.addAttribute("a2");
+    f.addAttribute("a3");
+    std::shared_ptr<MyAttributeFunctor> functor =
+        std::make_shared<MyAttributeFunctor>();
+    f._m.asyncForEachAttribute(functor);
+    EXPECT_EQUAL("a1,a2,a3", functor->getSortedNames());
+}
+
+TEST_F("require that we can acquire exclusive read access to attribute", Fixture)
+{
+    f.addAttribute("attr");
+    ExclusiveAttributeReadAccessor::UP attrAccessor = f._m.getExclusiveReadAccessor("attr");
+    ExclusiveAttributeReadAccessor::UP noneAccessor = f._m.getExclusiveReadAccessor("none");
+    EXPECT_TRUE(attrAccessor.get() != nullptr);
+    EXPECT_TRUE(noneAccessor.get() == nullptr);
+}
+
+TEST_MAIN()
+{
+    vespalib::rmdir(test_dir, true);
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore b/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore
new file mode 100644
index 00000000000..2400fd559e6
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore
@@ -0,0 +1 @@
+searchcore_attribute_populator_test_app
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt
new file mode 100644
index 00000000000..064759b88d1
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_populator_test_app
+    SOURCES
+    attribute_populator_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attribute_populator_test_app COMMAND searchcore_attribute_populator_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/DESC b/searchcore/src/tests/proton/attribute/attribute_populator/DESC
new file mode 100644
index 00000000000..5ef9dcb2709
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/DESC
@@ -0,0 +1 @@
+attribute_populator test. Take a look at attribute_populator_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/FILES b/searchcore/src/tests/proton/attribute/attribute_populator/FILES
new file mode 100644
index 00000000000..b6bf0bf8458
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/FILES
@@ -0,0 +1 @@
+attribute_populator_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
new file mode 100644
index 00000000000..36e50249b89
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_populator_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/attribute_populator.h>
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+
+typedef search::attribute::Config AVConfig;
+typedef search::attribute::BasicType AVBasicType;
+
+const vespalib::string TEST_DIR = "testdir";
+const uint64_t CREATE_SERIAL_NUM = 8u;
+
+Schema
+createSchema()
+{
+    Schema schema;
+    schema.addAttributeField(Schema::AttributeField("a1", Schema::DataType::INT32));
+    return schema;
+}
+
+struct DocContext
+{
+    Schema _schema;
+    DocBuilder _builder;
+    DocContext()
+        : _schema(createSchema()),
+          _builder(_schema)
+    {
+    }
+    Document::UP create(uint32_t id, int64_t fieldValue) {
+        vespalib::string docId =
+                vespalib::make_string("id:searchdocument:searchdocument::%u", id);
+        return _builder.startDocument(docId).
+                startAttributeField("a1").addInt(fieldValue).endField().
+                endDocument();
+    }
+};
+
+struct Fixture
+{
+    test::DirectoryHandler _testDir;
+    DummyFileHeaderContext _fileHeader;
+    ForegroundTaskExecutor _attributeFieldWriter;
+    AttributeManager::SP _mgr;
+    AttributePopulator _pop;
+    DocContext _ctx;
+    Fixture()
+        : _testDir(TEST_DIR),
+          _fileHeader(),
+          _attributeFieldWriter(),
+          _mgr(new AttributeManager(TEST_DIR, "test.subdb",
+                  TuneFileAttributes(),
+                                    _fileHeader, _attributeFieldWriter)),
+          _pop(_mgr, 1, "test"),
+          _ctx()
+    {
+        _mgr->addAttribute("a1", AVConfig(AVBasicType::INT32),
+                CREATE_SERIAL_NUM);
+    }
+    AttributeGuard::UP getAttr() {
+        return _mgr->getAttribute("a1");
+    }
+};
+
+TEST_F("require that reprocess with document populates attribute", Fixture)
+{
+    AttributeGuard::UP attr = f.getAttr();
+    EXPECT_EQUAL(1u, attr->get().getNumDocs());
+
+    f._pop.handleExisting(5, *f._ctx.create(0, 33));
+    EXPECT_EQUAL(6u, attr->get().getNumDocs());
+    EXPECT_EQUAL(33, attr->get().getInt(5));
+    EXPECT_EQUAL(1u, attr->get().getStatus().getLastSyncToken());
+
+    f._pop.handleExisting(6, *f._ctx.create(1, 44));
+    EXPECT_EQUAL(7u, attr->get().getNumDocs());
+    EXPECT_EQUAL(44, attr->get().getInt(6));
+    EXPECT_EQUAL(2u, attr->get().getStatus().getLastSyncToken());
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
new file mode 100644
index 00000000000..d5084273c6c
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -0,0 +1,607 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_test");
+
+#include <vespa/fastos/file.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
+#include <vespa/searchcore/proton/test/attribute_utils.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/common/idestructorcallback.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/util/filekit.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+#include <vespa/document/predicate/predicate_slime_builder.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/predicate_attribute.h>
+#include <vespa/searchlib/predicate/predicate_index.h>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
+#include <vespa/searchlib/predicate/predicate_hash.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+#include <vespa/vespalib/tensor/tensor.h>
+#include <vespa/vespalib/tensor/types.h>
+#include <vespa/vespalib/tensor/default_tensor.h>
+#include <vespa/vespalib/tensor/tensor_factory.h>
+#include <vespa/searchlib/attribute/tensorattribute.h>
+
+
+namespace vespa { namespace config { namespace search {}}}
+
+using std::string;
+using namespace vespa::config::search;
+using namespace config;
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+using search::attribute::TensorAttribute;
+using search::TuneFileAttributes;
+using search::index::DummyFileHeaderContext;
+using search::predicate::PredicateIndex;
+using search::predicate::PredicateHash;
+using vespalib::tensor::Tensor;
+using vespalib::tensor::TensorType;
+using vespalib::tensor::TensorCells;
+using vespalib::tensor::TensorDimensions;
+
+typedef search::attribute::Config AVConfig;
+typedef search::attribute::BasicType AVBasicType;
+typedef search::attribute::CollectionType AVCollectionType;
+typedef proton::AttributeCollectionSpec::Attribute AttrSpec;
+typedef proton::AttributeCollectionSpec::AttributeList AttrSpecList;
+typedef proton::AttributeCollectionSpec AttrMgrSpec;
+typedef SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> > Int32AttributeVector;
+
+namespace
+{
+
+const uint64_t createSerialNum = 42u;
+
+}
+
+AVConfig
+unregister(const AVConfig & cfg)
+{
+    AVConfig retval = cfg;
+    return retval;
+}
+
+const string test_dir = "test_output";
+const AVConfig INT32_SINGLE = unregister(AVConfig(AVBasicType::INT32));
+const AVConfig INT32_ARRAY = unregister(AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY));
+
+void
+fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken)
+{
+    test::AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken);
+}
+
+void
+fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken)
+{
+    test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken);
+}
+
+const std::shared_ptr<IDestructorCallback> emptyCallback;
+
+
+struct Fixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext   _fileHeaderContext;
+    ForegroundTaskExecutor   _attributeFieldWriter;
+    proton::AttributeManager::SP _m;
+    AttributeWriter aw;
+
+    Fixture()
+        : _dirHandler(test_dir),
+          _fileHeaderContext(),
+          _attributeFieldWriter(),
+          _m(std::make_shared<proton::AttributeManager>
+             (test_dir, "test.subdb", TuneFileAttributes(),
+              _fileHeaderContext, _attributeFieldWriter)),
+          aw(_m)
+    {
+    }
+    AttributeVector::SP addAttribute(const vespalib::string &name) {
+        return _m->addAttribute(name, AVConfig(AVBasicType::INT32),
+                                createSerialNum);
+    }
+    void put(SerialNum serialNum, const Document &doc, DocumentIdT lid,
+             bool immediateCommit = true) {
+        aw.put(serialNum, doc, lid, immediateCommit, emptyCallback);
+    }
+    void update(SerialNum serialNum, const DocumentUpdate &upd,
+                DocumentIdT lid, bool immediateCommit) {
+        aw.update(serialNum, upd, lid, immediateCommit, emptyCallback);
+    }
+    void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit = true) {
+        aw.remove(serialNum, lid, immediateCommit, emptyCallback);
+    }
+    void commit(SerialNum serialNum) {
+        aw.commit(serialNum, emptyCallback);
+    }
+};
+
+
+TEST_F("require that attribute adapter handles put", Fixture)
+{
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("a1", Schema::INT32, Schema::SINGLE));
+    s.addAttributeField(Schema::AttributeField("a2", Schema::INT32, Schema::ARRAY));
+    s.addAttributeField(Schema::AttributeField("a3", Schema::FLOAT, Schema::SINGLE));
+    s.addAttributeField(Schema::AttributeField("a4", Schema::STRING, Schema::SINGLE));
+
+    DocBuilder idb(s);
+
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 =
+        am.addAttribute("a2",
+                        AVConfig(AVBasicType::INT32,
+                                 AVCollectionType::ARRAY),
+                        createSerialNum);
+    AttributeVector::SP a3 =
+        am.addAttribute("a3", AVConfig(AVBasicType::FLOAT),
+                        createSerialNum);
+    AttributeVector::SP a4 = am.addAttribute("a4",
+                                             AVConfig(AVBasicType::STRING),
+                                             createSerialNum);
+
+    attribute::IntegerContent ibuf;
+    attribute::FloatContent fbuf;
+    attribute::ConstCharContent sbuf;
+    { // empty document should give default values
+        EXPECT_EQUAL(1u, a1->getNumDocs());
+        f.put(1, *idb.startDocument("doc::1").endDocument(), 1);
+        EXPECT_EQUAL(2u, a1->getNumDocs());
+        EXPECT_EQUAL(2u, a2->getNumDocs());
+        EXPECT_EQUAL(2u, a3->getNumDocs());
+        EXPECT_EQUAL(2u, a4->getNumDocs());
+        EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(1u, a2->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(1u, a3->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(1u, a4->getStatus().getLastSyncToken());
+        ibuf.fill(*a1, 1);
+        EXPECT_EQUAL(1u, ibuf.size());
+        EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ibuf[0]));
+        ibuf.fill(*a2, 1);
+        EXPECT_EQUAL(0u, ibuf.size());
+        fbuf.fill(*a3, 1);
+        EXPECT_EQUAL(1u, fbuf.size());
+        EXPECT_TRUE(search::attribute::isUndefined<float>(fbuf[0]));
+        sbuf.fill(*a4, 1);
+        EXPECT_EQUAL(1u, sbuf.size());
+        EXPECT_EQUAL(strcmp("", sbuf[0]), 0);
+    }
+    { // document with single value & multi value attribute
+        Document::UP doc = idb.startDocument("doc::2").
+            startAttributeField("a1").addInt(10).endField().
+            startAttributeField("a2").startElement().addInt(20).endElement().
+                                      startElement().addInt(30).endElement().endField().endDocument();
+        f.put(2, *doc, 2);
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+        EXPECT_EQUAL(3u, a2->getNumDocs());
+        EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(2u, a2->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(2u, a3->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(2u, a4->getStatus().getLastSyncToken());
+        ibuf.fill(*a1, 2);
+        EXPECT_EQUAL(1u, ibuf.size());
+        EXPECT_EQUAL(10u, ibuf[0]);
+        ibuf.fill(*a2, 2);
+        EXPECT_EQUAL(2u, ibuf.size());
+        EXPECT_EQUAL(20u, ibuf[0]);
+        EXPECT_EQUAL(30u, ibuf[1]);
+    }
+    { // replace existing document
+        Document::UP doc = idb.startDocument("doc::2").
+            startAttributeField("a1").addInt(100).endField().
+            startAttributeField("a2").startElement().addInt(200).endElement().
+                                      startElement().addInt(300).endElement().
+                                      startElement().addInt(400).endElement().endField().endDocument();
+        f.put(3, *doc, 2);
+        EXPECT_EQUAL(3u, a1->getNumDocs());
+        EXPECT_EQUAL(3u, a2->getNumDocs());
+        EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(3u, a2->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(3u, a3->getStatus().getLastSyncToken());
+        EXPECT_EQUAL(3u, a4->getStatus().getLastSyncToken());
+        ibuf.fill(*a1, 2);
+        EXPECT_EQUAL(1u, ibuf.size());
+        EXPECT_EQUAL(100u, ibuf[0]);
+        ibuf.fill(*a2, 2);
+        EXPECT_EQUAL(3u, ibuf.size());
+        EXPECT_EQUAL(200u, ibuf[0]);
+        EXPECT_EQUAL(300u, ibuf[1]);
+        EXPECT_EQUAL(400u, ibuf[2]);
+    }
+}
+
+TEST_F("require that attribute adapter handles predicate put", Fixture)
+{
+    Schema s;
+    s.addAttributeField(
+            Schema::AttributeField("a1", Schema::BOOLEANTREE, Schema::SINGLE));
+    DocBuilder idb(s);
+
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             AVConfig(AVBasicType::PREDICATE),
+                                             createSerialNum);
+
+    PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex();
+
+    // empty document should give default values
+    EXPECT_EQUAL(1u, a1->getNumDocs());
+    f.put(1, *idb.startDocument("doc::1").endDocument(), 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+    EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken());
+    EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size());
+
+    // document with single value attribute
+    PredicateSlimeBuilder builder;
+    Document::UP doc =
+        idb.startDocument("doc::2").startAttributeField("a1")
+        .addPredicate(builder.true_predicate().build())
+        .endField().endDocument();
+    f.put(2, *doc, 2);
+    EXPECT_EQUAL(3u, a1->getNumDocs());
+    EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken());
+    EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size());
+
+    auto it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar"));
+    EXPECT_FALSE(it.valid());
+
+    // replace existing document
+    doc = idb.startDocument("doc::2").startAttributeField("a1")
+          .addPredicate(builder.feature("foo").value("bar").build())
+          .endField().endDocument();
+    f.put(3, *doc, 2);
+    EXPECT_EQUAL(3u, a1->getNumDocs());
+    EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+
+    it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar"));
+    EXPECT_TRUE(it.valid());
+}
+
+TEST_F("require that attribute adapter handles remove", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("a1", Schema::INT32, Schema::SINGLE));
+    s.addAttributeField(Schema::AttributeField("a2", Schema::INT32, Schema::SINGLE));
+
+    DocBuilder idb(s);
+
+    fillAttribute(a1, 1, 10, 1);
+    fillAttribute(a2, 1, 20, 1);
+
+    f.remove(2, 0);
+
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a1->getInt(0)));
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(0)));
+
+    f.remove(2, 0); // same sync token as previous
+    try {
+        f.remove(1, 0); // lower sync token than previous
+        EXPECT_TRUE(true);	// update is ignored
+    } catch (vespalib::IllegalStateException & e) {
+        LOG(info, "Got expected exception: '%s'", e.getMessage().c_str());
+        EXPECT_TRUE(true);
+    }
+}
+
+void verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected)
+{
+    attribute::ConstCharContent sbuf;
+    sbuf.fill(v, lid);
+    EXPECT_EQUAL(1u, sbuf.size());
+    EXPECT_EQUAL(expected, sbuf[0]);
+}
+
+TEST_F("require that visibilitydelay is honoured", Fixture)
+{
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             AVConfig(AVBasicType::STRING),
+                                             createSerialNum);
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("a1", Schema::STRING, Schema::SINGLE));
+    DocBuilder idb(s);
+    EXPECT_EQUAL(1u, a1->getNumDocs());
+    EXPECT_EQUAL(0u, a1->getStatus().getLastSyncToken());
+    Document::UP doc = idb.startDocument("doc::1")
+                              .startAttributeField("a1").addStr("10").endField()
+                          .endDocument();
+    f.put(3, *doc, 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+    EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+    AttributeWriter awDelayed(f._m);
+    awDelayed.put(4, *doc, 2, false, emptyCallback);
+    EXPECT_EQUAL(3u, a1->getNumDocs());
+    EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+    awDelayed.put(5, *doc, 4, false, emptyCallback);
+    EXPECT_EQUAL(5u, a1->getNumDocs());
+    EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken());
+    awDelayed.commit(6, emptyCallback);
+    EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken());
+
+    AttributeWriter awDelayedShort(f._m);
+    awDelayedShort.put(7, *doc, 2, false, emptyCallback);
+    EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken());
+    awDelayedShort.put(8, *doc, 2, false, emptyCallback);
+    awDelayedShort.commit(8, emptyCallback);
+    EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken());
+
+    verifyAttributeContent(*a1, 2, "10");
+    awDelayed.put(9, *idb.startDocument("doc::1").startAttributeField("a1").addStr("11").endField().endDocument(),
+            2, false, emptyCallback);
+    awDelayed.put(10, *idb.startDocument("doc::1").startAttributeField("a1").addStr("20").endField().endDocument(),
+            2, false, emptyCallback);
+    awDelayed.put(11, *idb.startDocument("doc::1").startAttributeField("a1").addStr("30").endField().endDocument(),
+            2, false, emptyCallback);
+    EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken());
+    verifyAttributeContent(*a1, 2, "10");
+    awDelayed.commit(12, emptyCallback);
+    EXPECT_EQUAL(12u, a1->getStatus().getLastSyncToken());
+    verifyAttributeContent(*a1, 2, "30");
+    
+}
+
+TEST_F("require that attribute adapter handles predicate remove", Fixture)
+{
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             AVConfig(AVBasicType::PREDICATE),
+                                             createSerialNum);
+    Schema s;
+    s.addAttributeField(
+            Schema::AttributeField("a1", Schema::BOOLEANTREE, Schema::SINGLE));
+
+    DocBuilder idb(s);
+    PredicateSlimeBuilder builder;
+    Document::UP doc =
+        idb.startDocument("doc::1").startAttributeField("a1")
+        .addPredicate(builder.true_predicate().build())
+        .endField().endDocument();
+    f.put(1, *doc, 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+
+    PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex();
+    EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size());
+    f.remove(2, 1);
+    EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size());
+}
+
+TEST_F("require that attribute adapter handles update", Fixture)
+{
+    AttributeVector::SP a1 = f.addAttribute("a1");
+    AttributeVector::SP a2 = f.addAttribute("a2");
+
+    fillAttribute(a1, 1, 10, 1);
+    fillAttribute(a2, 1, 20, 1);
+
+    Schema schema;
+    schema.addAttributeField(Schema::AttributeField(
+                    "a1", Schema::INT32,
+                    Schema::SINGLE));
+    schema.addAttributeField(Schema::AttributeField(
+                    "a2", Schema::INT32,
+                    Schema::SINGLE));
+    DocBuilder idb(schema);
+    const document::DocumentType &dt(idb.getDocumentType());
+    DocumentUpdate upd(dt, DocumentId("doc::1"));
+    upd.addUpdate(FieldUpdate(upd.getType().getField("a1"))
+                  .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 5)));
+    upd.addUpdate(FieldUpdate(upd.getType().getField("a2"))
+                  .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)));
+
+    bool immediateCommit = true;
+    f.update(2, upd, 1, immediateCommit);
+
+    attribute::IntegerContent ibuf;
+    ibuf.fill(*a1, 1);
+    EXPECT_EQUAL(1u, ibuf.size());
+    EXPECT_EQUAL(15u, ibuf[0]);
+    ibuf.fill(*a2, 1);
+    EXPECT_EQUAL(1u, ibuf.size());
+    EXPECT_EQUAL(30u, ibuf[0]);
+
+    f.update(2, upd, 1, immediateCommit); // same sync token as previous
+    try {
+        f.update(1, upd, 1, immediateCommit); // lower sync token than previous
+        EXPECT_TRUE(true);	// update is ignored
+    } catch (vespalib::IllegalStateException & e) {
+        LOG(info, "Got expected exception: '%s'", e.getMessage().c_str());
+        EXPECT_TRUE(true);
+    }
+}
+
+TEST_F("require that attribute adapter handles predicate update", Fixture)
+{
+    proton::AttributeManager & am = *f._m;
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             AVConfig(AVBasicType::PREDICATE),
+                                             createSerialNum);
+    Schema schema;
+    schema.addAttributeField(Schema::AttributeField(
+                    "a1", Schema::BOOLEANTREE,
+                    Schema::SINGLE));
+
+    DocBuilder idb(schema);
+    PredicateSlimeBuilder builder;
+    Document::UP doc =
+        idb.startDocument("doc::1").startAttributeField("a1")
+        .addPredicate(builder.true_predicate().build())
+        .endField().endDocument();
+    f.put(1, *doc, 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+
+    const document::DocumentType &dt(idb.getDocumentType());
+    DocumentUpdate upd(dt, DocumentId("doc::1"));
+    PredicateFieldValue new_value(builder.feature("foo").value("bar").build());
+    upd.addUpdate(FieldUpdate(upd.getType().getField("a1"))
+                  .addUpdate(AssignValueUpdate(new_value)));
+
+    PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex();
+    EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size());
+    EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid());
+    bool immediateCommit = true;
+    f.update(2, upd, 1, immediateCommit);
+    EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size());
+    EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid());
+}
+
+struct AttributeCollectionSpecFixture
+{
+    AttributesConfigBuilder _builder;
+    AttributeCollectionSpecFactory _factory;
+    AttributeCollectionSpecFixture(bool fastAccessOnly)
+        : _builder(),
+          _factory(search::GrowStrategy(), 100, fastAccessOnly)
+    {
+        addAttribute("a1", false);
+        addAttribute("a2", true);
+    }
+    void addAttribute(const vespalib::string &name, bool fastAccess) {
+        AttributesConfigBuilder::Attribute attr;
+        attr.name = name;
+        attr.fastaccess = fastAccess;
+        _builder.attribute.push_back(attr);
+    }
+    AttributeCollectionSpec::UP create(uint32_t docIdLimit,
+                                       search::SerialNum serialNum) {
+        return _factory.create(_builder, docIdLimit, serialNum);
+    }
+};
+
+struct NormalAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture
+{
+    NormalAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(false) {}
+};
+
+struct FastAccessAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture
+{
+    FastAccessAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(true) {}
+};
+
+TEST_F("require that normal attribute collection spec can be created",
+        NormalAttributeCollectionSpecFixture)
+{
+    AttributeCollectionSpec::UP spec = f.create(10, 20);
+    EXPECT_EQUAL(2u, spec->getAttributes().size());
+    EXPECT_EQUAL("a1", spec->getAttributes()[0].getName());
+    EXPECT_EQUAL("a2", spec->getAttributes()[1].getName());
+    EXPECT_EQUAL(10u, spec->getDocIdLimit());
+    EXPECT_EQUAL(20u, spec->getCurrentSerialNum());
+}
+
+TEST_F("require that fast access attribute collection spec can be created",
+        FastAccessAttributeCollectionSpecFixture)
+{
+    AttributeCollectionSpec::UP spec = f.create(10, 20);
+    EXPECT_EQUAL(1u, spec->getAttributes().size());
+    EXPECT_EQUAL("a2", spec->getAttributes()[0].getName());
+    EXPECT_EQUAL(10u, spec->getDocIdLimit());
+    EXPECT_EQUAL(20u, spec->getCurrentSerialNum());
+}
+
+const FilterAttributeManager::AttributeSet ACCEPTED_ATTRIBUTES = {"a2"};
+
+struct FilterFixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext _fileHeaderContext;
+    ForegroundTaskExecutor _attributeFieldWriter;
+    proton::AttributeManager::SP _baseMgr;
+    FilterAttributeManager _filterMgr;
+    FilterFixture()
+        : _dirHandler(test_dir),
+          _fileHeaderContext(),
+          _attributeFieldWriter(),
+          _baseMgr(new proton::AttributeManager(test_dir, "test.subdb",
+                                                TuneFileAttributes(),
+                                                _fileHeaderContext,
+                                                _attributeFieldWriter)),
+          _filterMgr(ACCEPTED_ATTRIBUTES, _baseMgr)
+    {
+        _baseMgr->addAttribute("a1", INT32_SINGLE, createSerialNum);
+        _baseMgr->addAttribute("a2", INT32_SINGLE, createSerialNum);
+   }
+};
+
+TEST_F("require that filter attribute manager can filter attributes", FilterFixture)
+{
+    EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == NULL);
+    EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != NULL);
+    std::vector<AttributeGuard> attrs;
+    f._filterMgr.getAttributeList(attrs);
+    EXPECT_EQUAL(1u, attrs.size());
+    EXPECT_EQUAL("a2", attrs[0].get().getName());
+}
+
+TEST_F("require that filter attribute manager can return flushed serial number", FilterFixture)
+{
+    f._baseMgr->flushAll(100);
+    EXPECT_EQUAL(0u, f._filterMgr.getFlushedSerialNum("a1"));
+    EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2"));
+}
+
+namespace {
+
+Tensor::UP
+createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
+    vespalib::tensor::DefaultTensor::builder builder;
+    return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+}
+
+}
+
+
+TEST_F("Test that we can use attribute writer to write to tensor attribute",
+       Fixture)
+{
+    proton::AttributeManager & am = *f._m;
+    AVConfig cfg(AVBasicType::TENSOR);
+    cfg.setTensorType(TensorType::fromSpec("tensor(x{},y{})"));
+    AttributeVector::SP a1 = am.addAttribute("a1",
+                                             cfg,
+                                             createSerialNum);
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("a1", Schema::TENSOR,
+                                               Schema::SINGLE));
+    DocBuilder builder(s);
+    auto tensor = createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} },
+                               {"x", "y"});
+    Document::UP doc = builder.startDocument("doc::1").
+                       startAttributeField("a1").
+                       addTensor(tensor->clone()).endField().endDocument();
+    f.put(1, *doc, 1);
+    EXPECT_EQUAL(2u, a1->getNumDocs());
+    TensorAttribute *tensorAttribute =
+        dynamic_cast<TensorAttribute *>(a1.get());
+    EXPECT_TRUE(tensorAttribute != nullptr);
+    auto tensor2 = tensorAttribute->getTensor(1);
+    EXPECT_TRUE(static_cast<bool>(tensor2));
+    EXPECT_TRUE(tensor->equals(*tensor2));
+}
+
+TEST_MAIN()
+{
+    vespalib::rmdir(test_dir, true);
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.sh b/searchcore/src/tests/proton/attribute/attribute_test.sh
new file mode 100755
index 00000000000..950a9f92bb8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+rm -rf test_output
+$VALGRIND ./searchcore_attribute_test_app
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore b/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore
new file mode 100644
index 00000000000..2642c637ea0
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore
@@ -0,0 +1 @@
+searchcore_attribute_usage_filter_test_app
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt
new file mode 100644
index 00000000000..2dd66c2a3ec
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_usage_filter_test_app
+    SOURCES
+    attribute_usage_filter_test.cpp
+    DEPENDS
+    searchcore_attribute
+)
+vespa_add_test(NAME searchcore_attribute_usage_filter_test_app COMMAND searchcore_attribute_usage_filter_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC b/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC
new file mode 100644
index 00000000000..31b3afbcdf7
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC
@@ -0,0 +1 @@
+AttributeUsageFilter test. Take a look at attribute_usage_filter_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES b/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES
new file mode 100644
index 00000000000..b63aeb79d02
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES
@@ -0,0 +1 @@
+attribute_usage_filter_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp
new file mode 100644
index 00000000000..d8ede8030e2
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp
@@ -0,0 +1,143 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_usage_filter_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
+
+using proton::AttributeUsageFilter;
+using proton::AttributeUsageStats;
+
+namespace
+{
+
+search::AddressSpace enumStoreOverLoad(30 * 1024 * 1024 * UINT64_C(1024),
+                                       32 * 1024 * 1024 * UINT64_C(1024));
+
+search::AddressSpace multiValueOverLoad(127 * 1024 * 1024,
+                                        128 * 1024 * 1024);
+
+
+
+class MyAttributeStats : public AttributeUsageStats
+{
+public:
+    void triggerEnumStoreLimit() {
+        merge({        enumStoreOverLoad,
+                       search::AddressSpaceUsage::defaultMultiValueUsage() },
+              "enumeratedName",
+              "ready");
+    }
+
+    void triggerMultiValueLimit() {
+        merge({         search::AddressSpaceUsage::defaultEnumStoreUsage(),
+                       multiValueOverLoad },
+              "multiValueName",
+              "ready");
+    }
+};
+
+struct Fixture
+{
+    AttributeUsageFilter _filter;
+    using State = AttributeUsageFilter::State;
+    using Config = AttributeUsageFilter::Config;
+
+    Fixture()
+        : _filter()
+    {
+    }
+
+    void testWrite(const vespalib::string &exp) {
+        if (exp.empty()) {
+            EXPECT_TRUE(_filter.acceptWriteOperation());
+            State state = _filter.getAcceptState();
+            EXPECT_TRUE(state.acceptWriteOperation());
+            EXPECT_EQUAL(exp, state.message());
+        } else {
+            EXPECT_FALSE(_filter.acceptWriteOperation());
+            State state = _filter.getAcceptState();
+            EXPECT_FALSE(state.acceptWriteOperation());
+            EXPECT_EQUAL(exp, state.message());
+        }
+    }
+
+    void setAttributeStats(const AttributeUsageStats &stats) {
+        _filter.setAttributeStats(stats);
+    }
+};
+
+}
+
+TEST_F("Check that default filter allows write", Fixture)
+{
+    f.testWrite("");
+}
+
+
+TEST_F("Check that enum store limit can be reached", Fixture)
+{
+    f._filter.setConfig(Fixture::Config(0.8, 1.0));
+    MyAttributeStats stats;
+    stats.triggerEnumStoreLimit();
+    f.setAttributeStats(stats);
+    f.testWrite("enumStoreLimitReached: { "
+                "action: \""
+                "add more content nodes"
+                "\", "
+                "reason: \""
+                "enum store address space used (0.9375) > limit (0.8)"
+                "\", "
+                "enumStore: { used: 32212254720, limit: 34359738368}, "
+                "attributeName: \"enumeratedName\", subdb: \"ready\"}");
+}
+
+TEST_F("Check that multivalue limit can be reached", Fixture)
+{
+    f._filter.setConfig(Fixture::Config(1.0, 0.8));
+    MyAttributeStats stats;
+    stats.triggerMultiValueLimit();
+    f.setAttributeStats(stats);
+    f.testWrite("multiValueLimitReached: { "
+                "action: \""
+                "use 'huge' setting on attribute field "
+                "or add more content nodes"
+                "\", "
+                "reason: \""
+                "multiValue address space used (0.992188) > limit (0.8)"
+                "\", "
+                "multiValue: { used: 133169152, limit: 134217728}, "
+                "attributeName: \"multiValueName\", subdb: \"ready\"}");
+}
+
+TEST_F("Check that both enumstore limit and multivalue limit can be reached",
+       Fixture)
+{
+    f._filter.setConfig(Fixture::Config(0.8, 0.8));
+    MyAttributeStats stats;
+    stats.triggerEnumStoreLimit();
+    stats.triggerMultiValueLimit();
+    f.setAttributeStats(stats);
+    f.testWrite("enumStoreLimitReached: { "
+                "action: \""
+                "add more content nodes"
+                "\", "
+                "reason: \""
+                "enum store address space used (0.9375) > limit (0.8)"
+                "\", "
+                "enumStore: { used: 32212254720, limit: 34359738368}, "
+                "attributeName: \"enumeratedName\", subdb: \"ready\"}"
+                ", "
+                "multiValueLimitReached: { "
+                "action: \""
+                "use 'huge' setting on attribute field "
+                "or add more content nodes"
+                "\", "
+                "reason: \""
+                "multiValue address space used (0.992188) > limit (0.8)"
+                "\", "
+                "multiValue: { used: 133169152, limit: 134217728}, "
+                "attributeName: \"multiValueName\", subdb: \"ready\"}");
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
new file mode 100644
index 00000000000..53904e14658
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
@@ -0,0 +1,564 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attributeflush_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/flushableattribute.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/common/indexmetainfo.h>
+#include <vespa/searchlib/util/dirtraverse.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+
+#include <vespa/searchlib/attribute/attributevector.hpp>
+
+using namespace document;
+using namespace search;
+using namespace vespalib;
+
+using search::index::DummyFileHeaderContext;
+
+typedef search::attribute::Config AVConfig;
+typedef search::attribute::BasicType AVBasicType;
+typedef search::attribute::CollectionType AVCollectionType;
+
+typedef std::shared_ptr<Gate> GateSP;
+
+namespace proton {
+
+namespace
+{
+
+const uint64_t createSerialNum = 42u;
+
+}
+
+class TaskWrapper : public Executor::Task
+{
+private:
+    Executor::Task::UP _task;
+    GateSP             _gate;
+public:
+    TaskWrapper(Executor::Task::UP task, const GateSP &gate)
+        : _task(std::move(task)),
+          _gate(gate)
+    {
+    }
+
+    virtual void
+    run(void)
+    {
+        _task->run();
+        _gate->countDown();
+        LOG(info, "doneFlushing");
+    }
+};
+
+
+class FlushHandler
+{
+private:
+    ThreadStackExecutor _executor;
+public:
+    GateSP gate;
+
+    FlushHandler()
+        : _executor(1, 65536),
+          gate()
+    {
+    }
+
+    void
+    doFlushing(Executor::Task::UP task)
+    {
+        Executor::Task::UP wrapper(new TaskWrapper(std::move(task), gate));
+        Executor::Task::UP ok = _executor.execute(std::move(wrapper));
+        assert(ok.get() == NULL);
+    }
+};
+
+
+class UpdaterTask
+{
+private:
+    proton::AttributeManager & _am;
+public:
+    UpdaterTask(proton::AttributeManager & am)
+        :
+        _am(am)
+    {
+    }
+
+    void
+    startFlushing(uint64_t syncToken, FlushHandler & handler);
+
+    void
+    run(void);
+};
+
+
+void
+UpdaterTask::startFlushing(uint64_t syncToken, FlushHandler & handler)
+{
+    handler.gate.reset(new Gate());
+    IFlushTarget::SP flushable = _am.getFlushable("a1");
+    LOG(info, "startFlushing(%" PRIu64 ")", syncToken);
+    handler.doFlushing(flushable->initFlush(syncToken));
+}
+
+
+void
+UpdaterTask::run(void)
+{
+    LOG(info, "UpdaterTask::run(begin)");
+    uint32_t totalDocs = 2000000;
+    uint32_t totalDocsMax = 125000000; // XXX: Timing dependent.
+    uint32_t slowdownUpdateLim = 4000000;
+    bool slowedDown = false;
+    uint32_t incDocs = 1000;
+    uint64_t commits = 0;
+    uint32_t flushCount = 0;
+    uint64_t flushedToken = 0;
+    uint64_t needFlushToken = 0;
+    FlushHandler flushHandler;
+    for (uint32_t i = incDocs;
+         i <= totalDocs || (flushCount + (flushedToken <
+                                          needFlushToken) <= 2 &&
+                            i <= totalDocsMax);
+         i += incDocs) {
+        uint32_t startDoc = 0;
+        uint32_t lastDoc = 0;
+        AttributeGuard::UP agap = _am.getAttribute("a1");
+        AttributeGuard &ag(*agap);
+        IntegerAttribute & ia = static_cast<IntegerAttribute &>(*ag);
+        for (uint32_t j = i - incDocs; j < i; ++j) {
+            if (j >= ag->getNumDocs()) {
+                ag->addDocs(startDoc, lastDoc, incDocs);
+                if (i % (totalDocs / 20) == 0) {
+                    LOG(info,
+                        "addDocs(%u, %u, %u)",
+                        startDoc, lastDoc, ag->getNumDocs());
+                }
+            }
+            ia.update(j, i);
+        }
+        ia.commit(i-1, i); // save i as last sync token
+        needFlushToken = i;
+        assert(i + 1 == ag->getNumDocs());
+        if ((commits++ % 20 == 0) &&
+            (flushHandler.gate.get() == NULL ||
+             flushHandler.gate->getCount() == 0)) {
+            startFlushing(i, flushHandler);
+            ++flushCount;
+            flushedToken = i;
+            slowedDown = false;
+        }
+        if (needFlushToken > flushedToken + slowdownUpdateLim) {
+            FastOS_Thread::Sleep(100);
+            if (!slowedDown) {
+                LOG(warning,
+                    "Slowing down updates due to slow flushing (slow disk ?)");
+            }
+            slowedDown = true;
+        }
+    }
+    if (flushHandler.gate.get() != NULL) {
+        flushHandler.gate->await();
+    }
+    if (flushedToken < needFlushToken) {
+        startFlushing(needFlushToken, flushHandler);
+        flushHandler.gate->await();
+    }
+    LOG(info, "UpdaterTask::run(end)");
+}
+
+
+AVConfig
+getInt32Config()
+{
+    return AVConfig(AVBasicType::INT32);
+}
+
+
+class Test : public vespalib::TestApp
+{
+private:
+    void
+    requireThatUpdaterAndFlusherCanRunConcurrently(void);
+
+    void
+    requireThatFlushableAttributeReportsMemoryUsage(void);
+
+    void
+    requireThatFlushableAttributeManagesSyncTokenInfo(void);
+
+    void
+    requireThatFlushTargetsCanBeRetrieved(void);
+
+    void
+    requireThatCleanUpIsPerformedAfterFlush(void);
+
+    void
+    requireThatFlushStatsAreUpdated(void);
+
+    void
+    requireThatOnlyOneFlusherCanRunAtTheSameTime(void);
+
+    void
+    requireThatLastFlushTimeIsReported(void);
+
+    void
+    requireThatShrinkWorks();
+public:
+    int
+    Main(void);
+};
+
+
+const string test_dir = "flush";
+
+struct BaseFixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext   _fileHeaderContext;
+    ForegroundTaskExecutor   _attributeFieldWriter;
+    BaseFixture()
+        : _dirHandler(test_dir),
+          _fileHeaderContext(),
+          _attributeFieldWriter()
+    {
+    }
+};
+
+
+struct AttributeManagerFixture
+{
+    AttributeManager::SP _msp;
+    AttributeManager &_m;
+    AttributeWriter _aw;
+    AttributeManagerFixture(BaseFixture &bf)
+        : _msp(std::make_shared<AttributeManager>
+               (test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext,
+                bf._attributeFieldWriter)),
+          _m(*_msp),
+          _aw(_msp)
+    {
+    }
+    AttributeVector::SP addAttribute(const vespalib::string &name) {
+        return _m.addAttribute(name, getInt32Config(), createSerialNum);
+    }
+};
+
+struct Fixture : public BaseFixture, public AttributeManagerFixture
+{
+    Fixture()
+        : BaseFixture(),
+          AttributeManagerFixture(*static_cast<BaseFixture *>(this))
+    {
+    }
+};
+
+
+
+void
+Test::requireThatUpdaterAndFlusherCanRunConcurrently(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    EXPECT_TRUE(f.addAttribute("a1").get() != NULL);
+    IFlushTarget::SP ft = am.getFlushable("a1");
+    (static_cast<FlushableAttribute *>(ft.get()))->setCleanUpAfterFlush(false);
+    UpdaterTask updaterTask(am);
+    updaterTask.run();
+
+    IndexMetaInfo info("flush/a1");
+    EXPECT_TRUE(info.load());
+    EXPECT_TRUE(info.snapshots().size() > 2);
+    for (size_t i = 0; i < info.snapshots().size(); ++i) {
+        const IndexMetaInfo::Snapshot & snap = info.snapshots()[i];
+        LOG(info,
+            "Snapshot(%" PRIu64 ", %s)",
+            snap.syncToken, snap.dirName.c_str());
+        if (snap.syncToken > 0) {
+            EXPECT_TRUE(snap.valid);
+            std::string baseFileName = "flush/a1/" + snap.dirName + "/a1";
+            AttributeVector::SP attr =
+                AttributeFactory::createAttribute(baseFileName,
+                        getInt32Config());
+            EXPECT_TRUE(attr->load());
+            EXPECT_EQUAL((uint32_t)snap.syncToken + 1, attr->getNumDocs());
+        }
+    }
+}
+
+
+void
+Test::requireThatFlushableAttributeReportsMemoryUsage(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a2");
+    av->addDocs(100);
+    av->commit();
+    IFlushTarget::SP fa = am.getFlushable("a2");
+    EXPECT_TRUE(av->getStatus().getAllocated() >= 100u * sizeof(int32_t));
+    EXPECT_EQUAL(av->getStatus().getUsed(),
+               fa->getApproxMemoryGain().getBefore()+0lu);
+    // attributes stay in memory
+    EXPECT_EQUAL(fa->getApproxMemoryGain().getBefore(),
+               fa->getApproxMemoryGain().getAfter());
+}
+
+
+void
+Test::requireThatFlushableAttributeManagesSyncTokenInfo(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a3");
+    av->addDocs(1);
+    IFlushTarget::SP fa = am.getFlushable("a3");
+
+    IndexMetaInfo info("flush/a3");
+    EXPECT_EQUAL(0u, fa->getFlushedSerialNum());
+    EXPECT_TRUE(fa->initFlush(0).get() == NULL);
+    EXPECT_TRUE(info.load());
+    EXPECT_EQUAL(0u, info.snapshots().size());
+
+    av->commit(10, 10); // last sync token = 10
+    EXPECT_EQUAL(0u, fa->getFlushedSerialNum());
+    EXPECT_TRUE(fa->initFlush(10).get() != NULL);
+    fa->initFlush(10)->run();
+    EXPECT_EQUAL(10u, fa->getFlushedSerialNum());
+    EXPECT_TRUE(info.load());
+    EXPECT_EQUAL(1u, info.snapshots().size());
+    EXPECT_TRUE(info.snapshots()[0].valid);
+    EXPECT_EQUAL(10u, info.snapshots()[0].syncToken);
+
+    av->commit(20, 20); // last sync token = 20
+    EXPECT_EQUAL(10u, fa->getFlushedSerialNum());
+    fa->initFlush(20)->run();
+    EXPECT_EQUAL(20u, fa->getFlushedSerialNum());
+    EXPECT_TRUE(info.load());
+    EXPECT_EQUAL(1u, info.snapshots().size()); // snapshot 10 removed
+    EXPECT_TRUE(info.snapshots()[0].valid);
+    EXPECT_EQUAL(20u, info.snapshots()[0].syncToken);
+}
+
+
+void
+Test::requireThatFlushTargetsCanBeRetrieved(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    f.addAttribute("a4");
+    f.addAttribute("a5");
+    std::vector<IFlushTarget::SP> ftl = am.getFlushTargets();
+    EXPECT_EQUAL(2u, ftl.size());
+    EXPECT_EQUAL(am.getFlushable("a4").get(), ftl[0].get());
+    EXPECT_EQUAL(am.getFlushable("a5").get(), ftl[1].get());
+}
+
+
+void
+Test::requireThatCleanUpIsPerformedAfterFlush(void)
+{
+    Fixture f;
+    AttributeVector::SP av = f.addAttribute("a6");
+    av->addDocs(1);
+    av->commit(30, 30);
+
+    // fake up some snapshots
+    std::string snap10 = "flush/a6/snapshot-10";
+    std::string snap20 = "flush/a6/snapshot-20";
+    vespalib::mkdir(snap10, false);
+    vespalib::mkdir(snap20, false);
+    IndexMetaInfo info("flush/a6");
+    info.addSnapshot(IndexMetaInfo::Snapshot(true, 10, "snapshot-10"));
+    info.addSnapshot(IndexMetaInfo::Snapshot(false, 20, "snapshot-20"));
+    EXPECT_TRUE(info.save());
+
+    FlushableAttribute fa(av, "flush", TuneFileAttributes(),
+                          f._fileHeaderContext, f._attributeFieldWriter);
+    fa.initFlush(30)->run();
+
+    EXPECT_TRUE(info.load());
+    EXPECT_EQUAL(1u, info.snapshots().size()); // snapshots 10 & 20 removed
+    EXPECT_TRUE(info.snapshots()[0].valid);
+    EXPECT_EQUAL(30u, info.snapshots()[0].syncToken);
+    FastOS_StatInfo statInfo;
+    EXPECT_TRUE(!FastOS_File::Stat(snap10.c_str(), &statInfo));
+    EXPECT_TRUE(!FastOS_File::Stat(snap20.c_str(), &statInfo));
+}
+
+
+void
+Test::requireThatFlushStatsAreUpdated(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a7");
+    av->addDocs(1);
+    av->commit(100,100);
+    IFlushTarget::SP ft = am.getFlushable("a7");
+    ft->initFlush(101)->run();
+    FlushStats stats = ft->getLastFlushStats();
+    EXPECT_EQUAL("flush/a7/snapshot-101", stats.getPath());
+    EXPECT_EQUAL(8u, stats.getPathElementsToLog());
+}
+
+
+void
+Test::requireThatOnlyOneFlusherCanRunAtTheSameTime(void)
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a8");
+    av->addDocs(10000);
+    av->commit(9,9);
+    IFlushTarget::SP ft = am.getFlushable("a8");
+    (static_cast<FlushableAttribute *>(ft.get()))->setCleanUpAfterFlush(false);
+    vespalib::ThreadStackExecutor exec(16, 64000);
+
+    for (size_t i = 10; i < 100; ++i) {
+        av->commit(i, i);
+        vespalib::Executor::Task::UP task = ft->initFlush(i);
+        exec.execute(std::move(task));
+    }
+    exec.sync();
+    exec.shutdown();
+
+    IndexMetaInfo info("flush/a8");
+    ASSERT_TRUE(info.load());
+    LOG(info, "Found %zu snapshots", info.snapshots().size());
+    for (size_t i = 0; i < info.snapshots().size(); ++i) {
+        EXPECT_EQUAL(true, info.snapshots()[i].valid);
+    }
+    IndexMetaInfo::Snapshot best = info.getBestSnapshot();
+    EXPECT_EQUAL(true, best.valid);
+    EXPECT_EQUAL(99u, best.syncToken);
+    FlushStats stats = ft->getLastFlushStats();
+    EXPECT_EQUAL("flush/a8/snapshot-99", stats.getPath());
+}
+
+
+void
+Test::requireThatLastFlushTimeIsReported(void)
+{
+    BaseFixture f;
+    FastOS_StatInfo stat;
+    { // no meta info file yet
+        AttributeManagerFixture amf(f);
+        AttributeManager &am = amf._m;
+        AttributeVector::SP av = amf.addAttribute("a9");
+        EXPECT_EQUAL(0, am.getFlushable("a9")->getLastFlushTime().time());
+    }
+    { // no snapshot flushed yet
+        AttributeManagerFixture amf(f);
+        AttributeManager &am = amf._m;
+        AttributeVector::SP av = amf.addAttribute("a9");
+        IFlushTarget::SP ft = am.getFlushable("a9");
+        EXPECT_EQUAL(0, ft->getLastFlushTime().time());
+        ft->initFlush(5)->run();
+        EXPECT_TRUE(FastOS_File::Stat("flush/a9/snapshot-5", &stat));
+        EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime().time());
+    }
+    { // snapshot flushed
+        AttributeManagerFixture amf(f);
+        AttributeManager &am = amf._m;
+        amf.addAttribute("a9");
+        IFlushTarget::SP ft = am.getFlushable("a9");
+        EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime().time());
+        { // updated flush time after nothing to flush
+            FastOS_Thread::Sleep(8000);
+            fastos::TimeStamp now = fastos::ClockSystem::now();
+            Executor::Task::UP task = ft->initFlush(5);
+            EXPECT_TRUE(task.get() == NULL);
+            EXPECT_LESS(stat._modifiedTime, ft->getLastFlushTime().time());
+            EXPECT_APPROX(now.time(), ft->getLastFlushTime().time(), 8);
+        }
+    }
+}
+
+
+void
+Test::requireThatShrinkWorks()
+{
+    Fixture f;
+    AttributeManager &am = f._m;
+    AttributeVector::SP av = f.addAttribute("a10");
+    
+    av->addDocs(1000 - av->getNumDocs());
+    av->commit(10, 10);
+    IFlushTarget::SP ft = am.getFlushable("a10");
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+    AttributeGuard::UP g = am.getAttribute("a10");
+    EXPECT_FALSE(av->wantShrinkLidSpace());
+    EXPECT_FALSE(av->canShrinkLidSpace());
+    EXPECT_EQUAL(1000u, av->getNumDocs());
+    EXPECT_EQUAL(1000u, av->getCommittedDocIdLimit());
+    av->compactLidSpace(100);
+    EXPECT_TRUE(av->wantShrinkLidSpace());
+    EXPECT_FALSE(av->canShrinkLidSpace());
+    EXPECT_EQUAL(1000u, av->getNumDocs());
+    EXPECT_EQUAL(100u, av->getCommittedDocIdLimit());
+    f._aw.heartBeat(11);
+    EXPECT_TRUE(av->wantShrinkLidSpace());
+    EXPECT_FALSE(av->canShrinkLidSpace());
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+    g.reset();
+    f._aw.heartBeat(11);
+    EXPECT_TRUE(av->wantShrinkLidSpace());
+    EXPECT_TRUE(av->canShrinkLidSpace());
+    EXPECT_TRUE(ft->getApproxMemoryGain().getBefore() >
+                ft->getApproxMemoryGain().getAfter());
+    EXPECT_EQUAL(1000u, av->getNumDocs());
+    EXPECT_EQUAL(100u, av->getCommittedDocIdLimit());
+    vespalib::ThreadStackExecutor exec(1, 128 * 1024);
+    vespalib::Executor::Task::UP task = ft->initFlush(11);
+    exec.execute(std::move(task));
+    exec.sync();
+    exec.shutdown();
+    EXPECT_FALSE(av->wantShrinkLidSpace());
+    EXPECT_FALSE(av->canShrinkLidSpace());
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+    EXPECT_EQUAL(100u, av->getNumDocs());
+    EXPECT_EQUAL(100u, av->getCommittedDocIdLimit());
+}
+
+
+int
+Test::Main(void)
+{
+    TEST_INIT("attributeflush_test");
+
+    if (_argc > 0) {
+        DummyFileHeaderContext::setCreator(_argv[0]);
+    }
+    vespalib::rmdir(test_dir, true);
+    TEST_DO(requireThatUpdaterAndFlusherCanRunConcurrently());
+    TEST_DO(requireThatFlushableAttributeReportsMemoryUsage());
+    TEST_DO(requireThatFlushableAttributeManagesSyncTokenInfo());
+    TEST_DO(requireThatFlushTargetsCanBeRetrieved());
+    TEST_DO(requireThatCleanUpIsPerformedAfterFlush());
+    TEST_DO(requireThatFlushStatsAreUpdated());
+    TEST_DO(requireThatOnlyOneFlusherCanRunAtTheSameTime());
+    TEST_DO(requireThatLastFlushTimeIsReported());
+    TEST_DO(requireThatShrinkWorks());
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(proton::Test);
diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.sh b/searchcore/src/tests/proton/attribute/attributeflush_test.sh
new file mode 100755
index 00000000000..8ec2f5d8dd8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributeflush_test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+rm -rf flush
+$VALGRIND ./searchcore_attributeflush_test_app
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore b/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore
new file mode 100644
index 00000000000..3b612102a10
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore
@@ -0,0 +1 @@
+searchcore_attributes_state_explorer_test_app
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt
new file mode 100644
index 00000000000..322d22c8f0d
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attributes_state_explorer_test_app
+    SOURCES
+    attributes_state_explorer_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attributes_state_explorer_test_app COMMAND searchcore_attributes_state_explorer_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC b/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC
new file mode 100644
index 00000000000..1459d32ddae
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC
@@ -0,0 +1 @@
+attributes_state_explorer test. Take a look at attributes_state_explorer_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES b/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES
new file mode 100644
index 00000000000..f49eb2b8e86
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES
@@ -0,0 +1 @@
+attributes_state_explorer_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
new file mode 100644
index 00000000000..43eeec6086a
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attributes_state_explorer_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/test/attribute_vectors.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+
+using namespace proton;
+using namespace proton::test;
+using search::index::DummyFileHeaderContext;
+using search::AttributeVector;
+using search::TuneFileAttributes;
+using search::ForegroundTaskExecutor;
+
+const vespalib::string TEST_DIR = "test_output";
+
+struct Fixture
+{
+    DirectoryHandler _dirHandler;
+    DummyFileHeaderContext _fileHeaderContext;
+    ForegroundTaskExecutor _attributeFieldWriter;
+    AttributeManager::SP _mgr;
+    AttributeManagerExplorer _explorer;
+    Fixture()
+        : _dirHandler(TEST_DIR),
+          _fileHeaderContext(),
+          _attributeFieldWriter(),
+          _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(),
+                                    _fileHeaderContext,
+                                    _attributeFieldWriter)),
+          _explorer(_mgr)
+    {
+        addAttribute("regular");
+        addExtraAttribute("extra");
+    }
+    void addAttribute(const vespalib::string &name) {
+        _mgr->addAttribute(name, AttributeUtils::getInt32Config(), 1);
+    }
+    void addExtraAttribute(const vespalib::string &name) {
+        _mgr->addExtraAttribute(AttributeVector::SP(new Int32Attribute(name)));
+    }
+};
+
+typedef std::vector<vespalib::string> StringVector;
+
+TEST_F("require that attributes are exposed as children names", Fixture)
+{
+    StringVector children = f._explorer.get_children_names();
+    std::sort(children.begin(), children.end());
+    EXPECT_EQUAL(StringVector({"extra", "regular"}), children);
+}
+
+TEST_F("require that attributes are explorable", Fixture)
+{
+    EXPECT_TRUE(f._explorer.get_child("regular").get() != nullptr);
+    EXPECT_TRUE(f._explorer.get_child("extra").get() != nullptr);
+    EXPECT_TRUE(f._explorer.get_child("not").get() == nullptr);
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore b/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore
new file mode 100644
index 00000000000..45cd0a54f56
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore
@@ -0,0 +1 @@
+searchcore_document_field_populator_test_app
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt
new file mode 100644
index 00000000000..4c6da0a3397
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_document_field_populator_test_app
+    SOURCES
+    document_field_populator_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_document_field_populator_test_app COMMAND searchcore_document_field_populator_test_app)
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/DESC b/searchcore/src/tests/proton/attribute/document_field_populator/DESC
new file mode 100644
index 00000000000..cdc71250210
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/DESC
@@ -0,0 +1 @@
+document_field_populator test. Take a look at document_field_populator_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/FILES b/searchcore/src/tests/proton/attribute/document_field_populator/FILES
new file mode 100644
index 00000000000..21f62452acf
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/FILES
@@ -0,0 +1 @@
+document_field_populator_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp
new file mode 100644
index 00000000000..d0be50bfd2f
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("document_field_populator_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/attribute/document_field_populator.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+
+typedef search::attribute::Config AVConfig;
+typedef search::attribute::BasicType AVBasicType;
+
+Schema::AttributeField
+createAttributeField()
+{
+    return Schema::AttributeField("a1", Schema::DataType::INT32);
+}
+
+Schema
+createSchema()
+{
+    Schema schema;
+    schema.addAttributeField(createAttributeField());
+    return schema;
+}
+
+struct DocContext
+{
+    Schema _schema;
+    DocBuilder _builder;
+    DocContext()
+        : _schema(createSchema()),
+          _builder(_schema)
+    {
+    }
+    Document::UP create(uint32_t id) {
+        vespalib::string docId =
+                vespalib::make_string("id:searchdocument:searchdocument::%u", id);
+        return _builder.startDocument(docId).endDocument();
+    }
+};
+
+struct Fixture
+{
+    AttributeVector::SP _attr;
+    IntegerAttribute &_intAttr;
+    DocumentFieldPopulator _pop;
+    DocContext _ctx;
+    Fixture()
+        : _attr(search::AttributeFactory::createAttribute("a1", AVConfig(AVBasicType::INT32))),
+          _intAttr(dynamic_cast<IntegerAttribute &>(*_attr)),
+          _pop(createAttributeField(), _attr, "test"),
+          _ctx()
+    {
+        _intAttr.addDocs(2);
+        _intAttr.update(1, 100);
+        _intAttr.commit();
+    }
+};
+
+TEST_F("require that document field is populated based on attribute content", Fixture)
+{
+    // NOTE: DocumentFieldRetriever (used by DocumentFieldPopulator) is fully tested
+    // with all data types in searchcore/src/tests/proton/server/documentretriever_test.cpp.
+    {
+        Document::UP doc = f._ctx.create(1);
+        f._pop.handleExisting(1, *doc);
+        EXPECT_EQUAL(100, doc->getValue("a1")->getAsInt());
+    }
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore
new file mode 100644
index 00000000000..f3666eecb6e
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore
@@ -0,0 +1 @@
+searchcore_exclusive_attribute_read_accessor_test_app
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt
new file mode 100644
index 00000000000..c39025ae39f
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_exclusive_attribute_read_accessor_test_app
+    SOURCES
+    exclusive_attribute_read_accessor_test.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_exclusive_attribute_read_accessor_test_app COMMAND searchcore_exclusive_attribute_read_accessor_test_app)
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC
new file mode 100644
index 00000000000..ec5a407ddbd
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC
@@ -0,0 +1 @@
+exclusive_attribute_read_accessor test. Take a look at exclusive_attribute_read_accessor_test.cpp for details.
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES
new file mode 100644
index 00000000000..74a9ab77547
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES
@@ -0,0 +1 @@
+exclusive_attribute_read_accessor_test.cpp
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp
new file mode 100644
index 00000000000..7cb6a503ae8
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h>
+#include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sync.h>
+
+using namespace proton;
+using namespace search;
+using namespace vespalib;
+
+using ReadGuard = ExclusiveAttributeReadAccessor::Guard;
+
+AttributeVector::SP
+createAttribute()
+{
+    attribute::Config cfg(attribute::BasicType::INT32, attribute::CollectionType::SINGLE);
+    return search::AttributeFactory::createAttribute("myattr", cfg);
+}
+
+struct Fixture
+{
+    AttributeVector::SP attribute;
+    SequencedTaskExecutor writer;
+    ExclusiveAttributeReadAccessor accessor;
+
+    Fixture()
+        : attribute(createAttribute()),
+          writer(1),
+          accessor(attribute, writer)
+    {}
+};
+
+TEST_F("require that attribute write thread is blocked while guard is held", Fixture)
+{
+    ReadGuard::UP guard = f.accessor.takeGuard();
+    Gate gate;
+    f.writer.execute("myattr", [&gate]() { gate.countDown(); });
+    bool reachedZero = gate.await(100);
+    EXPECT_FALSE(reachedZero);
+    EXPECT_EQUAL(1u, gate.getCount());
+
+    guard.reset();
+    gate.await();
+    EXPECT_EQUAL(0u, gate.getCount());
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore b/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore
diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/.gitignore b/searchcore/src/tests/proton/bucketdb/bucketdb/.gitignore
new file mode 100644
index 00000000000..3512e4268a1
--- /dev/null
+++ b/searchcore/src/tests/proton/bucketdb/bucketdb/.gitignore
@@ -0,0 +1 @@
+searchcore_bucketdb_test_app
diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt b/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt
new file mode 100644
index 00000000000..f07ded6d89b
--- /dev/null
+++ b/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_bucketdb_test_app
+    SOURCES
+    bucketdb_test.cpp
+    DEPENDS
+    searchcore_bucketdb
+)
+vespa_add_test(NAME searchcore_bucketdb_test_app COMMAND searchcore_bucketdb_test_app)
diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/DESC b/searchcore/src/tests/proton/bucketdb/bucketdb/DESC
new file mode 100644
index 00000000000..59057628f89
--- /dev/null
+++ b/searchcore/src/tests/proton/bucketdb/bucketdb/DESC
@@ -0,0 +1 @@
+bucketdb test. Take a look at bucketdb_test.cpp for details.
diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/FILES b/searchcore/src/tests/proton/bucketdb/bucketdb/FILES
new file mode 100644
index 00000000000..c5cd1105c23
--- /dev/null
+++ b/searchcore/src/tests/proton/bucketdb/bucketdb/FILES
@@ -0,0 +1 @@
+bucketdb_test.cpp
diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp
new file mode 100644
index 00000000000..6895e469319
--- /dev/null
+++ b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp
@@ -0,0 +1,169 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("bucketdb_test");
+
+#include <vespa/searchcore/proton/bucketdb/bucket_db_explorer.h>
+#include <vespa/searchcore/proton/bucketdb/bucketdb.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+using namespace proton;
+using namespace proton::bucketdb;
+using namespace vespalib::slime;
+using storage::spi::BucketChecksum;
+using storage::spi::BucketInfo;
+using storage::spi::Timestamp;
+using vespalib::Slime;
+
+constexpr uint32_t MIN_NUM_BITS = 8u;
+const GlobalId GID_1("111111111111");
+const BucketId BUCKET_1(MIN_NUM_BITS, GID_1.convertToBucketId().getRawId());
+const Timestamp TIME_1(1u);
+const Timestamp TIME_2(2u);
+const Timestamp TIME_3(3u);
+
+typedef BucketInfo::ReadyState RS;
+typedef SubDbType SDT;
+
+void
+assertDocCount(uint32_t ready,
+               uint32_t notReady,
+               uint32_t removed,
+               const BucketState &state)
+{
+    EXPECT_EQUAL(ready, state.getReadyCount());
+    EXPECT_EQUAL(notReady, state.getNotReadyCount());
+    EXPECT_EQUAL(removed, state.getRemovedCount());
+    BucketInfo info = state;
+    EXPECT_EQUAL(ready + notReady, info.getDocumentCount());
+    EXPECT_EQUAL(ready + notReady + removed, info.getEntryCount());
+}
+
+void
+assertReady(bool expReady,
+            const BucketInfo &info)
+{
+    EXPECT_EQUAL(expReady, info.isReady());
+}
+
+struct Fixture
+{
+    BucketDB _db;
+    Fixture()
+        : _db()
+    {}
+    const BucketState &add(const Timestamp &timestamp,
+                           SubDbType subDbType) {
+        return _db.add(GID_1, BUCKET_1, timestamp, subDbType);
+    }
+    BucketState remove(const Timestamp &timestamp,
+                SubDbType subDbType) {
+        _db.remove(GID_1, BUCKET_1, timestamp, subDbType);
+        return get();
+    }
+    BucketState get() const {
+        return _db.get(BUCKET_1);
+    }
+    BucketChecksum getChecksum(const Timestamp &timestamp,
+                               SubDbType subDbType) {
+        BucketDB db;
+        BucketChecksum retval = db.add(GID_1, BUCKET_1, timestamp, subDbType).getChecksum();
+        // Must ensure empty bucket db before destruction.
+        db.remove(GID_1, BUCKET_1, timestamp, subDbType);
+        return retval;
+    }
+};
+
+TEST_F("require that bucket db tracks doc counts per sub db type", Fixture)
+{
+    assertDocCount(0, 0, 0, f.get());
+    assertDocCount(1, 0, 0, f.add(TIME_1, SDT::READY));
+    assertDocCount(1, 1, 0, f.add(TIME_2, SDT::NOTREADY));
+    assertDocCount(1, 1, 1, f.add(TIME_3, SDT::REMOVED));
+    assertDocCount(0, 1, 1, f.remove(TIME_1, SDT::READY));
+    assertDocCount(0, 0, 1, f.remove(TIME_2, SDT::NOTREADY));
+    assertDocCount(0, 0, 0, f.remove(TIME_3, SDT::REMOVED));
+}
+
+TEST_F("require that bucket checksum is a combination of sub db types", Fixture)
+{
+    BucketChecksum zero(0u);
+    BucketChecksum ready = f.getChecksum(TIME_1, SDT::READY);
+    BucketChecksum notReady = f.getChecksum(TIME_2, SDT::NOTREADY);
+
+    EXPECT_EQUAL(zero, f.get().getChecksum());
+    EXPECT_EQUAL(ready,            f.add(TIME_1, SDT::READY).getChecksum());
+    EXPECT_EQUAL(ready + notReady, f.add(TIME_2, SDT::NOTREADY).getChecksum());
+    EXPECT_EQUAL(ready + notReady, f.add(TIME_3, SDT::REMOVED).getChecksum());
+    EXPECT_EQUAL(notReady, f.remove(TIME_1, SDT::READY).getChecksum());
+    EXPECT_EQUAL(zero,     f.remove(TIME_2, SDT::NOTREADY).getChecksum());
+    EXPECT_EQUAL(zero,     f.remove(TIME_3, SDT::REMOVED).getChecksum());
+}
+
+TEST_F("require that bucket is ready when not having docs in notready sub db", Fixture)
+{
+    assertReady(true, f.get());
+    assertReady(true, f.add(TIME_1, SDT::READY));
+    assertReady(false, f.add(TIME_2, SDT::NOTREADY));
+    assertReady(false, f.add(TIME_3, SDT::REMOVED));
+    assertReady(true, f.remove(TIME_2, SDT::NOTREADY));
+    assertReady(true, f.remove(TIME_1, SDT::READY));
+    assertReady(true, f.remove(TIME_3, SDT::REMOVED));
+}
+
+TEST_F("require that bucket can be cached", Fixture)
+{
+    f.add(TIME_1, SDT::READY);
+    EXPECT_FALSE(f._db.isCachedBucket(BUCKET_1));
+    f._db.cacheBucket(BUCKET_1);
+    EXPECT_TRUE(f._db.isCachedBucket(BUCKET_1));
+
+    assertDocCount(1, 0, 0, f._db.cachedGet(BUCKET_1));
+    f.add(TIME_2, SDT::NOTREADY);
+    assertDocCount(1, 0, 0, f._db.cachedGet(BUCKET_1));
+
+    f._db.uncacheBucket();
+    EXPECT_FALSE(f._db.isCachedBucket(BUCKET_1));
+    assertDocCount(1, 1, 0, f._db.cachedGet(BUCKET_1));
+
+    // Must ensure empty bucket db before destruction.
+    f.remove(TIME_1, SDT::READY);
+    f.remove(TIME_2, SDT::NOTREADY);
+}
+
+TEST("require that bucket db can be explored")
+{
+    BucketDBOwner db;
+    db.takeGuard()->add(GID_1, BUCKET_1, TIME_1, SDT::READY);
+    {
+        BucketDBExplorer explorer(db.takeGuard());
+        Slime expectSlime;
+        vespalib::string expectJson =
+            "{"
+            "  numBuckets: 1,"
+            "  buckets: ["
+            "    {"
+            "      id: '0x2000000000000031',"
+            "      checksum: '0x93939394',"
+            "      readyCount: 1,"
+            "      notReadyCount: 0,"
+            "      removedCount: 0,"
+            "      active: false"
+            "    }"
+            "  ]"
+            "}";
+        EXPECT_TRUE(JsonFormat::decode(expectJson, expectSlime) > 0);
+        Slime actualSlime;
+        SlimeInserter inserter(actualSlime);
+        explorer.get_state(inserter, true);
+
+        EXPECT_EQUAL(expectSlime, actualSlime);
+    }
+
+    // Must ensure empty bucket db before destruction.
+    db.takeGuard()->remove(GID_1, BUCKET_1, TIME_1, SDT::READY);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/clean_tests.sh b/searchcore/src/tests/proton/clean_tests.sh
new file mode 100755
index 00000000000..c99f0a92ee8
--- /dev/null
+++ b/searchcore/src/tests/proton/clean_tests.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+for file in *
+do
+  if [ -d "$file" ]; then
+    (cd "$file" && make clean && echo "$file cleaned")
+  fi
+done
diff --git a/searchcore/src/tests/proton/common/.gitignore b/searchcore/src/tests/proton/common/.gitignore
new file mode 100644
index 00000000000..c03144e885d
--- /dev/null
+++ b/searchcore/src/tests/proton/common/.gitignore
@@ -0,0 +1,3 @@
+searchcore_cachedselect_test_app
+searchcore_schemautil_test_app
+searchcore_selectpruner_test_app
diff --git a/searchcore/src/tests/proton/common/CMakeLists.txt b/searchcore/src/tests/proton/common/CMakeLists.txt
new file mode 100644
index 00000000000..833bac4c065
--- /dev/null
+++ b/searchcore/src/tests/proton/common/CMakeLists.txt
@@ -0,0 +1,22 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_schemautil_test_app
+    SOURCES
+    schemautil_test.cpp
+    DEPENDS
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_schemautil_test_app COMMAND searchcore_schemautil_test_app)
+vespa_add_executable(searchcore_selectpruner_test_app
+    SOURCES
+    selectpruner_test.cpp
+    DEPENDS
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_selectpruner_test_app COMMAND searchcore_selectpruner_test_app)
+vespa_add_executable(searchcore_cachedselect_test_app
+    SOURCES
+    cachedselect_test.cpp
+    DEPENDS
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_cachedselect_test_app COMMAND searchcore_cachedselect_test_app)
diff --git a/searchcore/src/tests/proton/common/cachedselect_test.cpp b/searchcore/src/tests/proton/common/cachedselect_test.cpp
new file mode 100644
index 00000000000..e2c759f72aa
--- /dev/null
+++ b/searchcore/src/tests/proton/common/cachedselect_test.cpp
@@ -0,0 +1,710 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("cachedselect_test");
+
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/proton/common/cachedselect.h>
+#include <vespa/searchcore/proton/common/selectcontext.h>
+#include <vespa/searchlib/attribute/attributecontext.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/postinglistattribute.h>
+#include <vespa/searchlib/attribute/enumcomparator.h>
+#include <vespa/searchlib/attribute/singlenumericpostattribute.h>
+#include <vespa/searchlib/attribute/singleenumattribute.hpp>
+#include <vespa/searchlib/attribute/singlenumericenumattribute.hpp>
+#include <vespa/searchlib/attribute/singlenumericpostattribute.hpp>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/select/cloningvisitor.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+
+
+using search::index::Schema;
+using document::DocumentTypeRepo;
+using document::DocumentType;
+using document::select::Node;
+using document::select::Result;
+using document::select::ResultSet;
+using document::select::CloningVisitor;
+using document::select::Context;
+using vespalib::string;
+
+using document::config_builder::DocumenttypesConfigBuilderHelper;
+using document::config_builder::Struct;
+using document::config_builder::Array;
+using document::config_builder::Wset;
+using document::config_builder::Map;
+using document::DataType;
+using document::Document;
+using document::DocumentId;
+using document::StringFieldValue;
+using document::IntFieldValue;
+using proton::CachedSelect;
+using proton::SelectContext;
+using search::AttributeVector;
+using search::AttributeGuard;
+using search::AttributeEnumGuard;
+using search::AttributeContext;
+using search::EnumAttribute;
+using search::AttributePosting;
+using search::SingleValueNumericPostingAttribute;
+using search::IntegerAttribute;
+using search::IntegerAttributeTemplate;
+using search::attribute::IAttributeContext;
+
+typedef Node::UP NodeUP;
+typedef IntegerAttributeTemplate<int32_t> IATint32;
+typedef EnumAttribute<IATint32> IntEnumAttribute;
+
+#if 0
+extern template class SingleValueNumericPostingAttribute<IntPostingAttribute>;
+#endif
+
+typedef SingleValueNumericPostingAttribute<IntEnumAttribute> SvIntAttr;
+
+namespace
+{
+
+void
+makeSchema(Schema &s)
+{
+    s.addIndexField(Schema::IndexField("ia", Schema::STRING));
+    s.addAttributeField(Schema::AttributeField("aa", Schema::INT32));
+    s.addAttributeField(Schema::AttributeField("aaa", Schema::INT32,
+                                               Schema::ARRAY));
+    s.addAttributeField(Schema::AttributeField("aaw", Schema::INT32,
+                                               Schema::WEIGHTEDSET));
+}
+
+const int32_t doc_type_id = 787121340;
+const string type_name = "test";
+const string header_name = type_name + ".header";
+const string body_name = type_name + ".body";
+const string type_name_2 = "test_2";
+const string header_name_2 = type_name_2 + ".header";
+const string body_name_2 = type_name_2 + ".body";
+
+const int32_t noIntVal = std::numeric_limits<int32_t>::min();
+
+
+DocumentTypeRepo::UP
+makeDocTypeRepo(void)
+{
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name).
+                     addField("ia", DataType::T_STRING).
+                     addField("ib", DataType::T_STRING).
+                     addField("ibs", Struct("pair").
+                              addField("x", DataType::T_STRING).
+                              addField("y", DataType::T_STRING)).
+                     addField("iba", Array(DataType::T_STRING)).
+                     addField("ibw", Wset(DataType::T_STRING)).
+                     addField("ibm", Map(DataType::T_STRING,
+                                         DataType::T_STRING)).
+                     addField("aa", DataType::T_INT).
+                     addField("aaa", Array(DataType::T_INT)).
+                     addField("aaw", Wset(DataType::T_INT)).
+                     addField("ab", DataType::T_INT));
+    builder.document(doc_type_id + 1, type_name_2,
+                     Struct(header_name_2), Struct(body_name_2).
+                     addField("ic", DataType::T_STRING).
+                     addField("id", DataType::T_STRING).
+                     addField("ac", DataType::T_INT).
+                     addField("ad", DataType::T_INT));
+    return DocumentTypeRepo::UP(new DocumentTypeRepo(builder.config()));
+}
+
+
+Document::UP
+makeDoc(const DocumentTypeRepo &repo,
+        const string &docId,
+        const string &ia,
+        const string &ib,
+        int32_t aa,
+        int32_t ab)
+{
+    const DocumentType *docType = repo.getDocumentType("test");
+    Document::UP doc(new Document(*docType, DocumentId(docId)));
+    if (ia != "null")
+        doc->setValue("ia", StringFieldValue(ia));
+    if (ib != "null")
+        doc->setValue("ib", StringFieldValue(ib));
+    if (aa != noIntVal)
+        doc->setValue("aa", IntFieldValue(aa));
+    if (ab != noIntVal)
+        doc->setValue("ab", IntFieldValue(ab));
+    return doc;
+}
+
+
+bool
+checkSelect(const NodeUP &sel,
+            const Context &ctx,
+            const Result &exp)
+{
+    if (EXPECT_TRUE(sel->contains(ctx) == exp))
+        return true;
+    std::ostringstream os;
+    EXPECT_TRUE(sel->trace(ctx, os) == exp);
+    LOG(info,
+        "trace output: '%s'",
+        os.str().c_str());
+    return false;
+}
+
+bool
+checkSelect(const CachedSelect::SP &cs,
+            const Context &ctx,
+            const Result &exp)
+{
+    return checkSelect(cs->_select, ctx, exp);
+}
+
+
+bool
+checkSelect(const CachedSelect::SP &cs,
+            uint32_t docId,
+            const Result &exp)
+{
+    SelectContext ctx(*cs);
+    ctx._docId = docId;
+    ctx.getAttributeGuards();
+    return checkSelect(cs->_attrSelect, ctx, exp);
+}
+
+
+class MyIntAv : public SvIntAttr
+{
+    mutable uint32_t _gets;
+public:
+    MyIntAv(const string &name)
+        : SvIntAttr(name, Config(BasicType::INT32,
+                                 CollectionType::SINGLE,
+                                 true, false)),
+          _gets(0)
+    {
+    }
+
+    virtual uint32_t
+    get(AttributeVector::DocId doc, largeint_t *v, uint32_t sz) const
+    {
+        ++_gets;
+        return SvIntAttr::get(doc, v, sz);
+    }
+
+    uint32_t
+    getGets(void) const
+    {
+        return _gets;
+    }
+};
+
+
+
+class MyAttributeManager : public search::IAttributeManager
+{
+public:
+    typedef std::map<string, AttributeVector::SP> AttributeMap;
+
+    AttributeMap _attributes;
+
+    AttributeVector::SP
+    findAttribute(const vespalib::string &name) const
+    {
+        AttributeMap::const_iterator itr = _attributes.find(name);
+        if (itr != _attributes.end()) {
+            return itr->second;
+        }
+        return AttributeVector::SP();
+    }
+
+    virtual
+    AttributeGuard::UP
+    getAttribute(const string &name) const
+    {
+        AttributeVector::SP attr = findAttribute(name);
+        return AttributeGuard::UP(new AttributeGuard(attr));
+    }
+
+    virtual AttributeGuard::UP
+    getAttributeStableEnum(const string & name) const
+    {
+        AttributeVector::SP attr = findAttribute(name);
+        return AttributeGuard::UP(new AttributeEnumGuard(attr));
+    }
+
+    virtual void
+    getAttributeList(std::vector<AttributeGuard> & list) const
+    {
+        list.reserve(_attributes.size());
+        for (AttributeMap::const_iterator itr = _attributes.begin();
+             itr != _attributes.end();
+             ++itr) {
+            list.push_back(AttributeGuard(itr->second));
+        }
+    }
+
+    virtual IAttributeContext::UP
+    createContext() const
+    {
+        return IAttributeContext::UP(new AttributeContext(*this));
+    }
+
+    MyAttributeManager()
+        : _attributes()
+    {
+    }
+
+    void
+    addAttribute(const string &name)
+    {
+        if (findAttribute(name).get() != NULL)
+            return;
+        AttributeVector::SP av(new MyIntAv(name));
+        av->addReservedDoc();
+        _attributes[name] = av;
+    }
+};
+
+
+class MyDB
+{
+public:
+    typedef std::unique_ptr<MyDB> UP;
+
+    const Schema &_schema;
+    const DocumentTypeRepo &_repo;
+    MyAttributeManager &_amgr;
+    typedef std::map<string, uint32_t> DocIdToLid;
+    typedef std::map<uint32_t, Document::SP> LidToDocSP;
+    DocIdToLid _docIdToLid;
+    LidToDocSP _lidToDocSP;
+
+    MyDB(const Schema &schema,
+         const DocumentTypeRepo &repo,
+         MyAttributeManager &amgr)
+        : _schema(schema),
+          _repo(repo),
+          _amgr(amgr)
+    {
+    }
+
+    void
+    addDoc(uint32_t lid,
+           const string &docId,
+           const string &ia,
+           const string &ib,
+           int32_t aa,
+           int32_t ab);
+
+    const Document &
+    getDoc(uint32_t lid) const;
+};
+
+
+void
+MyDB::addDoc(uint32_t lid,
+             const string &docId,
+             const string &ia,
+             const string &ib,
+             int32_t aa,
+             int32_t ab)
+{
+    Document::UP doc(makeDoc(_repo, docId, ia, ib, aa, ab));
+
+    _docIdToLid[docId] = lid;
+    _lidToDocSP[lid] = Document::SP(doc.release());
+    AttributeVector &av(*_amgr.findAttribute("aa"));
+    if (lid >= av.getNumDocs()) {
+        AttributeVector::DocId checkDocId(0u);
+        ASSERT_TRUE(av.addDoc(checkDocId));
+        ASSERT_EQUAL(lid, checkDocId);
+    }
+    IntegerAttribute &iav(static_cast<IntegerAttribute &>(av));
+    AttributeVector::largeint_t laa(aa);
+    EXPECT_TRUE(iav.update(lid, laa));
+    av.commit();
+}
+
+
+const Document &
+MyDB::getDoc(uint32_t lid) const
+{
+    LidToDocSP::const_iterator it(_lidToDocSP.find(lid));
+    ASSERT_TRUE(it != _lidToDocSP.end());
+    return *it->second;
+}
+
+
+class TestFixture
+{
+public:
+    Schema _s;
+    DocumentTypeRepo::UP _repoUP;
+    bool _hasFields;
+    MyAttributeManager _amgr;
+    MyDB::UP _db;
+
+    TestFixture(void);
+
+    ~TestFixture(void);
+
+    CachedSelect::SP
+    testParse(const string &selection,
+              const string &docTypeName);
+
+};
+
+
+TestFixture::TestFixture(void)
+    : _s(),
+      _repoUP(),
+      _hasFields(true),
+      _amgr(),
+      _db()
+{
+    makeSchema(_s);
+    _repoUP = makeDocTypeRepo();
+
+    _amgr.addAttribute("aa");
+
+    _db.reset(new MyDB(_s, *_repoUP, _amgr));
+}
+
+
+TestFixture::~TestFixture(void)
+{
+}
+
+
+CachedSelect::SP
+TestFixture::testParse(const string &selection,
+                       const string &docTypeName)
+{
+    const DocumentTypeRepo &repo(*_repoUP);
+    const Schema &schema(_s);
+
+    CachedSelect::SP res(new CachedSelect);
+
+    const DocumentType *docType = repo.getDocumentType(docTypeName);
+    ASSERT_TRUE(docType != NULL);
+    Document::UP emptyDoc(new Document(*docType, DocumentId()));
+
+    res->set(selection,
+             docTypeName,
+             *emptyDoc,
+             repo,
+             schema,
+             &_amgr,
+             _hasFields);
+    
+    ASSERT_TRUE(res->_select.get() != NULL);
+    return res;
+}
+
+
+TEST_F("Test that test setup is OK", TestFixture)
+{
+    DocumentTypeRepo &repo = *f._repoUP;
+    const DocumentType *docType = repo.getDocumentType("test");
+    ASSERT_TRUE(docType);
+    EXPECT_EQUAL(10u, docType->getFieldCount());
+    EXPECT_EQUAL("String", docType->getField("ia").getDataType().getName());
+    EXPECT_EQUAL("String", docType->getField("ib").getDataType().getName());
+    EXPECT_EQUAL("Int", docType->getField("aa").getDataType().getName());
+    EXPECT_EQUAL("Int", docType->getField("ab").getDataType().getName());
+}
+
+
+TEST_F("Test that simple parsing works", TestFixture)
+{
+    f.testParse("not ((test))", "test");
+    f.testParse("not ((test and (test.aa > 3999)))", "test");
+    f.testParse("not ((test and (test.ab > 3999)))", "test");
+    f.testParse("not ((test and (test.af > 3999)))", "test");
+    f.testParse("not ((test_2 and (test_2.af > 3999)))", "test");
+}
+
+
+TEST_F("Test that const is flagged", TestFixture)
+{
+    CachedSelect::SP cs;
+
+    cs = f.testParse("false", "test");
+    EXPECT_TRUE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    cs = f.testParse("true", "test");
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_TRUE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    cs = f.testParse("test_2.ac > 4999", "test");
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_TRUE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    cs = f.testParse("test.aa > 4999", "test");
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(1u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(1u, cs->_svAttrFieldNodes);
+}
+
+
+TEST_F("Test that basic select works", TestFixture)
+{
+    MyDB &db(*f._db);
+    
+    db.addDoc(1u, "doc:test:1", "hello", "null", 45, 37);
+    db.addDoc(2u, "doc:test:2", "gotcha", "foo", 3, 25);
+    db.addDoc(3u, "doc:test:3", "gotcha", "foo", noIntVal, noIntVal);
+    db.addDoc(4u, "doc:test:4", "null", "foo", noIntVal, noIntVal);
+    
+    CachedSelect::SP cs;
+
+    cs = f.testParse("test.ia == \"hello\"", "test");
+    EXPECT_FALSE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(0u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::True));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::False));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::False));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::False));
+    
+    cs = f.testParse("test.ia.foo == \"hello\"", "test");
+    EXPECT_FALSE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_TRUE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    EXPECT_EQUAL(0u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::Invalid));
+    
+    cs = f.testParse("test.ia[2] == \"hello\"", "test");
+    EXPECT_FALSE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_TRUE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    EXPECT_EQUAL(0u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::Invalid));
+    
+    cs = f.testParse("test.ia{foo} == \"hello\"", "test");
+    EXPECT_FALSE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_TRUE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    EXPECT_EQUAL(0u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::Invalid));
+    
+    cs = f.testParse("test.ia < \"hello\"", "test");
+    EXPECT_FALSE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(0u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::False));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::True));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::True));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::Invalid));
+
+    cs = f.testParse("test.aa == 3", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(1u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(1u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::False));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::True));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::False));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::False));
+    TEST_DO(checkSelect(cs, 1u, Result::False));
+    TEST_DO(checkSelect(cs, 2u, Result::True));
+    TEST_DO(checkSelect(cs, 3u, Result::False));
+    TEST_DO(checkSelect(cs, 4u, Result::False));
+
+    cs = f.testParse("test.aa == 3", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(1u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(1u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::False));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::True));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::False));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::False));
+    TEST_DO(checkSelect(cs, 1u, Result::False));
+    TEST_DO(checkSelect(cs, 2u, Result::True));
+    TEST_DO(checkSelect(cs, 3u, Result::False));
+    TEST_DO(checkSelect(cs, 4u, Result::False));
+
+    cs = f.testParse("test.aa.foo == 3", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() == NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_TRUE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    EXPECT_EQUAL(0u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::Invalid));
+
+    cs = f.testParse("test.aa[2] == 3", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() == NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_TRUE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    EXPECT_EQUAL(0u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::Invalid));
+
+    cs = f.testParse("test.aa{4} > 3", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() == NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_TRUE(cs->_allInvalid);
+    EXPECT_EQUAL(0u, cs->_fieldNodes);
+    EXPECT_EQUAL(0u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::Invalid));
+
+    cs = f.testParse("test.aaa[2] == 3", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() == NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(1u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+
+    cs = f.testParse("test.aaw{4} > 3", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() == NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(1u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(0u, cs->_svAttrFieldNodes);
+
+    cs = f.testParse("test.aa < 45", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(1u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(1u, cs->_svAttrFieldNodes);
+    TEST_DO(checkSelect(cs, db.getDoc(1u), Result::False));
+    TEST_DO(checkSelect(cs, db.getDoc(2u), Result::True));
+    TEST_DO(checkSelect(cs, db.getDoc(3u), Result::Invalid));
+    TEST_DO(checkSelect(cs, db.getDoc(4u), Result::Invalid));
+    TEST_DO(checkSelect(cs, 1u, Result::False));
+    TEST_DO(checkSelect(cs, 2u, Result::True));
+    TEST_DO(checkSelect(cs, 3u, Result::Invalid));
+    TEST_DO(checkSelect(cs, 4u, Result::Invalid));
+
+    MyIntAv *v(dynamic_cast<MyIntAv *>(f._amgr.findAttribute("aa").get()));
+    EXPECT_TRUE(v != NULL);
+    EXPECT_EQUAL(6u, v->getGets());
+}
+
+
+TEST_F("Test performance when using attributes", TestFixture)
+{
+    MyDB &db(*f._db);
+    
+    db.addDoc(1u, "doc:test:1", "hello", "null", 45, 37);
+    db.addDoc(2u, "doc:test:2", "gotcha", "foo", 3, 25);
+    db.addDoc(3u, "doc:test:3", "gotcha", "foo", noIntVal, noIntVal);
+    db.addDoc(4u, "doc:test:4", "null", "foo", noIntVal, noIntVal);
+    
+    CachedSelect::SP cs;
+    cs = f.testParse("test.aa < 45", "test");
+    EXPECT_TRUE(cs->_attrSelect.get() != NULL);
+    EXPECT_FALSE(cs->_allFalse);
+    EXPECT_FALSE(cs->_allTrue);
+    EXPECT_FALSE(cs->_allInvalid);
+    EXPECT_EQUAL(1u, cs->_fieldNodes);
+    EXPECT_EQUAL(1u, cs->_attrFieldNodes);
+    EXPECT_EQUAL(1u, cs->_svAttrFieldNodes);
+    SelectContext ctx(*cs);
+    ctx.getAttributeGuards();
+    const NodeUP &sel(cs->_attrSelect);
+    uint32_t i;
+    const uint32_t loopcnt = 30000;
+    LOG(info, "Starting minibm loop, %u ierations of 4 docs each", loopcnt);
+    fastos::StopWatchT<fastos::ClockSystem> sw;
+    sw.start();
+    for (i = 0; i < loopcnt; ++i) {
+        ctx._docId = 1u;
+        if (sel->contains(ctx) != Result::False)
+            break;
+        ctx._docId = 2u;
+        if (sel->contains(ctx) != Result::True)
+            break;
+        ctx._docId = 3u;
+        if (sel->contains(ctx) != Result::Invalid)
+            break;
+        ctx._docId = 4u;
+        if (sel->contains(ctx) != Result::Invalid)
+            break;
+    }
+    sw.stop();
+    EXPECT_EQUAL(loopcnt, i);
+    LOG(info,
+        "Elapsed time for %u iterations of 4 docs each: %" PRId64 " ns, "
+        "%8.4f ns/doc",
+        i,
+        sw.elapsed().ns(),
+        static_cast<double>(sw.elapsed().ns()) / ( 4 * i));
+    
+}
+
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/common/document_type_inspector/.gitignore b/searchcore/src/tests/proton/common/document_type_inspector/.gitignore
new file mode 100644
index 00000000000..49db4ae7746
--- /dev/null
+++ b/searchcore/src/tests/proton/common/document_type_inspector/.gitignore
@@ -0,0 +1 @@
+searchcore_document_type_inspector_test_app
diff --git a/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt b/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt
new file mode 100644
index 00000000000..f5c4610fc1b
--- /dev/null
+++ b/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_document_type_inspector_test_app
+    SOURCES
+    document_type_inspector_test.cpp
+    DEPENDS
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_document_type_inspector_test_app COMMAND searchcore_document_type_inspector_test_app)
diff --git a/searchcore/src/tests/proton/common/document_type_inspector/DESC b/searchcore/src/tests/proton/common/document_type_inspector/DESC
new file mode 100644
index 00000000000..b40b3219939
--- /dev/null
+++ b/searchcore/src/tests/proton/common/document_type_inspector/DESC
@@ -0,0 +1,2 @@
+Test for document type inspector. Take a look at document_type_inspector_test.cpp for details.
+
diff --git a/searchcore/src/tests/proton/common/document_type_inspector/FILES b/searchcore/src/tests/proton/common/document_type_inspector/FILES
new file mode 100644
index 00000000000..1a0b0b31a76
--- /dev/null
+++ b/searchcore/src/tests/proton/common/document_type_inspector/FILES
@@ -0,0 +1 @@
+document_type_inspector_test.cpp
diff --git a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp b/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp
new file mode 100644
index 00000000000..c7fcd9ec72d
--- /dev/null
+++ b/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("document_type_inspector_test");
+
+#include <vespa/searchcore/proton/common/document_type_inspector.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+using namespace proton;
+using namespace search::index;
+
+Schema
+getSchema()
+{
+    Schema schema;
+    schema.addSummaryField(Schema::SummaryField("f1", Schema::STRING));
+    schema.addSummaryField(Schema::SummaryField("f2", Schema::STRING));
+    return schema;
+}
+
+struct Fixture
+{
+    Schema _schema;
+    DocBuilder _builder;
+    DocumentTypeInspector _inspector;
+    Fixture()
+        : _schema(getSchema()),
+          _builder(_schema),
+          _inspector(_builder.getDocumentType())
+    {
+    }
+};
+
+TEST_F("require that existing fields are known", Fixture)
+{
+    EXPECT_TRUE(f._inspector.hasField("f1"));
+    EXPECT_TRUE(f._inspector.hasField("f2"));
+}
+
+TEST_F("require that non-existing fields are NOT known", Fixture)
+{
+    EXPECT_FALSE(f._inspector.hasField("not"));
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/common/dummydbowner.h b/searchcore/src/tests/proton/common/dummydbowner.h
new file mode 100644
index 00000000000..8e3748b5072
--- /dev/null
+++ b/searchcore/src/tests/proton/common/dummydbowner.h
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/server/idocumentdbowner.h>
+#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton
+{
+
+struct DummyDBOwner : IDocumentDBOwner {
+    bool isInitializing() const override { return false; }
+
+    searchcorespi::IIndexManagerFactory::SP
+    getIndexManagerFactory(const vespalib::stringref & ) const override {
+        return searchcorespi::IIndexManagerFactory::SP();
+    }
+    uint32_t getDistributionKey() const override { return -1; }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/tests/proton/common/schemautil_test.cpp b/searchcore/src/tests/proton/common/schemautil_test.cpp
new file mode 100644
index 00000000000..c6519ecae06
--- /dev/null
+++ b/searchcore/src/tests/proton/common/schemautil_test.cpp
@@ -0,0 +1,132 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for schemautil.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("schemautil_test");
+
+#include <vespa/searchcore/proton/common/schemautil.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using search::index::Schema;
+using vespalib::string;
+
+using namespace proton;
+
+namespace {
+
+void addAllFieldTypes(const string &name, Schema &schema,
+                      fastos::TimeStamp timestamp)
+{
+    Schema::IndexField index_field(name, Schema::STRING);
+    index_field.setTimestamp(timestamp);
+    schema.addIndexField(index_field);
+
+    Schema::AttributeField attribute_field(name, Schema::STRING);
+    attribute_field.setTimestamp(timestamp);
+    schema.addAttributeField(attribute_field);
+
+    Schema::SummaryField summary_field(name, Schema::STRING);
+    summary_field.setTimestamp(timestamp);
+    schema.addSummaryField(summary_field);
+}
+
+TEST("require that makeHistorySchema sets timestamp")
+{
+    Schema old_schema;
+    Schema new_schema;
+    Schema old_history;
+
+    const fastos::TimeStamp now(84);
+    const string name = "foo";
+    addAllFieldTypes(name, old_schema, fastos::TimeStamp(0));
+
+    Schema::SP schema = SchemaUtil::makeHistorySchema(new_schema, old_schema,
+                                                      old_history, now);
+
+    ASSERT_EQUAL(1u, schema->getNumIndexFields());
+    EXPECT_EQUAL(name, schema->getIndexField(0).getName());
+    EXPECT_EQUAL(now, schema->getIndexField(0).getTimestamp());
+
+    ASSERT_EQUAL(1u, schema->getNumAttributeFields());
+    EXPECT_EQUAL(name, schema->getAttributeField(0).getName());
+    EXPECT_EQUAL(now, schema->getAttributeField(0).getTimestamp());
+
+    ASSERT_EQUAL(1u, schema->getNumSummaryFields());
+    EXPECT_EQUAL(name, schema->getSummaryField(0).getName());
+    EXPECT_EQUAL(now, schema->getSummaryField(0).getTimestamp());
+}
+
+TEST("require that makeHistorySchema preserves timestamp")
+{
+    Schema old_schema;
+    Schema new_schema;
+    Schema old_history;
+
+    const fastos::TimeStamp timestamp(42);
+    const string name = "foo";
+    addAllFieldTypes("bar", old_schema, fastos::TimeStamp(0));
+    addAllFieldTypes(name, old_history, timestamp);
+
+    Schema::SP schema =
+        SchemaUtil::makeHistorySchema(new_schema, old_schema, old_history);
+
+    ASSERT_EQUAL(2u, schema->getNumIndexFields());
+    uint32_t id = schema->getIndexFieldId(name);
+    ASSERT_NOT_EQUAL(id, Schema::UNKNOWN_FIELD_ID);
+    EXPECT_EQUAL(timestamp, schema->getIndexField(id).getTimestamp());
+
+    ASSERT_EQUAL(2u, schema->getNumAttributeFields());
+    id = schema->getAttributeFieldId(name);
+    ASSERT_NOT_EQUAL(id, Schema::UNKNOWN_FIELD_ID);
+    EXPECT_EQUAL(timestamp, schema->getAttributeField(id).getTimestamp());
+
+    ASSERT_EQUAL(2u, schema->getNumSummaryFields());
+    id = schema->getSummaryFieldId(name);
+    ASSERT_NOT_EQUAL(id, Schema::UNKNOWN_FIELD_ID);
+    EXPECT_EQUAL(timestamp, schema->getSummaryField(id).getTimestamp());
+}
+
+struct ListSchemaResult {
+    std::vector<vespalib::string> fieldNames;
+    std::vector<vespalib::string> fieldDataTypes;
+    std::vector<vespalib::string> fieldCollectionTypes;
+    std::vector<vespalib::string> fieldLocations;
+};
+
+void
+assertSchemaResult(const vespalib::string &name,
+                   const vespalib::string &dataType,
+                   const vespalib::string &collectionType,
+                   const vespalib::string &location,
+                   const ListSchemaResult &r,
+                   size_t i)
+{
+    EXPECT_EQUAL(name, r.fieldNames[i]);
+    EXPECT_EQUAL(dataType, r.fieldDataTypes[i]);
+    EXPECT_EQUAL(collectionType, r.fieldCollectionTypes[i]);
+    EXPECT_EQUAL(location, r.fieldLocations[i]);
+}
+
+TEST("require that listSchema can list all fields")
+{
+    Schema schema;
+    schema.addIndexField(Schema::IndexField("if", Schema::STRING));
+    schema.addAttributeField(Schema::AttributeField("af", Schema::INT32));
+    schema.addSummaryField(Schema::SummaryField("sf", Schema::FLOAT, Schema::ARRAY));
+
+    ListSchemaResult r;
+    SchemaUtil::listSchema(schema, r.fieldNames, r.fieldDataTypes, r.fieldCollectionTypes, r.fieldLocations);
+    EXPECT_EQUAL(3u, r.fieldNames.size());
+    EXPECT_EQUAL(3u, r.fieldDataTypes.size());
+    EXPECT_EQUAL(3u, r.fieldCollectionTypes.size());
+    EXPECT_EQUAL(3u, r.fieldLocations.size());
+    assertSchemaResult("af", "INT32", "SINGLE", "a", r, 0);
+    assertSchemaResult("if", "STRING", "SINGLE", "i", r, 1);
+    assertSchemaResult("sf", "FLOAT", "ARRAY", "s", r, 2);
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp
new file mode 100644
index 00000000000..e8ec0ae7cb5
--- /dev/null
+++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp
@@ -0,0 +1,778 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("selectpruner_test");
+
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/proton/common/selectpruner.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/select/cloningvisitor.h>
+
+
+using search::index::Schema;
+using document::DocumentTypeRepo;
+using document::DocumentType;
+using document::select::Node;
+using document::select::Result;
+using document::select::ResultSet;
+using document::select::CloningVisitor;
+using vespalib::string;
+
+using document::config_builder::DocumenttypesConfigBuilderHelper;
+using document::config_builder::Struct;
+using document::config_builder::Array;
+using document::config_builder::Wset;
+using document::config_builder::Map;
+using document::DataType;
+using document::Document;
+using proton::SelectPruner;
+
+typedef Node::UP NodeUP;
+
+namespace
+{
+
+void
+makeSchema(Schema &s)
+{
+    s.addIndexField(Schema::IndexField("ia", Schema::STRING));
+    s.addAttributeField(Schema::AttributeField("aa", Schema::INT32));
+    s.addAttributeField(Schema::AttributeField("aaa", Schema::INT32,
+                                               Schema::ARRAY));
+    s.addAttributeField(Schema::AttributeField("aaw", Schema::INT32,
+                                               Schema::WEIGHTEDSET));
+}
+
+const int32_t doc_type_id = 787121340;
+const string type_name = "test";
+const string header_name = type_name + ".header";
+const string body_name = type_name + ".body";
+const string type_name_2 = "test_2";
+const string header_name_2 = type_name_2 + ".header";
+const string body_name_2 = type_name_2 + ".body";
+const string false_name("false");
+const string true_name("true");
+const string not_name("not");
+const string valid_name("test.aa > 3999");
+const string valid2_name("test.ab > 4999");
+const string rvalid_name("test.aa <= 3999");
+const string rvalid2_name("test.ab <= 4999");
+const string invalid_name("test_2.ac > 3999");
+const string invalid2_name("test_2.ac > 4999");
+const string empty("");
+
+const document::DocumentId docId("doc:test:1");
+
+
+DocumentTypeRepo::UP
+makeDocTypeRepo(void)
+{
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name).
+                     addField("ia", DataType::T_STRING).
+                     addField("ib", DataType::T_STRING).
+                     addField("ibs", Struct("pair").
+                              addField("x", DataType::T_STRING).
+                              addField("y", DataType::T_STRING)).
+                     addField("iba", Array(DataType::T_STRING)).
+                     addField("ibw", Wset(DataType::T_STRING)).
+                     addField("ibm", Map(DataType::T_STRING,
+                                         DataType::T_STRING)).
+                     addField("aa", DataType::T_INT).
+                     addField("aaa", Array(DataType::T_INT)).
+                     addField("aaw", Wset(DataType::T_INT)).
+                     addField("ab", DataType::T_INT));
+    builder.document(doc_type_id + 1, type_name_2,
+                     Struct(header_name_2), Struct(body_name_2).
+                     addField("ic", DataType::T_STRING).
+                     addField("id", DataType::T_STRING).
+                     addField("ac", DataType::T_INT).
+                     addField("ad", DataType::T_INT));
+    return DocumentTypeRepo::UP(new DocumentTypeRepo(builder.config()));
+}
+
+
+std::string
+rsString(const ResultSet &s)
+{
+    std::ostringstream os;
+    bool first = true;
+    uint32_t erange = Result::enumRange();
+    for (uint32_t e = 0; e < erange; ++e) {
+        if (s.hasEnum(e)) {
+            if (!first)
+                os << ",";
+            first = false;
+            Result::fromEnum(e).print(os, false, "");
+        }
+    }
+    if (first) {
+        os << "empty";
+    }
+    return os.str();
+}
+
+
+const char *
+csString(const SelectPruner &pruner)
+{
+    if (!pruner.isConst())
+        return "not const";
+    if (pruner.isFalse())
+        return "const false";
+    if (pruner.isTrue())
+        return "const true";
+    if (pruner.isInvalid())
+        return "const invalid";
+    return "const something";
+}
+
+
+class TestFixture
+{
+public:
+    Schema _s;
+    DocumentTypeRepo::UP _repoUP;
+    bool _hasFields;
+
+    TestFixture(void);
+
+    ~TestFixture(void);
+
+    void
+    testParse(const string &selection);
+
+    void
+    testParseFail(const string &selection);
+
+    void
+    testPrune(const string &selection,
+              const string &exp);
+
+    void
+    testPrune(const string &selection,
+              const string &exp,
+              const string &docTypeName);
+};
+
+
+TestFixture::TestFixture(void)
+    : _s(),
+      _repoUP(),
+      _hasFields(true)
+{
+    makeSchema(_s);
+    _repoUP = makeDocTypeRepo();
+}
+
+
+TestFixture::~TestFixture(void)
+{
+}
+
+
+void
+TestFixture::testParse(const string &selection)
+{
+    const DocumentTypeRepo &repo(*_repoUP);
+    document::select::Parser parser(repo,
+                                    document::BucketIdFactory());
+
+    NodeUP select;
+
+    try {
+        LOG(info,
+            "Trying to parse '%s'",
+            selection.c_str());
+        select = parser.parse(selection);
+    } catch (document::select::ParsingFailedException &e) {
+        LOG(info,
+            "Parse failed: %s", e.what());
+        select.reset(0);
+    }
+    ASSERT_TRUE(select.get() != NULL);
+}
+
+
+void
+TestFixture::testParseFail(const string &selection)
+{
+    const DocumentTypeRepo &repo(*_repoUP);
+    document::select::Parser parser(repo,
+                                    document::BucketIdFactory());
+
+    NodeUP select;
+
+    try {
+        LOG(info,
+            "Trying to parse '%s'",
+            selection.c_str());
+        select = parser.parse(selection);
+    } catch (document::select::ParsingFailedException &e) {
+        LOG(info,
+            "Parse failed: %s",
+            e.getMessage().c_str());
+        select.reset(0);
+    }
+    ASSERT_TRUE(select.get() == NULL);
+}
+
+
+void
+TestFixture::testPrune(const string &selection,
+                       const string &exp,
+                       const string &docTypeName)
+{
+    const DocumentTypeRepo &repo(*_repoUP);
+    const Schema &schema(_s);
+    document::select::Parser parser(repo,
+                                    document::BucketIdFactory());
+
+    NodeUP select;
+
+    try {
+        LOG(info,
+            "Trying to parse '%s' with docType=%s",
+            selection.c_str(),
+            docTypeName.c_str());
+        select = parser.parse(selection);
+    } catch (document::select::ParsingFailedException &e) {
+        LOG(info,
+            "Parse failed: %s", e.what());
+        select.reset(0);
+    }
+    ASSERT_TRUE(select.get() != NULL);
+    std::ostringstream os;
+    select->print(os, true, "");
+    LOG(info, "ParseTree: '%s'", os.str().c_str());
+    const DocumentType *docType = repo.getDocumentType(docTypeName);
+    ASSERT_TRUE(docType != NULL);
+    Document::UP emptyDoc(new Document(*docType, docId));
+    emptyDoc->setRepo(repo);
+    SelectPruner pruner(docTypeName, schema, *emptyDoc, repo, _hasFields);
+    pruner.process(*select);
+    std::ostringstream pos;
+    pruner.getNode()->print(pos, true, "");
+    EXPECT_EQUAL(exp, pos.str());
+    LOG(info,
+        "Pruned ParseTree: '%s', fieldNodes=%u,%u, %s, rs=%s",
+        pos.str().c_str(),
+        pruner.getFieldNodes(),
+        pruner.getAttrFieldNodes(),
+        csString(pruner),
+        rsString(pruner.getResultSet()).c_str());
+    if (pruner.isConst()) {
+        ResultSet t;
+        if (pruner.isFalse())
+            t.add(Result::False);
+        if (pruner.isTrue())
+            t.add(Result::True);
+        if (pruner.isInvalid())
+            t.add(Result::Invalid);
+        ASSERT_TRUE(t == pruner.getResultSet());
+    }
+    CloningVisitor cv;
+    pruner.getNode()->visit(cv);
+    std::ostringstream cvpos;
+    cv.getNode()->print(cvpos, true, "");
+    EXPECT_EQUAL(exp, cvpos.str());
+#if 0
+    std::ostringstream os2;
+    pruner.trace(os2);
+    LOG(info, "trace pruned: %s", os2.str().c_str());
+#endif
+}
+
+
+void
+TestFixture::testPrune(const string &selection,
+                       const string &exp)
+{
+    testPrune(selection, exp, "test");
+}
+
+
+TEST_F("Test that test setup is OK", TestFixture)
+{
+    DocumentTypeRepo &repo = *f._repoUP;
+    const DocumentType *docType = repo.getDocumentType("test");
+    ASSERT_TRUE(docType);
+    EXPECT_EQUAL(10u, docType->getFieldCount());
+    EXPECT_EQUAL("String", docType->getField("ia").getDataType().getName());
+    EXPECT_EQUAL("String", docType->getField("ib").getDataType().getName());
+    EXPECT_EQUAL("Int", docType->getField("aa").getDataType().getName());
+    EXPECT_EQUAL("Int", docType->getField("ab").getDataType().getName());
+}
+
+
+TEST_F("Test that simple parsing works", TestFixture)
+{
+    f.testParse("not ((test))");
+    f.testParse("not ((test and (test.aa > 3999)))");
+    f.testParse("not ((test and (test.ab > 3999)))");
+    f.testParse("not ((test and (test.af > 3999)))");
+    f.testParse("not ((test_2 and (test_2.af > 3999)))");
+}
+
+
+TEST_F("Test that wrong doctype causes parse error", TestFixture)
+{
+    f.testParseFail("not ((test_3 and (test_3.af > 3999)))");
+}
+
+
+TEST_F("Test that boolean const shortcuts are OK", TestFixture)
+{
+    f.testPrune("false and false",
+                "false");
+    f.testPrune(false_name + " and " + invalid2_name,
+                "false");
+    f.testPrune(false_name + " and " + valid2_name,
+                "false");
+    f.testPrune("false and true",
+                "false");
+
+    f.testPrune(invalid_name + " and false",
+                "false");
+    f.testPrune(invalid_name + " and " + invalid2_name,
+                "invalid");
+    f.testPrune(invalid_name + " and " + valid2_name,
+                empty + "invalid and " + valid2_name);
+    f.testPrune(invalid_name + " and true",
+                "invalid");
+
+    f.testPrune(valid_name + " and false",
+                "false");
+    f.testPrune(valid_name + " and " + invalid2_name,
+                empty + valid_name + " and invalid");
+    f.testPrune(valid_name + " and " + valid2_name,
+                valid_name + " and " + valid2_name);
+    f.testPrune(valid_name + " and true",
+                valid_name);
+
+    f.testPrune("true and false",
+                "false");
+    f.testPrune(true_name + " and " + invalid2_name,
+                "invalid");
+    f.testPrune(true_name + " and " + valid2_name,
+                valid2_name);
+    f.testPrune("true and true",
+                "true");
+
+    f.testPrune("false or false",
+                "false");
+    f.testPrune(false_name + " or " + invalid2_name,
+                "invalid");
+    f.testPrune(false_name + " or " + valid2_name,
+                valid2_name);
+    f.testPrune("false or true",
+                "true");
+
+    f.testPrune(invalid_name + " or false",
+                "invalid");
+    f.testPrune(invalid_name + " or " + invalid2_name,
+                "invalid");
+    f.testPrune(invalid_name + " or " + valid2_name,
+                empty + "invalid or " + valid2_name);
+    f.testPrune(invalid_name + " or true",
+                "true");
+
+    f.testPrune(valid_name + " or false",
+                valid_name);
+    f.testPrune(valid_name + " or " + invalid2_name,
+                valid_name + " or invalid");
+    f.testPrune(valid_name + " or " + valid2_name,
+                valid_name + " or " + valid2_name);
+    f.testPrune(valid_name + " or true",
+                "true");
+
+    f.testPrune("true or false",
+                "true");
+    f.testPrune(true_name + " or " + invalid2_name,
+                "true");
+    f.testPrune(true_name + " or " + valid2_name,
+                "true");
+    f.testPrune("true or true",
+                "true");
+}
+
+
+TEST_F("Test that selection expressions are pruned", TestFixture)
+{
+    f.testPrune("not ((test))",
+                "false");
+    f.testPrune("not ((test and (test.aa > 3999)))",
+                "test.aa <= 3999");
+    f.testPrune("not ((test and (test.ab > 3999)))",
+                     "test.ab <= 3999");
+    f.testPrune("not ((test and (test.af > 3999)))",
+                "invalid");
+    f.testPrune("not ((test and (test_2.ac > 3999)))",
+                "invalid");
+    f.testPrune("not ((test and (test.af > 3999)))",
+                "true",
+                "test_2");
+    const char *combined =
+        "not ((test and (test.aa > 3999)) or (test_2 and (test_2.ac > 4999)))";
+    f.testPrune(combined,
+                "test.aa <= 3999");
+    f.testPrune(combined,
+                "test_2.ac <= 4999",
+                "test_2");
+}
+
+
+TEST_F("Test that De Morgan's laws are applied", TestFixture)
+{
+    f.testPrune("not (test.aa < 3901 and test.ab < 3902)",
+                "test.aa >= 3901 or test.ab >= 3902");
+    f.testPrune("not (test.aa < 3903 or test.ab < 3904)",
+                "test.aa >= 3903 and test.ab >= 3904");
+    f.testPrune("not (not (test.aa < 3903 or test.ab < 3904))",
+                "test.aa < 3903 or test.ab < 3904");
+
+    f.testPrune("not (false and false)",
+                "true");
+    f.testPrune(empty + "not (false and " + invalid2_name + ")",
+                "true");
+    f.testPrune(empty + "not (false and " + valid2_name + ")",
+                "true");
+    f.testPrune("not (false and true)",
+                "true");
+
+    f.testPrune(empty + "not (" + invalid_name + " and false)",
+                "true");
+    f.testPrune(empty + "not (" + invalid_name + " and " + invalid2_name + ")",
+                "invalid");
+    f.testPrune(empty + "not (" + invalid_name + " and " + valid2_name + ")",
+                empty + "invalid or " + rvalid2_name);
+    f.testPrune(empty + "not (" + invalid_name + " and true)",
+                "invalid");
+
+    f.testPrune(empty + "not (" + valid_name + " and false)",
+                "true");
+    f.testPrune(empty + "not (" + valid_name + " and " + invalid2_name + ")",
+                empty + rvalid_name + " or invalid");
+    f.testPrune(empty + "not (" + valid_name + " and " + valid2_name + ")",
+                rvalid_name + " or " + rvalid2_name);
+    f.testPrune(empty + "not (" + valid_name + " and true)",
+                rvalid_name);
+
+    f.testPrune("not (true and false)",
+                "true");
+    f.testPrune(empty + "not (true and " + invalid2_name + ")",
+                "invalid");
+    f.testPrune(empty + "not (true and " + valid2_name + ")",
+                rvalid2_name);
+    f.testPrune("not (true and true)",
+                "false");
+
+    f.testPrune("not (false or false)",
+                "true");
+    f.testPrune(empty + "not (false or " + invalid2_name + ")",
+                "invalid");
+    f.testPrune(empty + "not (false or " + valid2_name + ")",
+                rvalid2_name);
+    f.testPrune("not (false or true)",
+                "false");
+
+    f.testPrune(empty + "not (" + invalid_name + " or false)",
+                "invalid");
+    f.testPrune(empty + "not (" + invalid_name + " or " + invalid2_name + ")",
+                "invalid");
+    f.testPrune(empty + "not (" + invalid_name + " or " + valid2_name + ")",
+                empty + "invalid and " + rvalid2_name);
+    f.testPrune(empty + "not (" + invalid_name + " or true)",
+                "false");
+
+    f.testPrune(empty + "not (" + valid_name + " or false)",
+                rvalid_name);
+    f.testPrune(empty + "not (" + valid_name + " or " + invalid2_name + ")",
+                rvalid_name + " and invalid");
+    f.testPrune(empty + "not (" + valid_name + " or " + valid2_name + ")",
+                rvalid_name + " and " + rvalid2_name);
+    f.testPrune(empty + "not (" + valid_name + " or true)",
+                "false");
+
+    f.testPrune("not (true or false)",
+                "false");
+    f.testPrune(empty + "not (true or " + invalid2_name + ")",
+                "false");
+    f.testPrune(empty + "not (true or " + valid2_name + ")",
+                "false");
+    f.testPrune("not (true or true)",
+                "false");
+
+}
+
+
+TEST_F("Test that attribute fields and constants are evaluated"
+       " before other fields",
+       TestFixture)
+{
+    f.testPrune("test.ia == \"hello\" and test.aa > 5",
+                "test.aa > 5 and test.ia == \"hello\"");
+}
+
+
+TEST_F("Test that functions are visited", TestFixture)
+{
+    f.testPrune("test.ia.lowercase() == \"hello\"",
+                "test.ia.lowercase() == \"hello\"");
+    f.testPrune("test_2.ac.lowercase() == \"hello\"",
+                "invalid");
+    f.testPrune("test.ia.hash() == 45",
+                "test.ia.hash() == 45");
+    f.testPrune("test_2.ic.hash() == 45",
+                "invalid");
+    f.testPrune("test.aa.abs() == 45",
+                "test.aa.abs() == 45");
+    f.testPrune("test_2.ac.abs() == 45",
+                "invalid");
+}
+
+
+TEST_F("Test that arithmethic values are visited", TestFixture)
+{
+    f.testPrune("test.aa + 4 < 3999",
+                "test.aa + 4 < 3999");
+    f.testPrune("test_2.ac + 4 < 3999",
+                "invalid");
+    f.testPrune("test.aa + 4.2 < 3999",
+                "test.aa + 4.2 < 3999");
+    f.testPrune("test_2.ac + 5.2 < 3999",
+                "invalid");
+}
+
+
+TEST_F("Test that addition is associative", TestFixture)
+{
+    f.testPrune("test.aa + 4 + 5 < 3999",
+                "test.aa + 4 + 5 < 3999");
+    f.testPrune("(test.aa + 6) + 7 < 3999",
+                "test.aa + 6 + 7 < 3999");
+    f.testPrune("test.aa + (8 + 9) < 3999",
+                "test.aa + 8 + 9 < 3999");
+}
+
+
+TEST_F("Test that subtraction is left associative", TestFixture)
+{
+    f.testPrune("test.aa - 4 - 5 < 3999",
+                "test.aa - 4 - 5 < 3999");
+    f.testPrune("(test.aa - 6) - 7 < 3999",
+                "test.aa - 6 - 7 < 3999");
+    f.testPrune("test.aa - (8 - 9) < 3999",
+                "test.aa - (8 - 9) < 3999");
+}
+
+
+TEST_F("Test that multiplication is associative", TestFixture)
+{
+    f.testPrune("test.aa * 4 * 5 < 3999",
+                "test.aa * 4 * 5 < 3999");
+    f.testPrune("(test.aa * 6) * 7 < 3999",
+                "test.aa * 6 * 7 < 3999");
+    f.testPrune("test.aa * (8 * 9) < 3999",
+                "test.aa * 8 * 9 < 3999");
+}
+
+
+TEST_F("Test that division is left associative", TestFixture)
+{
+    f.testPrune("test.aa / 4 / 5 < 3999",
+                "test.aa / 4 / 5 < 3999");
+    f.testPrune("(test.aa / 6) / 7 < 3999",
+                "test.aa / 6 / 7 < 3999");
+    f.testPrune("test.aa / (8 / 9) < 3999",
+                "test.aa / (8 / 9) < 3999");
+}
+
+
+TEST_F("Test that mod is left associative", TestFixture)
+{
+    f.testPrune("test.aa % 4 % 5 < 3999",
+                "test.aa % 4 % 5 < 3999");
+    f.testPrune("(test.aa % 6) % 7 < 3999",
+                "test.aa % 6 % 7 < 3999");
+    f.testPrune("test.aa % (8 % 9) < 3999",
+                "test.aa % (8 % 9) < 3999");
+}
+
+
+TEST_F("Test that multiplication has higher priority than addition",
+       TestFixture)
+{
+    f.testPrune("test.aa + 4 * 5 < 3999",
+                "test.aa + 4 * 5 < 3999");
+    f.testPrune("(test.aa + 6) * 7 < 3999",
+                "(test.aa + 6) * 7 < 3999");
+    f.testPrune("test.aa + (8 * 9) < 3999",
+                "test.aa + 8 * 9 < 3999");
+    f.testPrune("test.aa * 4 + 5 < 3999",
+                "test.aa * 4 + 5 < 3999");
+    f.testPrune("(test.aa * 6) + 7 < 3999",
+                "test.aa * 6 + 7 < 3999");
+    f.testPrune("test.aa * (8 + 9) < 3999",
+                "test.aa * (8 + 9) < 3999");
+}
+
+
+TEST_F("Test that toplevel functions are visited", TestFixture)
+{
+    f.testPrune("searchcolumn.15 == 4",
+                "searchcolumn.15 == 4");
+    f.testPrune("id.scheme == \"doc\"",
+                "id.scheme == \"doc\"");
+    f.testPrune("test.aa < now() - 7200",
+                "test.aa < now() - 7200");
+}
+
+
+TEST_F("Test that variables are visited", TestFixture)
+{
+    f.testPrune("$foovar == 4.3",
+                "$foovar == 4.3");
+}
+
+
+TEST_F("Test that null is visited", TestFixture)
+{
+    f.testPrune("test.aa",
+                "test.aa != null");
+    f.testPrune("test.aa == null",
+                "test.aa == null");
+    f.testPrune("not test.aa",
+                "test.aa == null");
+}
+
+
+TEST_F("Test that operator inversion works", TestFixture)
+{
+    f.testPrune("not test.aa < 3999",
+                "test.aa >= 3999");
+    f.testPrune("not test.aa <= 3999",
+                "test.aa > 3999");
+    f.testPrune("not test.aa > 3999",
+                "test.aa <= 3999");
+    f.testPrune("not test.aa >= 3999",
+                "test.aa < 3999");
+    f.testPrune("not test.aa == 3999",
+                "test.aa != 3999");
+    f.testPrune("not test.aa != 3999",
+                "test.aa == 3999");
+}
+
+
+TEST_F("Test that fields are not present in removed sub db", TestFixture)
+{
+    f._hasFields = true;
+    f.testPrune("test.aa > 5",
+                "test.aa > 5");
+    f.testPrune("test.aa == test.ab",
+                "test.aa == test.ab");
+    f.testPrune("test.aa != test.ab",
+                "test.aa != test.ab");
+    f.testPrune("not test.aa == test.ab",
+                "test.aa != test.ab");
+    f.testPrune("not test.aa != test.ab",
+                "test.aa == test.ab");
+    f.testPrune("test.ia == \"hello\"",
+                "test.ia == \"hello\"");
+    f._hasFields = false;
+    f.testPrune("test.aa > 5",
+                "invalid");
+    f.testPrune("test.aa == test.ab",
+                "true");
+    f.testPrune("test.aa != test.ab",
+                "false");
+    f.testPrune("test.aa < test.ab",
+                "invalid");
+    f.testPrune("test.aa > test.ab",
+                "invalid");
+    f.testPrune("test.aa <= test.ab",
+                "invalid");
+    f.testPrune("test.aa >= test.ab",
+                "invalid");
+    f.testPrune("not test.aa == test.ab",
+                "false");
+    f.testPrune("not test.aa != test.ab",
+                "true");
+    f.testPrune("test.ia == \"hello\"",
+                "invalid");
+    f.testPrune("not test.aa < test.ab",
+                "invalid");
+    f.testPrune("not test.aa > test.ab",
+                "invalid");
+    f.testPrune("not test.aa <= test.ab",
+                "invalid");
+    f.testPrune("not test.aa >= test.ab",
+                "invalid");
+}
+
+
+TEST_F("Test that some operators cannot be inverted", TestFixture)
+{
+    f.testPrune("test.ia == \"hello\"",
+                "test.ia == \"hello\"");
+    f.testPrune("not test.ia == \"hello\"",
+                "test.ia != \"hello\"");
+    f.testPrune("test.ia = \"hello\"",
+                "test.ia = \"hello\"");
+    f.testPrune("not test.ia = \"hello\"",
+                "not test.ia = \"hello\"");
+    f.testPrune("not (test.ia == \"hello\" or test.ia == \"hi\")",
+                "test.ia != \"hello\" and test.ia != \"hi\"");
+    f.testPrune("not (test.ia == \"hello\" or test.ia = \"hi\")",
+                "not (not test.ia != \"hello\" or test.ia = \"hi\")");
+    f.testPrune("not (test.ia = \"hello\" or test.ia == \"hi\")",
+                "not (test.ia = \"hello\" or not test.ia != \"hi\")");
+    f.testPrune("not (test.ia = \"hello\" or test.ia = \"hi\")",
+                "not (test.ia = \"hello\" or test.ia = \"hi\")");
+}
+
+
+TEST_F("Test that complex field refs are handled", TestFixture)
+{
+    f.testPrune("test.ia",
+                "test.ia != null");
+    f.testPrune("test.ia == \"hello\"",
+                "test.ia == \"hello\"");
+    f.testPrune("test.ia.foo == \"hello\"",
+                "invalid");
+    f.testPrune("test.ibs.foo == \"hello\"",
+                "invalid");
+    f.testPrune("test.ibs.x == \"hello\"",
+                "test.ibs.x == \"hello\"");
+    f.testPrune("test.ia[2] == \"hello\"",
+                "invalid");
+    f.testPrune("test.iba[2] == \"hello\"",
+                "test.iba[2] == \"hello\"");
+    f.testPrune("test.ia{foo} == \"hello\"",
+                "invalid");
+    f.testPrune("test.ibw{foo} == 4",
+                "test.ibw{foo} == 4");
+    f.testPrune("test.ibw{foo} == \"hello\"",
+                "test.ibw{foo} == \"hello\"");
+    f.testPrune("test.ibm{foo} == \"hello\"",
+                "test.ibm{foo} == \"hello\"");
+    f.testPrune("test.aa == 4",
+                "test.aa == 4");
+    f.testPrune("test.aa[4] == 4",
+                "invalid");
+    f.testPrune("test.aaa[4] == 4",
+                "test.aaa[4] == 4");
+    f.testPrune("test.aa{4} == 4",
+                "invalid");
+    f.testPrune("test.aaw{4} == 4",
+                "test.aaw{4} == 4");
+}
+
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore b/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore
new file mode 100644
index 00000000000..bb0963e5ec3
--- /dev/null
+++ b/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore
@@ -0,0 +1 @@
+searchcore_state_reporter_utils_test_app
diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt b/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt
new file mode 100644
index 00000000000..9b1ffd8aef2
--- /dev/null
+++ b/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_state_reporter_utils_test_app
+    SOURCES
+    state_reporter_utils_test.cpp
+    DEPENDS
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_state_reporter_utils_test_app COMMAND searchcore_state_reporter_utils_test_app)
diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/DESC b/searchcore/src/tests/proton/common/state_reporter_utils/DESC
new file mode 100644
index 00000000000..7e4bc287738
--- /dev/null
+++ b/searchcore/src/tests/proton/common/state_reporter_utils/DESC
@@ -0,0 +1 @@
+state reporter utils test. Take a look at state_reporter_utils_test.cpp for details.
diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/FILES b/searchcore/src/tests/proton/common/state_reporter_utils/FILES
new file mode 100644
index 00000000000..cbea1ab5ad2
--- /dev/null
+++ b/searchcore/src/tests/proton/common/state_reporter_utils/FILES
@@ -0,0 +1 @@
+state_reporter_utils_test.cpp
diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp b/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp
new file mode 100644
index 00000000000..9b04a126a48
--- /dev/null
+++ b/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("state_reporter_utils_test");
+
+#include <vespa/searchcore/proton/common/state_reporter_utils.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace proton;
+using namespace vespalib::slime;
+using vespalib::Slime;
+
+vespalib::string
+toString(const StatusReport &statusReport)
+{
+    Slime slime;
+    StateReporterUtils::convertToSlime(statusReport, SlimeInserter(slime));
+    return slime.toString();
+}
+
+TEST("require that simple status report is correctly converted to slime")
+{
+    EXPECT_EQUAL(
+            "{\n"
+            "    \"state\": \"ONLINE\"\n"
+            "}\n",
+            toString(StatusReport(StatusReport::Params("").
+                    internalState("ONLINE"))));
+}
+
+TEST("require that advanced status report is correctly converted to slime")
+{
+    EXPECT_EQUAL(
+            "{\n"
+            "    \"state\": \"REPLAY\",\n"
+            "    \"progress\": 65.5,\n"
+            "    \"configState\": \"OK\",\n"
+            "    \"message\": \"foo\"\n"
+            "}\n",
+            toString(StatusReport(StatusReport::Params("").
+                    internalState("REPLAY").
+                    progress(65.5).
+                    internalConfigState("OK").
+                    message("foo"))));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/config/.cvsignore b/searchcore/src/tests/proton/config/.cvsignore
new file mode 100644
index 00000000000..13fb04d2a35
--- /dev/null
+++ b/searchcore/src/tests/proton/config/.cvsignore
@@ -0,0 +1,3 @@
+.depend
+Makefile
+config_test
diff --git a/searchcore/src/tests/proton/config/.gitignore b/searchcore/src/tests/proton/config/.gitignore
new file mode 100644
index 00000000000..72c49479fc1
--- /dev/null
+++ b/searchcore/src/tests/proton/config/.gitignore
@@ -0,0 +1 @@
+searchcore_config_test_app
diff --git a/searchcore/src/tests/proton/config/CMakeLists.txt b/searchcore/src/tests/proton/config/CMakeLists.txt
new file mode 100644
index 00000000000..803556c2ae1
--- /dev/null
+++ b/searchcore/src/tests/proton/config/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_config_test_app
+    SOURCES
+    config.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_config_test_app COMMAND searchcore_config_test_app)
diff --git a/searchcore/src/tests/proton/config/config.cpp b/searchcore/src/tests/proton/config/config.cpp
new file mode 100644
index 00000000000..24cf4cec4cc
--- /dev/null
+++ b/searchcore/src/tests/proton/config/config.cpp
@@ -0,0 +1,268 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("config_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/linkedptr.h>
+#include <map>
+#include <vespa/searchcore/proton/server/bootstrapconfigmanager.h>
+#include <vespa/searchcore/proton/server/bootstrapconfig.h>
+#include <vespa/searchcore/proton/server/documentdbconfigmanager.h>
+#include <vespa/searchcore/proton/server/protonconfigurer.h>
+#include <vespa/vespalib/util/varholder.h>
+
+using namespace config;
+using namespace proton;
+using namespace vespa::config::search::core;
+using namespace vespa::config::search::summary;
+using namespace vespa::config::search;
+
+using config::ConfigUri;
+using document::DocumentTypeRepo;
+using document::DocumenttypesConfig;
+using document::DocumenttypesConfigBuilder;
+using search::TuneFileDocumentDB;
+using std::map;
+using vespalib::LinkedPtr;
+using vespalib::VarHolder;
+
+struct DoctypeFixture {
+    typedef vespalib::LinkedPtr<DoctypeFixture> LP;
+    AttributesConfigBuilder attributesBuilder;
+    RankProfilesConfigBuilder rankProfilesBuilder;
+    IndexschemaConfigBuilder indexschemaBuilder;
+    SummaryConfigBuilder summaryBuilder;
+    SummarymapConfigBuilder summarymapBuilder;
+    JuniperrcConfigBuilder juniperrcBuilder;
+};
+
+struct ConfigTestFixture {
+    const std::string configId;
+    ProtonConfigBuilder protonBuilder;
+    DocumenttypesConfigBuilder documenttypesBuilder;
+    map<std::string, DoctypeFixture::LP> dbConfig;
+    ConfigSet set;
+    IConfigContext::SP context;
+    int idcounter;
+
+    ConfigTestFixture(const std::string & id)
+        : configId(id),
+          protonBuilder(),
+          documenttypesBuilder(),
+          dbConfig(),
+          set(),
+          context(new ConfigContext(set)),
+          idcounter(-1)
+    {
+        set.addBuilder(configId, &protonBuilder);
+        set.addBuilder(configId, &documenttypesBuilder);
+        addDocType("_alwaysthere_");
+    }
+
+    void addDocType(const std::string & name)
+    {
+        DocumenttypesConfigBuilder::Documenttype dt;
+        dt.bodystruct = -1270491200;
+        dt.headerstruct = 306916075;
+        dt.id = idcounter--;
+        dt.name = name;
+        dt.version = 0;
+        documenttypesBuilder.documenttype.push_back(dt);
+
+        ProtonConfigBuilder::Documentdb db;
+        db.inputdoctypename = name;
+        db.configid = configId + "/" + name;
+        protonBuilder.documentdb.push_back(db);
+
+        DoctypeFixture::LP fixture(new DoctypeFixture());
+        set.addBuilder(db.configid, &fixture->attributesBuilder);
+        set.addBuilder(db.configid, &fixture->rankProfilesBuilder);
+        set.addBuilder(db.configid, &fixture->indexschemaBuilder);
+        set.addBuilder(db.configid, &fixture->summaryBuilder);
+        set.addBuilder(db.configid, &fixture->summarymapBuilder);
+        set.addBuilder(db.configid, &fixture->juniperrcBuilder);
+        dbConfig[name] = fixture;
+    }
+
+    void removeDocType(const std::string & name)
+    {
+        for (DocumenttypesConfigBuilder::DocumenttypeVector::iterator it(documenttypesBuilder.documenttype.begin()),
+                                                                      mt(documenttypesBuilder.documenttype.end());
+             it != mt;
+             it++) {
+            if ((*it).name.compare(name) == 0) {
+                documenttypesBuilder.documenttype.erase(it);
+                break;
+            }
+        }
+
+        for (ProtonConfigBuilder::DocumentdbVector::iterator it(protonBuilder.documentdb.begin()),
+                                                             mt(protonBuilder.documentdb.end());
+             it != mt;
+             it++) {
+            if ((*it).inputdoctypename.compare(name) == 0) {
+                protonBuilder.documentdb.erase(it);
+                break;
+            }
+        }
+    }
+
+    bool configEqual(const std::string & name, DocumentDBConfig::SP dbc) {
+        DoctypeFixture::LP fixture(dbConfig[name]);
+        return (fixture->attributesBuilder == dbc->getAttributesConfig() &&
+                fixture->rankProfilesBuilder == dbc->getRankProfilesConfig() &&
+                fixture->indexschemaBuilder == dbc->getIndexschemaConfig() &&
+                fixture->summaryBuilder == dbc->getSummaryConfig() &&
+                fixture->summarymapBuilder == dbc->getSummarymapConfig() &&
+                fixture->juniperrcBuilder == dbc->getJuniperrcConfig());
+    }
+
+    bool configEqual(BootstrapConfig::SP bootstrapConfig) {
+        return (protonBuilder == bootstrapConfig->getProtonConfig() &&
+                documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig());
+    }
+
+    BootstrapConfig::SP getBootstrapConfig(int64_t generation) const {
+        return BootstrapConfig::SP(new BootstrapConfig(generation,
+                                                       BootstrapConfig::DocumenttypesConfigSP(new DocumenttypesConfig(documenttypesBuilder)),
+                                                       DocumentTypeRepo::SP(new DocumentTypeRepo(documenttypesBuilder)),
+                                                       BootstrapConfig::ProtonConfigSP(new ProtonConfig(protonBuilder)),
+                                                       TuneFileDocumentDB::SP(new TuneFileDocumentDB())));
+    }
+
+    void reload() { context->reload(); }
+};
+
+template <typename ConfigType, typename ConfigOwner>
+struct OwnerFixture : public ConfigOwner
+{
+    volatile bool configured;
+    VarHolder<ConfigType> config;
+
+    OwnerFixture() : configured(false), config() { }
+    bool waitUntilConfigured(int timeout) {
+        FastOS_Time timer;
+        timer.SetNow();
+        while (timer.MilliSecsToNow() < timeout) {
+            if (configured)
+                break;
+            FastOS_Thread::Sleep(100);
+        }
+        return configured;
+    }
+    void reconfigure(const ConfigType & cfg) {
+        assert(cfg->valid());
+        config.set(cfg);
+        configured = true;
+    }
+    bool addExtraConfigs(DocumentDBConfigManager & dbCfgMan) {
+        (void) dbCfgMan;
+        return false;
+    }
+};
+
+typedef OwnerFixture<BootstrapConfig::SP, IBootstrapOwner> BootstrapOwner;
+typedef OwnerFixture<DocumentDBConfig::SP, IDocumentDBConfigOwner> DBOwner;
+
+TEST_F("require that bootstrap config manager creats correct key set", BootstrapConfigManager("foo")) {
+    const ConfigKeySet set(f1.createConfigKeySet());
+    ASSERT_EQUAL(2u, set.size());
+    ConfigKey protonKey(ConfigKey::create<ProtonConfig>("foo"));
+    ConfigKey dtKey(ConfigKey::create<DocumenttypesConfig>("foo"));
+    ASSERT_TRUE(set.find(protonKey) != set.end());
+    ASSERT_TRUE(set.find(dtKey) != set.end());
+}
+
+TEST_FFF("require_that_bootstrap_config_manager_updates_config", ConfigTestFixture("search"),
+                                                                 BootstrapConfigManager(f1.configId),
+                                                                 ConfigRetriever(f2.createConfigKeySet(), f1.context)) {
+    f2.update(f3.getBootstrapConfigs());
+    ASSERT_TRUE(f1.configEqual(f2.getConfig()));
+    f1.protonBuilder.rpcport = 9010;
+    ASSERT_FALSE(f1.configEqual(f2.getConfig()));
+    f1.reload();
+    f2.update(f3.getBootstrapConfigs());
+    ASSERT_TRUE(f1.configEqual(f2.getConfig()));
+
+    f1.addDocType("foobar");
+    ASSERT_FALSE(f1.configEqual(f2.getConfig()));
+    f1.reload();
+    f2.update(f3.getBootstrapConfigs());
+    ASSERT_TRUE(f1.configEqual(f2.getConfig()));
+}
+
+TEST_FF("require_that_documentdb_config_manager_subscribes_for_config",
+        ConfigTestFixture("search"),
+        DocumentDBConfigManager(f1.configId + "/typea", "typea")) {
+    f1.addDocType("typea");
+    const ConfigKeySet keySet(f2.createConfigKeySet());
+    ASSERT_EQUAL(6u, keySet.size());
+    ConfigRetriever retriever(keySet, f1.context);
+    f2.forwardConfig(f1.getBootstrapConfig(1));
+    f2.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs
+    ASSERT_TRUE(f1.configEqual("typea", f2.getConfig()));
+}
+
+TEST_FFF("require_that_protonconfigurer_follows_changes_to_bootstrap",
+         ConfigTestFixture("search"),
+         BootstrapOwner(),
+         ProtonConfigurer(ConfigUri(f1.configId, f1.context), &f2, 60000)) {
+    f3.start();
+    ASSERT_TRUE(f2.configured);
+    ASSERT_TRUE(f1.configEqual(f2.config.get()));
+    f2.configured = false;
+    f1.protonBuilder.rpcport = 9010;
+    f1.reload();
+    ASSERT_TRUE(f2.waitUntilConfigured(120000));
+    ASSERT_TRUE(f1.configEqual(f2.config.get()));
+    f3.close();
+}
+
+TEST_FFF("require_that_protonconfigurer_follows_changes_to_doctypes",
+         ConfigTestFixture("search"),
+         BootstrapOwner(),
+         ProtonConfigurer(ConfigUri(f1.configId, f1.context), &f2, 60000)) {
+    f3.start();
+
+    f2.configured = false;
+    f1.addDocType("typea");
+    f1.reload();
+    ASSERT_TRUE(f2.waitUntilConfigured(60000));
+    ASSERT_TRUE(f1.configEqual(f2.config.get()));
+
+    f2.configured = false;
+    f1.removeDocType("typea");
+    f1.reload();
+    ASSERT_TRUE(f2.waitUntilConfigured(60000));
+    ASSERT_TRUE(f1.configEqual(f2.config.get()));
+    f3.close();
+}
+
+TEST_FFF("require_that_protonconfigurer_reconfigures_dbowners",
+         ConfigTestFixture("search"),
+         BootstrapOwner(),
+         ProtonConfigurer(ConfigUri(f1.configId, f1.context), &f2, 60000)) {
+    f3.start();
+
+    DBOwner dbA;
+    f3.registerDocumentDB(DocTypeName("typea"), &dbA);
+
+    // Add db and verify that we get an initial callback
+    f2.configured = false;
+    f1.addDocType("typea");
+    f1.reload();
+    ASSERT_TRUE(f2.waitUntilConfigured(60000));
+    ASSERT_TRUE(f1.configEqual(f2.config.get()));
+    ASSERT_TRUE(dbA.waitUntilConfigured(60000));
+    ASSERT_TRUE(f1.configEqual("typea", dbA.config.get()));
+
+    // Remove and verify that we don't get any callback
+    dbA.configured = false;
+    f1.removeDocType("typea");
+    f1.reload();
+    ASSERT_TRUE(f2.waitUntilConfigured(60000));
+    ASSERT_FALSE(dbA.waitUntilConfigured(60000));
+    f3.close();
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/create-test.sh b/searchcore/src/tests/proton/create-test.sh
new file mode 100755
index 00000000000..2e3b7fc4af9
--- /dev/null
+++ b/searchcore/src/tests/proton/create-test.sh
@@ -0,0 +1,73 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+gen_ignore_file() {
+    echo "generating '$1' ..."
+    echo ".depend"       > $1
+    echo "Makefile"     >> $1
+    echo "${test}_test" >> $1
+}
+
+gen_project_file() {
+    echo "generating '$1' ..."
+    echo "APPLICATION ${test}_test"               > $1
+    echo "OBJS $test"                            >> $1
+    echo "EXTERNALLIBS searchlib document fnet"  >> $1
+    echo "EXTERNALLIBS vespalib config vespalog" >> $1
+    echo ""                                      >> $1
+    echo "CUSTOMMAKE"                            >> $1
+    echo "test: ${test}_test"                    >> $1
+    echo -e "\t\$(VALGRIND) ./${test}_test"      >> $1
+}
+
+gen_source() {
+    echo "generating '$1' ..."
+    echo "#include <vespa/log/log.h>"                   > $1
+    echo "LOG_SETUP(\"${test}_test\");"          >> $1
+    echo "#include <vespa/fastos/fastos.h>"            >> $1
+    echo "#include <vespa/vespalib/testkit/testapp.h>" >> $1
+    echo ""                                      >> $1
+    echo "// using namespace ;"                  >> $1
+    echo ""                                      >> $1
+    echo "TEST_SETUP(Test);"                     >> $1
+    echo ""                                      >> $1
+    echo "int"                                   >> $1
+    echo "Test::Main()"                          >> $1
+    echo "{"                                     >> $1
+    echo "    TEST_INIT(\"${test}_test\");"      >> $1
+    echo "    TEST_DONE();"                      >> $1
+    echo "}"                                     >> $1
+}
+
+gen_desc() {
+    echo "generating '$1' ..."
+    echo "$test test. Take a look at $test.cpp for details." > $1
+}
+
+gen_file_list() {
+    echo "generating '$1' ..."
+    echo "$test.cpp" > $1
+}
+
+if [ $# -ne 1 ]; then
+	echo "usage: $0 <name>"
+	echo "  name: name of the test to create"
+	exit 1
+fi
+
+test=$1
+if [ -e $test ]; then
+    echo "$test already present, don't want to mess it up..."
+    exit 1
+fi
+
+echo "creating directory '$test' ..."
+mkdir -p $test || exit 1
+cd $test || exit 1
+test=`basename $test`
+
+gen_ignore_file  .cvsignore
+gen_project_file fastos.project
+gen_source       $test.cpp
+gen_desc         DESC
+gen_file_list    FILES
diff --git a/searchcore/src/tests/proton/docsummary/.gitignore b/searchcore/src/tests/proton/docsummary/.gitignore
new file mode 100644
index 00000000000..f5e934f84da
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/.gitignore
@@ -0,0 +1,6 @@
+.depend
+Makefile
+docsummary_test
+
+searchcore_docsummary_test_app
+searchcore_summaryfieldconverter_test_app
diff --git a/searchcore/src/tests/proton/docsummary/CMakeLists.txt b/searchcore/src/tests/proton/docsummary/CMakeLists.txt
new file mode 100644
index 00000000000..dca65528840
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/CMakeLists.txt
@@ -0,0 +1,32 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_docsummary_test_app
+    SOURCES
+    docsummary.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_initializer
+    searchcore_reprocessing
+    searchcore_index
+    searchcore_persistenceengine
+    searchcore_feedoperation
+    searchcore_docsummary
+    searchcore_matchengine
+    searchcore_summaryengine
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_flushengine
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_fconfig
+    searchcore_util
+)
+vespa_add_executable(searchcore_summaryfieldconverter_test_app
+    SOURCES
+    summaryfieldconverter_test.cpp
+    DEPENDS
+    searchcore_docsummary
+)
+vespa_add_test(NAME searchcore_docsummary_test_app COMMAND sh docsummary_test.sh)
diff --git a/searchcore/src/tests/proton/docsummary/DESC b/searchcore/src/tests/proton/docsummary/DESC
new file mode 100644
index 00000000000..ba16d5453b6
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/DESC
@@ -0,0 +1 @@
+docsummary test. Take a look at docsummary.cpp for details.
diff --git a/searchcore/src/tests/proton/docsummary/FILES b/searchcore/src/tests/proton/docsummary/FILES
new file mode 100644
index 00000000000..e63fca83f2e
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/FILES
@@ -0,0 +1 @@
+docsummary.cpp
diff --git a/searchcore/src/tests/proton/docsummary/attributes.cfg b/searchcore/src/tests/proton/docsummary/attributes.cfg
new file mode 100644
index 00000000000..3866731b410
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/attributes.cfg
@@ -0,0 +1,45 @@
+attribute[16]
+attribute[0].name "ba"
+attribute[0].datatype INT32
+attribute[1].name "bb"
+attribute[1].datatype FLOAT
+attribute[2].name "bc"
+attribute[2].datatype STRING
+attribute[3].name "bd"
+attribute[3].datatype INT32
+attribute[3].collectiontype ARRAY
+attribute[4].name "be"
+attribute[4].datatype FLOAT
+attribute[4].collectiontype ARRAY
+attribute[5].name "bf"
+attribute[5].datatype STRING
+attribute[5].collectiontype ARRAY
+attribute[6].name "bg"
+attribute[6].datatype INT32
+attribute[6].collectiontype WEIGHTEDSET
+attribute[7].name "bh"
+attribute[7].datatype FLOAT
+attribute[7].collectiontype WEIGHTEDSET
+attribute[8].name "bi"
+attribute[8].datatype STRING
+attribute[8].collectiontype WEIGHTEDSET
+attribute[9].name "sp1"
+attribute[9].datatype INT32
+attribute[10].name "sp2"
+attribute[10].datatype INT64
+attribute[11].name "ap1"
+attribute[11].datatype INT32
+attribute[11].collectiontype ARRAY
+attribute[12].name "ap2"
+attribute[12].datatype INT64
+attribute[12].collectiontype ARRAY
+attribute[13].name "wp1"
+attribute[13].datatype INT32
+attribute[13].collectiontype WEIGHTEDSET
+attribute[14].name "wp2"
+attribute[14].datatype INT64
+attribute[14].collectiontype WEIGHTEDSET
+attribute[15].name "bj"
+attribute[15].datatype TENSOR
+attribute[15].tensortype "tensor(x{},y{})"
+attribute[15].collectiontype SINGLE
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp
new file mode 100644
index 00000000000..80eaf56bcba
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp
@@ -0,0 +1,1296 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("docsummary_test");
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/docsummary/docsumcontext.h>
+#include <vespa/searchcore/proton/docsummary/documentstoreadapter.h>
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include <vespa/searchcore/proton/server/documentdb.h>
+#include <vespa/searchcore/proton/server/memoryconfigstore.h>
+#include <vespa/searchcore/proton/metrics/metricswireservice.h>
+#include <vespa/searchcore/proton/server/summaryadapter.h>
+#include <vespa/searchlib/common/idestructorcallback.h>
+#include <vespa/searchlib/docstore/logdocumentstore.h>
+#include <vespa/searchlib/engine/docsumapi.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/transactionlog/translogserver.h>
+#include <tests/proton/common/dummydbowner.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchlib/transactionlog/nosyncproxy.h>
+#include <vespa/vespalib/tensor/tensor_factory.h>
+#include <vespa/vespalib/tensor/default_tensor.h>
+#include <vespa/searchlib/attribute/tensorattribute.h>
+
+using namespace document;
+using namespace search;
+using namespace search::docsummary;
+using namespace search::engine;
+using namespace search::index;
+using namespace search::transactionlog;
+using search::TuneFileDocumentDB;
+using document::DocumenttypesConfig;
+using storage::spi::Timestamp;
+using search::index::DummyFileHeaderContext;
+using vespa::config::search::core::ProtonConfig;
+using vespalib::tensor::Tensor;
+using vespalib::tensor::TensorCells;
+using vespalib::tensor::TensorDimensions;
+using vespalib::tensor::TensorFactory;
+
+typedef std::unique_ptr<GeneralResult> GeneralResultPtr;
+
+namespace proton {
+
+class DirMaker
+{
+public:
+    DirMaker(const vespalib::string & dir) :
+        _dir(dir)
+    {
+        FastOS_File::MakeDirectory(dir.c_str());
+    }
+    ~DirMaker()
+    {
+        FastOS_File::EmptyAndRemoveDirectory(_dir.c_str());
+    }
+private:
+    vespalib::string _dir;
+};
+
+class BuildContext
+{
+public:
+    DirMaker _dmk;
+    DocBuilder _bld;
+    DocumentTypeRepo::SP _repo;
+    DummyFileHeaderContext _fileHeaderContext;
+    vespalib::ThreadStackExecutor _summaryExecutor;
+    search::transactionlog::NoSyncProxy _noTlSyncer;
+    search::LogDocumentStore _str;
+    uint64_t _serialNum;
+
+    BuildContext(const Schema &schema)
+        : _dmk("summary"),
+          _bld(schema),
+          _repo(new DocumentTypeRepo(_bld.getDocumentType())),
+          _summaryExecutor(4, 128 * 1024),
+          _noTlSyncer(),
+          _str(_summaryExecutor, "summary",
+               LogDocumentStore::Config(
+                       DocumentStore::Config(),
+                       LogDataStore::Config()),
+               GrowStrategy(),
+               TuneFileSummary(),
+               _fileHeaderContext,
+               _noTlSyncer,
+               NULL),
+          _serialNum(1)
+    {
+    }
+
+    ~BuildContext(void)
+    {
+    }
+
+    void
+    endDocument(uint32_t docId)
+    {
+        Document::UP doc = _bld.endDocument();
+        _str.write(_serialNum++, *doc, docId);
+    }
+
+    FieldCacheRepo::UP createFieldCacheRepo(const ResultConfig &resConfig) const {
+        return FieldCacheRepo::UP(new FieldCacheRepo(resConfig, _bld.getDocumentType()));
+    }
+};
+
+
+namespace {
+
+const char *
+getDocTypeName(void)
+{
+    return "searchdocument";
+}
+
+Tensor::UP createTensor(const TensorCells &cells,
+                        const TensorDimensions &dimensions) {
+    vespalib::tensor::DefaultTensor::builder builder;
+    return TensorFactory::create(cells, dimensions, builder);
+}
+
+}  // namespace
+
+
+class DBContext : public DummyDBOwner
+{
+public:
+    DirMaker _dmk;
+    DummyFileHeaderContext _fileHeaderContext;
+    TransLogServer _tls;
+    vespalib::ThreadStackExecutor _summaryExecutor;
+    bool _mkdirOk;
+    matching::QueryLimiter _queryLimiter;
+    vespalib::Clock _clock;
+    DummyWireService _dummy;
+    config::DirSpec _spec;
+    DocumentDBConfigHelper _configMgr;
+    DocumentDBConfig::DocumenttypesConfigSP _documenttypesConfig;
+    const DocumentTypeRepo::SP _repo;
+    TuneFileDocumentDB::SP _tuneFileDocumentDB;
+    std::unique_ptr<DocumentDB> _ddb;
+    AttributeWriter::UP _aw;
+    ISummaryAdapter::SP _sa;
+
+    DBContext(const DocumentTypeRepo::SP &repo, const char *docTypeName)
+        : _dmk(docTypeName),
+          _fileHeaderContext(),
+          _tls("tmp", 9013, ".", _fileHeaderContext),
+          _summaryExecutor(8, 128*1024),
+          _mkdirOk(FastOS_File::MakeDirectory("tmpdb")),
+          _queryLimiter(),
+          _clock(),
+          _dummy(),
+          _spec("."),
+          _configMgr(_spec, getDocTypeName()),
+          _documenttypesConfig(new DocumenttypesConfig()),
+          _repo(repo),
+          _tuneFileDocumentDB(new TuneFileDocumentDB()),
+          _ddb(),
+          _aw(),
+          _sa()
+    {
+        assert(_mkdirOk);
+        BootstrapConfig::SP b(new BootstrapConfig(1,
+                                                  _documenttypesConfig,
+                                                  _repo,
+                                                  BootstrapConfig::ProtonConfigSP(new ProtonConfig()),
+                                                  _tuneFileDocumentDB));
+        _configMgr.forwardConfig(b);
+        _configMgr.nextGeneration(0);
+        if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { abort(); }
+        _ddb.reset(new DocumentDB("tmpdb",
+                                  _configMgr.getConfig(),
+                                  "tcp/localhost:9013",
+                                  _queryLimiter,
+                                  _clock,
+                                  DocTypeName(docTypeName),
+                                  ProtonConfig(),
+                                  *this,
+                                  _summaryExecutor,
+                                  _summaryExecutor,
+                                  NULL,
+                                  _dummy,
+                                  _fileHeaderContext,
+                                  ConfigStore::UP(new MemoryConfigStore),
+                                  std::make_shared<vespalib::
+                                                   ThreadStackExecutor>
+                                  (16, 128 * 1024))),
+        _ddb->start();
+        _ddb->waitForOnlineState();
+        _aw = AttributeWriter::UP(new AttributeWriter(_ddb->
+                                            getReadySubDB()->
+                                            getAttributeManager()));
+        _sa = _ddb->getReadySubDB()->getSummaryAdapter();
+    }
+    ~DBContext()
+    {
+        _sa.reset();
+        _aw.reset();
+        _ddb.reset();
+        FastOS_File::EmptyAndRemoveDirectory("tmp");
+        FastOS_File::EmptyAndRemoveDirectory("tmpdb");
+    }
+
+    void
+    put(const document::Document &doc, const search::DocumentIdT lid)
+    {
+        const document::DocumentId &docId = doc.getId();
+        typedef DocumentMetaStore::Result PutRes;
+        IDocumentMetaStore &dms = _ddb->getReadySubDB()->getDocumentMetaStoreContext().get();
+        PutRes putRes(dms.put(docId.getGlobalId(),
+                              BucketFactory::getBucketId(docId),
+                              Timestamp(0u),
+                              lid));
+        LOG_ASSERT(putRes.ok());
+        uint64_t serialNum = _ddb->getFeedHandler().incSerialNum();
+        _aw->put(serialNum, doc, lid, true, std::shared_ptr<IDestructorCallback>());
+        _ddb->getReadySubDB()->
+            getAttributeManager()->getAttributeFieldWriter().sync();
+        _sa->put(serialNum, doc, lid);
+        const GlobalId &gid = docId.getGlobalId();
+        BucketId bucketId(gid.convertToBucketId());
+        bucketId.setUsedBits(8);
+        storage::spi::Timestamp ts(0);
+        DbDocumentId dbdId(lid);
+        DbDocumentId prevDbdId(0);
+        document::Document::SP xdoc(new document::Document(doc));
+        PutOperation op(bucketId,
+                        ts,
+                        xdoc,
+                        serialNum,
+                        dbdId,
+                        prevDbdId);
+        _ddb->getFeedHandler().storeOperation(op);
+        SearchView *sv(dynamic_cast<SearchView *>
+                       (_ddb->getReadySubDB()->getSearchView().get()));
+        if (sv != NULL) {
+            // cf. FeedView::putAttributes()
+            DocIdLimit &docIdLimit = sv->getDocIdLimit();
+            if (docIdLimit.get() <= lid)
+                docIdLimit.set(lid + 1);
+        }
+    }
+};
+
+class Test : public vespalib::TestApp
+{
+private:
+    std::unique_ptr<vespa::config::search::SummaryConfig> _summaryCfg;
+    ResultConfig          _resultCfg;
+    std::set<vespalib::string> _markupFields;
+
+    const vespa::config::search::SummaryConfig &
+    getSummaryConfig() const
+    {
+        return *_summaryCfg;
+    }
+
+    const ResultConfig &getResultConfig() const
+    {
+        return _resultCfg;
+    }
+
+    const std::set<vespalib::string> &
+    getMarkupFields(void) const
+    {
+        return _markupFields;
+    }
+
+    GeneralResultPtr
+    getResult(DocumentStoreAdapter & dsa, uint32_t docId);
+
+    GeneralResultPtr
+    getResult(const DocsumReply & reply, uint32_t id, uint32_t resultClassID);
+
+    bool
+    assertString(const std::string & exp,
+                 const std::string & fieldName,
+                 DocumentStoreAdapter &dsa,
+                 uint32_t id);
+
+    bool
+    assertString(const std::string &exp,
+                 const std::string &fieldName,
+                 const DocsumReply &reply,
+                 uint32_t id,
+                 uint32_t resultClassID);
+
+    bool
+    assertSlime(const std::string &exp,
+                const DocsumReply &reply,
+                uint32_t id);
+
+    void
+    requireThatAdapterHandlesAllFieldTypes();
+
+    void
+    requireThatAdapterHandlesMultipleDocuments();
+
+    void
+    requireThatAdapterHandlesDocumentIdField();
+
+    void
+    requireThatDocsumRequestIsProcessed();
+
+    void
+    requireThatRewritersAreUsed();
+
+    void
+    requireThatAttributesAreUsed();
+
+    void
+    requireThatSummaryAdapterHandlesPutAndRemove();
+
+    void
+    requireThatAnnotationsAreUsed();
+
+    void
+    requireThatUrisAreUsed();
+
+    void
+    requireThatPositionsAreUsed();
+
+    void
+    requireThatRawFieldsWorks();
+
+    void
+    requireThatFieldCacheRepoCanReturnDefaultFieldCache();
+
+public:
+    Test();
+    int Main();
+};
+
+
+GeneralResultPtr
+Test::getResult(DocumentStoreAdapter & dsa, uint32_t docId)
+{
+    DocsumStoreValue docsum = dsa.getMappedDocsum(docId, false);
+    ASSERT_TRUE(docsum.pt() != NULL);
+    GeneralResultPtr retval(new GeneralResult(dsa.getResultClass(),
+                                              0, 0, 0));
+    // skip the 4 byte class id
+    ASSERT_TRUE(retval->unpack(docsum.pt() + 4,
+                               docsum.len() - 4) == 0);
+    return retval;
+}
+
+
+GeneralResultPtr
+Test::getResult(const DocsumReply & reply, uint32_t id, uint32_t resultClassID)
+{
+    GeneralResultPtr retval(new GeneralResult(getResultConfig().
+                                              LookupResultClass(resultClassID),
+                                              0, 0, 0));
+    const DocsumReply::Docsum & docsum = reply.docsums[id];
+    // skip the 4 byte class id
+    ASSERT_EQUAL(0, retval->unpack(docsum.data.c_str() + 4, docsum.data.size() - 4));
+    return retval;
+}
+
+
+bool
+Test::assertString(const std::string & exp, const std::string & fieldName,
+                   DocumentStoreAdapter &dsa,
+                   uint32_t id)
+{
+    GeneralResultPtr res = getResult(dsa, id);
+    return EXPECT_EQUAL(exp, std::string(res->GetEntry(fieldName.c_str())->
+                                       _stringval,
+                                       res->GetEntry(fieldName.c_str())->
+                                       _stringlen));
+}
+
+
+bool
+Test::assertString(const std::string & exp, const std::string & fieldName,
+                   const DocsumReply & reply,
+                   uint32_t id, uint32_t resultClassID)
+{
+    GeneralResultPtr res = getResult(reply, id, resultClassID);
+    return EXPECT_EQUAL(exp, std::string(res->GetEntry(fieldName.c_str())->
+                                       _stringval,
+                                       res->GetEntry(fieldName.c_str())->
+                                       _stringlen));
+}
+
+
+bool
+Test::assertSlime(const std::string &exp, const DocsumReply &reply, uint32_t id)
+{
+    const DocsumReply::Docsum & docsum = reply.docsums[id];
+    uint32_t classId;
+    ASSERT_LESS_EQUAL(sizeof(classId), docsum.data.size());
+    memcpy(&classId, docsum.data.c_str(), sizeof(classId));
+    ASSERT_EQUAL(::search::fs4transport::SLIME_MAGIC_ID, classId);
+    vespalib::Slime slime;
+    vespalib::slime::Memory serialized(docsum.data.c_str() + sizeof(classId),
+                                       docsum.data.size() - sizeof(classId));
+    size_t decodeRes = vespalib::slime::BinaryFormat::decode(serialized,
+                                                             slime);
+    ASSERT_EQUAL(decodeRes, serialized.size);
+    vespalib::Slime expSlime;
+    size_t used = vespalib::slime::JsonFormat::decode(exp, expSlime);
+    EXPECT_EQUAL(exp.size(), used);
+    return EXPECT_EQUAL(expSlime, slime);
+}
+
+void
+Test::requireThatAdapterHandlesAllFieldTypes()
+{
+    Schema s;
+    s.addSummaryField(Schema::SummaryField("a", Schema::INT8));
+    s.addSummaryField(Schema::SummaryField("b", Schema::INT16));
+    s.addSummaryField(Schema::SummaryField("c", Schema::INT32));
+    s.addSummaryField(Schema::SummaryField("d", Schema::INT64));
+    s.addSummaryField(Schema::SummaryField("e", Schema::FLOAT));
+    s.addSummaryField(Schema::SummaryField("f", Schema::DOUBLE));
+    s.addSummaryField(Schema::SummaryField("g", Schema::STRING));
+    s.addSummaryField(Schema::SummaryField("h", Schema::STRING));
+    s.addSummaryField(Schema::SummaryField("i", Schema::RAW));
+    s.addSummaryField(Schema::SummaryField("j", Schema::RAW));
+    s.addSummaryField(Schema::SummaryField("k", Schema::STRING));
+    s.addSummaryField(Schema::SummaryField("l", Schema::STRING));
+
+    BuildContext bc(s);
+    bc._bld.startDocument("doc::0");
+    bc._bld.startSummaryField("a").addInt(255).endField();
+    bc._bld.startSummaryField("b").addInt(32767).endField();
+    bc._bld.startSummaryField("c").addInt(2147483647).endField();
+    bc._bld.startSummaryField("d").addInt(2147483648).endField();
+    bc._bld.startSummaryField("e").addFloat(1234.56).endField();
+    bc._bld.startSummaryField("f").addFloat(9876.54).endField();
+    bc._bld.startSummaryField("g").addStr("foo").endField();
+    bc._bld.startSummaryField("h").addStr("bar").endField();
+    bc._bld.startSummaryField("i").addStr("baz").endField();
+    bc._bld.startSummaryField("j").addStr("qux").endField();
+    bc._bld.startSummaryField("k").addStr("<foo>").endField();
+    bc._bld.startSummaryField("l").addStr("{foo:10}").endField();
+    bc.endDocument(0);
+
+    DocumentStoreAdapter dsa(bc._str,
+                             *bc._repo,
+                             getResultConfig(), "class0",
+                             bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"),
+                             getMarkupFields());
+    GeneralResultPtr res = getResult(dsa, 0);
+    EXPECT_EQUAL(255u,        res->GetEntry("a")->_intval);
+    EXPECT_EQUAL(32767u,      res->GetEntry("b")->_intval);
+    EXPECT_EQUAL(2147483647u, res->GetEntry("c")->_intval);
+    EXPECT_EQUAL(2147483648u, res->GetEntry("d")->_int64val);
+    EXPECT_APPROX(1234.56,    res->GetEntry("e")->_doubleval, 10e-5);
+    EXPECT_APPROX(9876.54,    res->GetEntry("f")->_doubleval, 10e-5);
+    EXPECT_EQUAL("foo",       std::string(res->GetEntry("g")->_stringval,
+                                        res->GetEntry("g")->_stringlen));
+    EXPECT_EQUAL("bar",       std::string(res->GetEntry("h")->_stringval,
+                                        res->GetEntry("h")->_stringlen));
+    EXPECT_EQUAL("baz",       std::string(res->GetEntry("i")->_dataval,
+                                        res->GetEntry("i")->_datalen));
+    EXPECT_EQUAL("qux",       std::string(res->GetEntry("j")->_dataval,
+                                        res->GetEntry("j")->_datalen));
+    EXPECT_EQUAL("<foo>",     std::string(res->GetEntry("k")->_stringval,
+                                        res->GetEntry("k")->_stringlen));
+    EXPECT_EQUAL("{foo:10}",  std::string(res->GetEntry("l")->_stringval,
+                                        res->GetEntry("l")->_stringlen));
+}
+
+
+void
+Test::requireThatAdapterHandlesMultipleDocuments()
+{
+    Schema s;
+    s.addSummaryField(Schema::SummaryField("a", Schema::INT32));
+
+    BuildContext bc(s);
+    bc._bld.startDocument("doc::0").
+        startSummaryField("a").
+        addInt(1000).
+        endField();
+    bc.endDocument(0);
+    bc._bld.startDocument("doc::1").
+        startSummaryField("a").
+        addInt(2000).endField();
+    bc.endDocument(1);
+
+    DocumentStoreAdapter dsa(bc._str, *bc._repo, getResultConfig(), "class1",
+                             bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class1"),
+                             getMarkupFields());
+    { // doc 0
+        GeneralResultPtr res = getResult(dsa, 0);
+        EXPECT_EQUAL(1000u, res->GetEntry("a")->_intval);
+    }
+    { // doc 1
+        GeneralResultPtr res = getResult(dsa, 1);
+        EXPECT_EQUAL(2000u, res->GetEntry("a")->_intval);
+    }
+    { // doc 2
+        DocsumStoreValue docsum = dsa.getMappedDocsum(2, false);
+        EXPECT_TRUE(docsum.pt() == NULL);
+    }
+    { // doc 0 (again)
+        GeneralResultPtr res = getResult(dsa, 0);
+        EXPECT_EQUAL(1000u, res->GetEntry("a")->_intval);
+    }
+    EXPECT_EQUAL(0u, bc._str.lastSyncToken());
+    uint64_t flushToken = bc._str.initFlush(bc._serialNum - 1);
+    bc._str.flush(flushToken);
+}
+
+
+void
+Test::requireThatAdapterHandlesDocumentIdField()
+{
+    Schema s;
+    s.addSummaryField(Schema::SummaryField("documentid",
+                              Schema::STRING));
+    BuildContext bc(s);
+    bc._bld.startDocument("doc::0").
+        startSummaryField("documentid").
+        addStr("foo").
+        endField();
+    bc.endDocument(0);
+    DocumentStoreAdapter dsa(bc._str, *bc._repo, getResultConfig(), "class4",
+                             bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class4"),
+                             getMarkupFields());
+    GeneralResultPtr res = getResult(dsa, 0);
+    EXPECT_EQUAL("doc::0", std::string(res->GetEntry("documentid")->_stringval,
+                                     res->GetEntry("documentid")->_stringlen));
+}
+
+
+GlobalId gid1 = DocumentId("doc::1").getGlobalId(); // lid 1
+GlobalId gid2 = DocumentId("doc::2").getGlobalId(); // lid 2
+GlobalId gid3 = DocumentId("doc::3").getGlobalId(); // lid 3
+GlobalId gid4 = DocumentId("doc::4").getGlobalId(); // lid 4
+GlobalId gid9 = DocumentId("doc::9").getGlobalId(); // not existing
+
+
+void
+Test::requireThatDocsumRequestIsProcessed()
+{
+    Schema s;
+    s.addSummaryField(Schema::SummaryField("a", Schema::INT32));
+
+    BuildContext bc(s);
+    DBContext dc(bc._repo, getDocTypeName());
+    dc.put(*bc._bld.startDocument("doc::1").
+           startSummaryField("a").
+           addInt(10).
+           endField().
+           endDocument(),
+           1);
+    dc.put(*bc._bld.startDocument("doc::2").
+           startSummaryField("a").
+           addInt(20).
+           endField().
+           endDocument(),
+           2);
+    dc.put(*bc._bld.startDocument("doc::3").
+           startSummaryField("a").
+           addInt(30).
+           endField().
+           endDocument(),
+           3);
+    dc.put(*bc._bld.startDocument("doc::4").
+           startSummaryField("a").
+           addInt(40).
+           endField().
+           endDocument(),
+           4);
+    dc.put(*bc._bld.startDocument("doc::5").
+           startSummaryField("a").
+           addInt(50).
+           endField().
+           endDocument(),
+           5);
+
+    DocsumRequest req;
+    req.resultClassName = "class1";
+    req.hits.push_back(DocsumRequest::Hit(gid2));
+    req.hits.push_back(DocsumRequest::Hit(gid4));
+    req.hits.push_back(DocsumRequest::Hit(gid9));
+    DocsumReply::UP rep = dc._ddb->getDocsums(req);
+    EXPECT_EQUAL(3u, rep->docsums.size());
+    EXPECT_EQUAL(2u, rep->docsums[0].docid);
+    EXPECT_EQUAL(gid2, rep->docsums[0].gid);
+    EXPECT_EQUAL(20u, getResult(*rep, 0, 1)->GetEntry("a")->_intval);
+    EXPECT_EQUAL(4u, rep->docsums[1].docid);
+    EXPECT_EQUAL(gid4, rep->docsums[1].gid);
+    EXPECT_EQUAL(40u, getResult(*rep, 1, 1)->GetEntry("a")->_intval);
+    EXPECT_EQUAL(search::endDocId, rep->docsums[2].docid);
+    EXPECT_EQUAL(gid9, rep->docsums[2].gid);
+    EXPECT_TRUE(rep->docsums[2].data.get() == NULL);
+}
+
+
+void
+Test::requireThatRewritersAreUsed()
+{
+    Schema s;
+    s.addSummaryField(Schema::SummaryField("aa", Schema::INT32));
+    s.addSummaryField(Schema::SummaryField("ab", Schema::INT32));
+
+    BuildContext bc(s);
+    DBContext dc(bc._repo, getDocTypeName());
+    dc.put(*bc._bld.startDocument("doc::1").
+           startSummaryField("aa").
+           addInt(10).
+           endField().
+           startSummaryField("ab").
+           addInt(20).
+           endField().
+           endDocument(),
+           1);
+
+    DocsumRequest req;
+    req.resultClassName = "class2";
+    req.hits.push_back(DocsumRequest::Hit(gid1));
+    DocsumReply::UP rep = dc._ddb->getDocsums(req);
+    EXPECT_EQUAL(1u, rep->docsums.size());
+    EXPECT_EQUAL(20u, getResult(*rep, 0, 2)->GetEntry("aa")->_intval);
+    EXPECT_EQUAL(0u,  getResult(*rep, 0, 2)->GetEntry("ab")->_intval);
+}
+
+
+void
+addField(Schema & s,
+         const std::string &name,
+         Schema::DataType dtype,
+         Schema::CollectionType ctype)
+{
+    s.addSummaryField(Schema::SummaryField(name, dtype, ctype));
+    s.addAttributeField(Schema::AttributeField(name, dtype, ctype));
+}
+
+
+void
+Test::requireThatAttributesAreUsed()
+{
+    Schema s;
+    addField(s, "ba",
+             Schema::INT32, Schema::SINGLE);
+    addField(s, "bb",
+             Schema::FLOAT, Schema::SINGLE);
+    addField(s, "bc",
+             Schema::STRING, Schema::SINGLE);
+    addField(s, "bd",
+             Schema::INT32, Schema::ARRAY);
+    addField(s, "be",
+             Schema::FLOAT, Schema::ARRAY);
+    addField(s, "bf",
+             Schema::STRING, Schema::ARRAY);
+    addField(s, "bg",
+             Schema::INT32, Schema::WEIGHTEDSET);
+    addField(s, "bh",
+             Schema::FLOAT, Schema::WEIGHTEDSET);
+    addField(s, "bi",
+             Schema::STRING, Schema::WEIGHTEDSET);
+    addField(s, "bj", Schema::TENSOR, Schema::SINGLE);
+
+    BuildContext bc(s);
+    DBContext dc(bc._repo, getDocTypeName());
+    dc.put(*bc._bld.startDocument("doc::1").
+           endDocument(),
+           1); // empty doc
+    dc.put(*bc._bld.startDocument("doc::2").
+           startAttributeField("ba").
+           addInt(10).
+           endField().
+           startAttributeField("bb").
+           addFloat(10.1).
+           endField().
+           startAttributeField("bc").
+           addStr("foo").
+           endField().
+           startAttributeField("bd").
+           startElement().
+           addInt(20).
+           endElement().
+           startElement().
+           addInt(30).
+           endElement().
+           endField().
+           startAttributeField("be").
+           startElement().
+           addFloat(20.2).
+           endElement().
+           startElement().
+           addFloat(30.3).
+           endElement().
+           endField().
+           startAttributeField("bf").
+           startElement().
+           addStr("bar").
+           endElement().
+           startElement().
+           addStr("baz").
+           endElement().
+           endField().
+           startAttributeField("bg").
+           startElement(2).
+           addInt(40).
+           endElement().
+           startElement(3).
+           addInt(50).
+           endElement().
+           endField().
+           startAttributeField("bh").
+           startElement(4).
+           addFloat(40.4).
+           endElement().
+           startElement(5).
+           addFloat(50.5).
+           endElement().
+           endField().
+           startAttributeField("bi").
+           startElement(7).
+           addStr("quux").
+           endElement().
+           startElement(6).
+           addStr("qux").
+           endElement().
+           endField().
+           startAttributeField("bj").
+           addTensor(createTensor({ {{}, 3} }, { "x", "y"})).
+           endField().
+           endDocument(),
+           2);
+    dc.put(*bc._bld.startDocument("doc::3").
+           endDocument(),
+           3); // empty doc
+
+    DocsumRequest req;
+    req.resultClassName = "class3";
+    req.hits.push_back(DocsumRequest::Hit(gid2));
+    req.hits.push_back(DocsumRequest::Hit(gid3));
+    DocsumReply::UP rep = dc._ddb->getDocsums(req);
+    uint32_t rclass = 3;
+
+    EXPECT_EQUAL(2u, rep->docsums.size());
+    EXPECT_EQUAL(10u, getResult(*rep, 0, rclass)->GetEntry("ba")->_intval);
+    EXPECT_APPROX(10.1, getResult(*rep, 0, rclass)->GetEntry("bb")->_doubleval,
+                10e-5);
+    EXPECT_TRUE(assertString("foo", "bc", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("[\"20\",\"30\"]",     "bd", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("[\"20.2\",\"30.3\"]", "be", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("[\"bar\",\"baz\"]",   "bf", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("[[\"40\",2],[\"50\",3]]",     "bg",
+                            *rep, 0, rclass));
+    EXPECT_TRUE(assertString("[[\"40.4\",4],[\"50.5\",5]]", "bh",
+                            *rep, 0, rclass));
+    EXPECT_TRUE(assertString("[[\"quux\",7],[\"qux\",6]]",  "bi",
+                            *rep, 0, rclass));
+    EXPECT_TRUE(assertString("{\"dimensions\":[\"x\",\"y\"],"
+                             "\"cells\":[{\"address\":{},\"value\":3}]}",
+                             "bj", *rep, 0, rclass));
+
+    // empty doc
+    EXPECT_TRUE(search::attribute::isUndefined<int32_t>
+               (getResult(*rep, 1, rclass)->GetEntry("ba")->_intval));
+    EXPECT_TRUE(search::attribute::isUndefined<float>
+               (getResult(*rep, 1, rclass)->GetEntry("bb")->_doubleval));
+    EXPECT_TRUE(assertString("", "bc", *rep, 1, rclass));
+    EXPECT_TRUE(assertString("[]", "bd", *rep, 1, rclass));
+    EXPECT_TRUE(assertString("[]", "be", *rep, 1, rclass));
+    EXPECT_TRUE(assertString("[]", "bf", *rep, 1, rclass));
+    EXPECT_TRUE(assertString("[]", "bg", *rep, 1, rclass));
+    EXPECT_TRUE(assertString("[]", "bh", *rep, 1, rclass));
+    EXPECT_TRUE(assertString("[]", "bi", *rep, 1, rclass));
+    EXPECT_TRUE(assertString("", "bj", *rep, 1, rclass));
+
+    proton::IAttributeManager::SP attributeManager =
+        dc._ddb->getReadySubDB()->getAttributeManager();
+    search::ISequencedTaskExecutor &attributeFieldWriter =
+        attributeManager->getAttributeFieldWriter();
+    search::AttributeVector *bjAttr =
+        attributeManager->getWritableAttribute("bj");
+    search::attribute::TensorAttribute *bjTensorAttr =
+        dynamic_cast<search::attribute::TensorAttribute *>(bjAttr);
+
+    attributeFieldWriter.
+        execute("bj",
+                [&]() { bjTensorAttr->setTensor(3,
+                            *createTensor({ {{}, 4} }, { "x"}));
+                           bjTensorAttr->commit(); });
+    attributeFieldWriter.sync();
+
+    DocsumReply::UP rep2 = dc._ddb->getDocsums(req);
+    EXPECT_TRUE(assertString("{\"dimensions\":[\"x\",\"y\"],"
+                             "\"cells\":[{\"address\":{},\"value\":4}]}",
+                             "bj", *rep2, 1, rclass));
+
+    DocsumRequest req3;
+    req3.resultClassName = "class3";
+    req3._flags = ::search::fs4transport::GDFLAG_ALLOW_SLIME;
+    req3.hits.push_back(DocsumRequest::Hit(gid3));
+    DocsumReply::UP rep3 = dc._ddb->getDocsums(req3);
+
+    EXPECT_TRUE(assertSlime("{bd:[],be:[],bf:[],bg:[],"
+                            "bh:[],bi:[],"
+                            "bj:{dimensions:['x','y'],"
+                            "cells:[{address:{},value:4.0}]}}",
+                            *rep3, 0));
+}
+
+
+void
+Test::requireThatSummaryAdapterHandlesPutAndRemove()
+{
+    Schema s;
+    s.addSummaryField(Schema::SummaryField("f1",
+                              Schema::STRING,
+                              Schema::SINGLE));
+    BuildContext bc(s);
+    DBContext dc(bc._repo, getDocTypeName());
+    Document::UP exp = bc._bld.startDocument("doc::1").
+                       startSummaryField("f1").
+                       addStr("foo").
+                       endField().
+                       endDocument();
+    dc._sa->put(1, *exp, 1);
+    IDocumentStore & store =
+        dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore();
+    Document::UP act = store.read(1, *bc._repo);
+    EXPECT_TRUE(act.get() != NULL);
+    EXPECT_EQUAL(exp->getType(), act->getType());
+    EXPECT_EQUAL("foo", act->getValue("f1")->toString());
+    dc._sa->remove(2, 1);
+    EXPECT_TRUE(store.read(1, *bc._repo).get() == NULL);
+}
+
+
+const std::string TERM_ORIG = "\357\277\271";
+const std::string TERM_INDEX = "\357\277\272";
+const std::string TERM_END = "\357\277\273";
+const std::string TERM_SEP = "\037";
+const std::string TERM_EMPTY = "";
+namespace
+{
+  const std::string empty;
+}
+
+void
+Test::requireThatAnnotationsAreUsed()
+{
+    Schema s;
+    s.addIndexField(Schema::IndexField("g",
+                                       Schema::STRING,
+                                       Schema::SINGLE));
+    s.addSummaryField(Schema::SummaryField("g",
+                              Schema::STRING,
+                              Schema::SINGLE));
+    s.addIndexField(Schema::IndexField("dynamicstring",
+                                       Schema::STRING,
+                                       Schema::SINGLE));
+    s.addSummaryField(Schema::SummaryField("dynamicstring",
+                              Schema::STRING,
+                              Schema::SINGLE));
+    BuildContext bc(s);
+    DBContext dc(bc._repo, getDocTypeName());
+    Document::UP exp = bc._bld.startDocument("doc::0").
+                       startIndexField("g").
+                       addStr("foo").
+                       addStr("bar").
+                       addTermAnnotation("baz").
+                       endField().
+                       startIndexField("dynamicstring").
+                       setAutoAnnotate(false).
+                       addStr("foo").
+                       addSpan().
+                       addAlphabeticTokenAnnotation().
+                       addTermAnnotation().
+                       addNoWordStr(" ").
+                       addSpan().
+                       addSpaceTokenAnnotation().
+                       addStr("bar").
+                       addSpan().
+                       addAlphabeticTokenAnnotation().
+                       addTermAnnotation("baz").
+                       setAutoAnnotate(true).
+                       endField().
+                       endDocument();
+    dc._sa->put(1, *exp, 1);
+
+    IDocumentStore & store =
+        dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore();
+    Document::UP act = store.read(1, *bc._repo);
+    EXPECT_TRUE(act.get() != NULL);
+    EXPECT_EQUAL(exp->getType(), act->getType());
+    EXPECT_EQUAL("foo bar", act->getValue("g")->getAsString());
+    EXPECT_EQUAL("foo bar", act->getValue("dynamicstring")->getAsString());
+
+    DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0",
+                             bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"),
+                             getMarkupFields());
+    EXPECT_TRUE(assertString("foo bar", "g", dsa, 1));
+    EXPECT_TRUE(assertString(TERM_EMPTY + "foo" + TERM_SEP +
+                            " " + TERM_SEP +
+                            TERM_ORIG + "bar" + TERM_INDEX + "baz" + TERM_END +
+                            TERM_SEP,
+                            "dynamicstring", dsa, 1));
+}
+
+void
+Test::requireThatUrisAreUsed()
+{
+    Schema s;
+    s.addUriIndexFields(Schema::IndexField("urisingle",
+                                Schema::STRING,
+                                Schema::SINGLE));
+    s.addSummaryField(Schema::SummaryField("urisingle",
+                              Schema::STRING,
+                              Schema::SINGLE));
+    s.addUriIndexFields(Schema::IndexField("uriarray",
+                                Schema::STRING,
+                                Schema::ARRAY));
+    s.addSummaryField(Schema::SummaryField("uriarray",
+                              Schema::STRING,
+                              Schema::ARRAY));
+    s.addUriIndexFields(Schema::IndexField("uriwset",
+                                Schema::STRING,
+                                Schema::WEIGHTEDSET));
+    s.addSummaryField(Schema::SummaryField("uriwset",
+                              Schema::STRING,
+                              Schema::WEIGHTEDSET));
+    BuildContext bc(s);
+    DBContext dc(bc._repo, getDocTypeName());
+    Document::UP exp = bc._bld.startDocument("doc::0").
+                       startIndexField("urisingle").
+                       startSubField("all").
+                       addUrlTokenizedString(
+                               "http://www.yahoo.com:81/fluke?ab=2#4").
+                       endSubField().
+                       startSubField("scheme").
+                       addUrlTokenizedString("http").
+                       endSubField().
+                       startSubField("host").
+                       addUrlTokenizedString("www.yahoo.com").
+                       endSubField().
+                       startSubField("port").
+                       addUrlTokenizedString("81").
+                       endSubField().
+                       startSubField("path").
+                       addUrlTokenizedString("/fluke").
+                       endSubField().
+                       startSubField("query").
+                       addUrlTokenizedString("ab=2").
+                       endSubField().
+                       startSubField("fragment").
+                       addUrlTokenizedString("4").
+                       endSubField().
+                       endField().
+                       startIndexField("uriarray").
+                       startElement(1).
+                       startSubField("all").
+                       addUrlTokenizedString(
+                               "http://www.yahoo.com:82/fluke?ab=2#8").
+                       endSubField().
+                       startSubField("scheme").
+                       addUrlTokenizedString("http").
+                       endSubField().
+                       startSubField("host").
+                       addUrlTokenizedString("www.yahoo.com").
+                       endSubField().
+                       startSubField("port").
+                       addUrlTokenizedString("82").
+                       endSubField().
+                       startSubField("path").
+                       addUrlTokenizedString("/fluke").
+                       endSubField().
+                       startSubField("query").
+                       addUrlTokenizedString("ab=2").
+                       endSubField().
+                       startSubField("fragment").
+                       addUrlTokenizedString("8").
+                       endSubField().
+                       endElement().
+                       startElement(1).
+                       startSubField("all").
+                       addUrlTokenizedString(
+                               "http://www.flickr.com:82/fluke?ab=2#9").
+                       endSubField().
+                       startSubField("scheme").
+                       addUrlTokenizedString("http").
+                       endSubField().
+                       startSubField("host").
+                       addUrlTokenizedString("www.flickr.com").
+                       endSubField().
+                       startSubField("port").
+                       addUrlTokenizedString("82").
+                       endSubField().
+                       startSubField("path").
+                       addUrlTokenizedString("/fluke").
+                       endSubField().
+                       startSubField("query").
+                       addUrlTokenizedString("ab=2").
+                       endSubField().
+                       startSubField("fragment").
+                       addUrlTokenizedString("9").
+                       endSubField().
+                       endElement().
+                       endField().
+                       startIndexField("uriwset").
+                       startElement(4).
+                       startSubField("all").
+                       addUrlTokenizedString(
+                               "http://www.yahoo.com:83/fluke?ab=2#12").
+                       endSubField().
+                       startSubField("scheme").
+                       addUrlTokenizedString("http").
+                       endSubField().
+                       startSubField("host").
+                       addUrlTokenizedString("www.yahoo.com").
+                       endSubField().
+                       startSubField("port").
+                       addUrlTokenizedString("83").
+                       endSubField().
+                       startSubField("path").
+                       addUrlTokenizedString("/fluke").
+                       endSubField().
+                       startSubField("query").
+                       addUrlTokenizedString("ab=2").
+                       endSubField().
+                       startSubField("fragment").
+                       addUrlTokenizedString("12").
+                       endSubField().
+                       endElement().
+                       startElement(7).
+                       startSubField("all").
+                       addUrlTokenizedString(
+                               "http://www.flickr.com:85/fluke?ab=2#13").
+                       endSubField().
+                       startSubField("scheme").
+                       addUrlTokenizedString("http").
+                       endSubField().
+                       startSubField("host").
+                       addUrlTokenizedString("www.flickr.com").
+                       endSubField().
+                       startSubField("port").
+                       addUrlTokenizedString("85").
+                       endSubField().
+                       startSubField("path").
+                       addUrlTokenizedString("/fluke").
+                       endSubField().
+                       startSubField("query").
+                       addUrlTokenizedString("ab=2").
+                       endSubField().
+                       startSubField("fragment").
+                       addUrlTokenizedString("13").
+                       endSubField().
+                       endElement().
+                       endField().
+                       endDocument();
+    dc._sa->put(1, *exp, 1);
+
+    IDocumentStore & store =
+        dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore();
+    Document::UP act = store.read(1, *bc._repo);
+    EXPECT_TRUE(act.get() != NULL);
+    EXPECT_EQUAL(exp->getType(), act->getType());
+
+    DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0",
+                             bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"),
+                             getMarkupFields());
+    EXPECT_TRUE(assertString("http://www.yahoo.com:81/fluke?ab=2#4",
+                            "urisingle", dsa, 1));
+    EXPECT_TRUE(assertString("[\"http://www.yahoo.com:82/fluke?ab=2#8\","
+                            "\"http://www.flickr.com:82/fluke?ab=2#9\"]",
+                            "uriarray", dsa, 1));
+    EXPECT_TRUE(assertString("["
+                               "{\"item\":\"http://www.yahoo.com:83/fluke?ab=2#12\",\"weight\":4}"
+                               ","
+                               "{\"item\":\"http://www.flickr.com:85/fluke?ab=2#13\",\"weight\":7}"
+                            "]",
+                            "uriwset", dsa, 1));
+}
+
+
+void
+Test::requireThatPositionsAreUsed()
+{
+    Schema s;
+    s.addAttributeField(Schema::AttributeField("sp2",
+                                Schema::INT64));
+    s.addAttributeField(Schema::AttributeField("ap2",
+                                Schema::INT64,
+                                Schema::ARRAY));
+    s.addAttributeField(Schema::AttributeField("wp2",
+                                Schema::INT64,
+                                Schema::WEIGHTEDSET));
+
+    BuildContext bc(s);
+    DBContext dc(bc._repo, getDocTypeName());
+    Document::UP exp = bc._bld.startDocument("doc::1").
+                       startAttributeField("sp2").
+                       addPosition(1002, 1003).
+                       endField().
+                       startAttributeField("ap2").
+                       startElement().addPosition(1006, 1007).endElement().
+                       startElement().addPosition(1008, 1009).endElement().
+                       endField().
+                       startAttributeField("wp2").
+                       startElement(43).addPosition(1012, 1013).endElement().
+                       startElement(44).addPosition(1014, 1015).endElement().
+                       endField().
+                       endDocument();
+    dc.put(*exp, 1);
+
+    IDocumentStore & store =
+        dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore();
+    Document::UP act = store.read(1, *bc._repo);
+    EXPECT_TRUE(act.get() != NULL);
+    EXPECT_EQUAL(exp->getType(), act->getType());
+
+    DocsumRequest req;
+    req.resultClassName = "class5";
+    req.hits.push_back(DocsumRequest::Hit(gid1));
+    DocsumReply::UP rep = dc._ddb->getDocsums(req);
+    uint32_t rclass = 5;
+
+    EXPECT_EQUAL(1u, rep->docsums.size());
+    EXPECT_EQUAL(1u, rep->docsums[0].docid);
+    EXPECT_EQUAL(gid1, rep->docsums[0].gid);
+    EXPECT_TRUE(assertString("1047758",
+                            "sp2", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("<position x=\"1002\" y=\"1003\" latlong=\"N0.001003;E0.001002\" />",
+                            "sp2x", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("[1047806,1048322]",
+                             "ap2", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("<position x=\"1006\" y=\"1007\" latlong=\"N0.001007;E0.001006\" />"
+                            "<position x=\"1008\" y=\"1009\" latlong=\"N0.001009;E0.001008\" />",
+                            "ap2x", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("[{\"item\":1048370,\"weight\":43},{\"item\":1048382,\"weight\":44}]",
+                            "wp2", *rep, 0, rclass));
+    EXPECT_TRUE(assertString("<position x=\"1012\" y=\"1013\" latlong=\"N0.001013;E0.001012\" />"
+                            "<position x=\"1014\" y=\"1015\" latlong=\"N0.001015;E0.001014\" />",
+                            "wp2x", *rep, 0, rclass));
+}
+
+
+void
+Test::requireThatRawFieldsWorks()
+{
+    Schema s;
+    s.addSummaryField(Schema::AttributeField("i",
+                                Schema::RAW));
+    s.addSummaryField(Schema::AttributeField("araw",
+                                Schema::RAW,
+                                Schema::ARRAY));
+    s.addSummaryField(Schema::AttributeField("wraw",
+                              Schema::RAW,
+                              Schema::WEIGHTEDSET));
+
+    std::vector<char> binaryBlob;
+    binaryBlob.push_back('\0');
+    binaryBlob.push_back('\2');
+    binaryBlob.push_back('\1');
+    std::string raw1s("Single Raw Element");
+    std::string raw1a0("Array Raw Element 0");
+    std::string raw1a1("Array Raw Element  1");
+    std::string raw1w0("Weighted Set Raw Element 0");
+    std::string raw1w1("Weighted Set Raw Element  1");
+    raw1s += std::string(&binaryBlob[0],
+                         &binaryBlob[0] + binaryBlob.size());
+    raw1a0 += std::string(&binaryBlob[0],
+                          &binaryBlob[0] + binaryBlob.size());
+    raw1a1 += std::string(&binaryBlob[0],
+                          &binaryBlob[0] + binaryBlob.size());
+    raw1w0 += std::string(&binaryBlob[0],
+                          &binaryBlob[0] + binaryBlob.size());
+    raw1w1 += std::string(&binaryBlob[0],
+                          &binaryBlob[0] + binaryBlob.size());
+
+    BuildContext bc(s);
+    DBContext dc(bc._repo, getDocTypeName());
+    Document::UP exp = bc._bld.startDocument("doc::0").
+                       startSummaryField("i").
+                       addRaw(raw1s.c_str(), raw1s.size()).
+                       endField().
+                       startSummaryField("araw").
+                       startElement().
+                       addRaw(raw1a0.c_str(), raw1a0.size()).
+                       endElement().
+                       startElement().
+                       addRaw(raw1a1.c_str(), raw1a1.size()).
+                       endElement().
+                       endField().
+                       startSummaryField("wraw").
+                       startElement(46).
+                       addRaw(raw1w1.c_str(), raw1w1.size()).
+                       endElement().
+                       startElement(45).
+                       addRaw(raw1w0.c_str(), raw1w0.size()).
+                       endElement().
+                       endField().
+                       endDocument();
+    dc._sa->put(1, *exp, 1);
+
+    IDocumentStore & store =
+        dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore();
+    Document::UP act = store.read(1, *bc._repo);
+    EXPECT_TRUE(act.get() != NULL);
+    EXPECT_EQUAL(exp->getType(), act->getType());
+
+    DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0",
+                             bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"),
+                             getMarkupFields());
+
+    ASSERT_TRUE(assertString(raw1s,
+                            "i", dsa, 1));
+    ASSERT_TRUE(assertString(empty + "[\"" +
+                             vespalib::Base64::encode(raw1a0) +
+                             "\",\"" +
+                             vespalib::Base64::encode(raw1a1) +
+                             "\"]",
+                             "araw", dsa, 1));
+    ASSERT_TRUE(assertString(empty + "[{\"item\":\"" +
+                             vespalib::Base64::encode(raw1w1) +
+                             "\",\"weight\":46},{\"item\":\"" +
+                             vespalib::Base64::encode(raw1w0) +
+                             "\",\"weight\":45}]",
+                             "wraw", dsa, 1));
+}
+
+
+void
+Test::requireThatFieldCacheRepoCanReturnDefaultFieldCache()
+{
+    Schema s;
+    s.addSummaryField(Schema::SummaryField("a", Schema::INT32));
+    BuildContext bc(s);
+    FieldCacheRepo::UP repo = bc.createFieldCacheRepo(getResultConfig());
+    FieldCache::CSP cache = repo->getFieldCache("");
+    EXPECT_TRUE(cache.get() == repo->getFieldCache("class1").get());
+    EXPECT_EQUAL(1u, cache->size());
+    EXPECT_EQUAL("a", cache->getField(0)->getName());
+}
+
+
+Test::Test()
+    : _summaryCfg(),
+      _resultCfg(),
+      _markupFields()
+{
+    std::string cfgId("summary");
+    _summaryCfg = config::ConfigGetter<vespa::config::search::SummaryConfig>::getConfig(cfgId, config::FileSpec("summary.cfg"));
+    _resultCfg.ReadConfig(*_summaryCfg, cfgId.c_str());
+    std::string mapCfgId("summarymap");
+    std::unique_ptr<vespa::config::search::SummarymapConfig> mapCfg = config::ConfigGetter<vespa::config::search::SummarymapConfig>::getConfig(mapCfgId, config::FileSpec("summarymap.cfg"));
+    for (size_t i = 0; i < mapCfg->override.size(); ++i) {
+        const vespa::config::search::SummarymapConfig::Override & o = mapCfg->override[i];
+        if (o.command == "dynamicteaser") {
+            vespalib::string markupField = o.arguments;
+            if (markupField.empty())
+                continue;
+            // Assume just one argument: source field that must contain markup
+            _markupFields.insert(markupField);
+            LOG(info,
+                "Field %s has markup",
+                markupField.c_str());
+        }
+    }
+}
+
+
+int
+Test::Main()
+{
+    TEST_INIT("docsummary_test");
+
+    if (_argc > 0) {
+        DummyFileHeaderContext::setCreator(_argv[0]);
+    }
+    TEST_DO(requireThatSummaryAdapterHandlesPutAndRemove());
+    TEST_DO(requireThatAdapterHandlesAllFieldTypes());
+    TEST_DO(requireThatAdapterHandlesMultipleDocuments());
+    TEST_DO(requireThatAdapterHandlesDocumentIdField());
+    TEST_DO(requireThatDocsumRequestIsProcessed());
+    TEST_DO(requireThatRewritersAreUsed());
+    TEST_DO(requireThatAttributesAreUsed());
+    TEST_DO(requireThatAnnotationsAreUsed());
+    TEST_DO(requireThatUrisAreUsed());
+    TEST_DO(requireThatPositionsAreUsed());
+    TEST_DO(requireThatRawFieldsWorks());
+    TEST_DO(requireThatFieldCacheRepoCanReturnDefaultFieldCache());
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(proton::Test);
diff --git a/searchcore/src/tests/proton/docsummary/docsummary_test.sh b/searchcore/src/tests/proton/docsummary/docsummary_test.sh
new file mode 100755
index 00000000000..4871911e1cd
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/docsummary_test.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+rm -rf tmp
+rm -rf tmpdb
+rm -rf summary
+rm -rf indexingdocument
+rm -rf searchdocument
+rm -rf *.dat
+$VALGRIND ./searchcore_docsummary_test_app
+rm -rf tmp
+rm -rf tmpdb
+rm -rf summary
+rm -rf indexingdocument
+rm -rf searchdocument
+rm -rf *.dat
+$VALGRIND ./searchcore_summaryfieldconverter_test_app
diff --git a/searchcore/src/tests/proton/docsummary/documentmanager.cfg b/searchcore/src/tests/proton/docsummary/documentmanager.cfg
new file mode 100644
index 00000000000..91c69cc0c70
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/documentmanager.cfg
@@ -0,0 +1,81 @@
+enablecompression false
+datatype[6]
+datatype[0].id -1636745577
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name typea.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[4]
+datatype[0].structtype[0].field[0].name floatfield
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 1
+datatype[0].structtype[0].field[1].name stringfield
+datatype[0].structtype[0].field[1].id[0]
+datatype[0].structtype[0].field[1].datatype 2
+datatype[0].structtype[0].field[2].name longfield
+datatype[0].structtype[0].field[2].id[0]
+datatype[0].structtype[0].field[2].datatype 4
+datatype[0].structtype[0].field[3].name urifield
+datatype[0].structtype[0].field[3].id[0]
+datatype[0].structtype[0].field[3].datatype 10
+datatype[0].documenttype[0]
+datatype[1].id 1878320748
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name typea.body
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[4]
+datatype[1].structtype[0].field[0].name intfield
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[0].datatype 0
+datatype[1].structtype[0].field[1].name rawfield
+datatype[1].structtype[0].field[1].id[0]
+datatype[1].structtype[0].field[1].datatype 3
+datatype[1].structtype[0].field[2].name doublefield
+datatype[1].structtype[0].field[2].id[0]
+datatype[1].structtype[0].field[2].datatype 5
+datatype[1].structtype[0].field[3].name bytefield
+datatype[1].structtype[0].field[3].id[0]
+datatype[1].structtype[0].field[3].datatype 16
+datatype[1].documenttype[0]
+datatype[2].id -1175657560
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name typea
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct -1636745577
+datatype[2].documenttype[0].bodystruct 1878320748
+datatype[3].id 192273965
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name typeb.header
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[0]
+datatype[3].documenttype[0]
+datatype[4].id -72846462
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name typeb.body
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[1]
+datatype[4].structtype[0].field[0].name intfield
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 0
+datatype[4].documenttype[0]
+datatype[5].id -1146158894
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name typeb
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0]
+datatype[5].documenttype[0].headerstruct 192273965
+datatype[5].documenttype[0].bodystruct -72846462
diff --git a/searchcore/src/tests/proton/docsummary/indexingdocument.cfg b/searchcore/src/tests/proton/docsummary/indexingdocument.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/indexingdocument.cfg
diff --git a/searchcore/src/tests/proton/docsummary/indexschema.cfg b/searchcore/src/tests/proton/docsummary/indexschema.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/indexschema.cfg
diff --git a/searchcore/src/tests/proton/docsummary/juniperrc.cfg b/searchcore/src/tests/proton/docsummary/juniperrc.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/juniperrc.cfg
diff --git a/searchcore/src/tests/proton/docsummary/rank-profiles.cfg b/searchcore/src/tests/proton/docsummary/rank-profiles.cfg
new file mode 100644
index 00000000000..34d8f0245df
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/rank-profiles.cfg
@@ -0,0 +1,2 @@
+rankprofile[1]
+rankprofile[0].name default
diff --git a/searchcore/src/tests/proton/docsummary/summary.cfg b/searchcore/src/tests/proton/docsummary/summary.cfg
new file mode 100644
index 00000000000..52f300ae3e0
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/summary.cfg
@@ -0,0 +1,108 @@
+defaultsummaryid 1
+classes[6]
+classes[0].name "class0"
+classes[0].id 0
+classes[0].fields[24]
+classes[0].fields[0].name "a"
+classes[0].fields[0].type "byte"
+classes[0].fields[1].name "b"
+classes[0].fields[1].type "short"
+classes[0].fields[2].name "c"
+classes[0].fields[2].type "integer"
+classes[0].fields[3].name "d"
+classes[0].fields[3].type "int64"
+classes[0].fields[4].name "e"
+classes[0].fields[4].type "float"
+classes[0].fields[5].name "f"
+classes[0].fields[5].type "double"
+classes[0].fields[6].name "g"
+classes[0].fields[6].type "string"
+classes[0].fields[7].name "h"
+classes[0].fields[7].type "longstring"
+classes[0].fields[8].name "i"
+classes[0].fields[8].type "data"
+classes[0].fields[9].name "j"
+classes[0].fields[9].type "longdata"
+classes[0].fields[10].name "k"
+classes[0].fields[10].type "xmlstring"
+classes[0].fields[11].name "l"
+classes[0].fields[11].type "jsonstring"
+classes[0].fields[12].name "dynamicstring"
+classes[0].fields[12].type "string"
+classes[0].fields[13].name "urisingle"
+classes[0].fields[13].type "string"
+classes[0].fields[14].name "uriarray"
+classes[0].fields[14].type "jsonstring"
+classes[0].fields[15].name "uriwset"
+classes[0].fields[15].type "jsonstring"
+classes[0].fields[16].name "sp1"
+classes[0].fields[16].type "string"
+classes[0].fields[17].name "sp2"
+classes[0].fields[17].type "string"
+classes[0].fields[18].name "ap1"
+classes[0].fields[18].type "jsonstring"
+classes[0].fields[19].name "ap2"
+classes[0].fields[19].type "jsonstring"
+classes[0].fields[20].name "wp1"
+classes[0].fields[20].type "jsonstring"
+classes[0].fields[21].name "wp2"
+classes[0].fields[21].type "jsonstring"
+classes[0].fields[22].name "araw"
+classes[0].fields[22].type "jsonstring"
+classes[0].fields[23].name "wraw"
+classes[0].fields[23].type "jsonstring"
+classes[1].name "class1"
+classes[1].id 1
+classes[1].fields[1]
+classes[1].fields[0].name "a"
+classes[1].fields[0].type "integer"
+classes[2].name "class2"
+classes[2].id 2
+classes[2].fields[2]
+classes[2].fields[0].name "aa"
+classes[2].fields[0].type "integer"
+classes[2].fields[1].name "ab"
+classes[2].fields[1].type "integer"
+classes[3].name "class3"
+classes[3].id 3
+classes[3].fields[10]
+classes[3].fields[0].name "ba"
+classes[3].fields[0].type "integer"
+classes[3].fields[1].name "bb"
+classes[3].fields[1].type "float"
+classes[3].fields[2].name "bc"
+classes[3].fields[2].type "longstring"
+classes[3].fields[3].name "bd"
+classes[3].fields[3].type "jsonstring"
+classes[3].fields[4].name "be"
+classes[3].fields[4].type "jsonstring"
+classes[3].fields[5].name "bf"
+classes[3].fields[5].type "jsonstring"
+classes[3].fields[6].name "bg"
+classes[3].fields[6].type "jsonstring"
+classes[3].fields[7].name "bh"
+classes[3].fields[7].type "jsonstring"
+classes[3].fields[8].name "bi"
+classes[3].fields[8].type "jsonstring"
+classes[3].fields[9].name "bj"
+classes[3].fields[9].type "jsonstring"
+classes[4].name "class4"
+classes[4].id 4
+classes[4].fields[1]
+classes[4].fields[0].name "documentid"
+classes[4].fields[0].type "longstring"
+classes[5].id 5
+classes[5].name "class5"
+classes[5].fields[6]
+classes[5].fields[0].name "sp2"
+classes[5].fields[0].type "string"
+classes[5].fields[1].name "sp2x"
+classes[5].fields[1].type "xmlstring"
+classes[5].fields[2].name "ap2"
+classes[5].fields[2].type "jsonstring"
+classes[5].fields[3].name "ap2x"
+classes[5].fields[3].type "xmlstring"
+classes[5].fields[4].name "wp2"
+classes[5].fields[4].type "jsonstring"
+classes[5].fields[5].name "wp2x"
+classes[5].fields[5].type "xmlstring"
diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
new file mode 100644
index 00000000000..f2e5f1a508b
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
@@ -0,0 +1,713 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for summaryfieldconverter.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("summaryfieldconverter_test");
+
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/document/annotation/span.h>
+#include <vespa/document/annotation/spanlist.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/annotationtype.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/document/datatype/urldatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+#include <vespa/document/fieldvalue/shortfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+#include <vespa/document/predicate/predicate.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/proton/docsummary/summaryfieldconverter.h>
+#include <vespa/searchcore/proton/docsummary/linguisticsannotation.h>
+#include <vespa/searchcore/proton/docsummary/searchdatatype.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/config-summarymap.h>
+#include <vespa/vespalib/geo/zcurve.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/data/slime/json_format.h>
+#include <vespa/vespalib/data/slime/binary_format.h>
+#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
+#include <vespa/vespalib/tensor/tensor.h>
+#include <vespa/vespalib/tensor/types.h>
+#include <vespa/vespalib/tensor/default_tensor.h>
+#include <vespa/vespalib/tensor/tensor_factory.h>
+
+using vespa::config::search::SummarymapConfig;
+using vespa::config::search::SummarymapConfigBuilder;
+using document::Annotation;
+using document::AnnotationType;
+using document::ArrayDataType;
+using document::ArrayFieldValue;
+using document::ByteFieldValue;
+using document::DataType;
+using document::Document;
+using document::DocumenttypesConfig;
+using document::DocumenttypesConfigBuilder;
+using document::DocumentId;
+using document::DocumentType;
+using document::DocumentTypeRepo;
+using document::DoubleFieldValue;
+using document::FeatureSet;
+using document::Field;
+using document::FieldNotFoundException;
+using document::FieldValue;
+using document::FloatFieldValue;
+using document::IntFieldValue;
+using document::LongFieldValue;
+using document::Predicate;
+using document::PredicateFieldValue;
+using document::RawFieldValue;
+using document::ShortFieldValue;
+using document::Span;
+using document::SpanList;
+using document::SpanTree;
+using document::StringFieldValue;
+using document::StructDataType;
+using document::StructFieldValue;
+using document::UrlDataType;
+using document::WeightedSetDataType;
+using document::WeightedSetFieldValue;
+using document::TensorFieldValue;
+using search::index::Schema;
+using vespalib::Slime;
+using vespalib::slime::Cursor;
+using vespalib::string;
+using namespace proton;
+using namespace proton::linguistics;
+using vespalib::geo::ZCurve;
+using vespalib::tensor::Tensor;
+using vespalib::tensor::TensorCells;
+using vespalib::tensor::TensorDimensions;
+
+typedef SummaryFieldConverter SFC;
+
+namespace {
+
+struct FieldBlock {
+    vespalib::string input;
+    Slime slime;
+    search::RawBuf binary;
+    vespalib::string json;
+
+    explicit FieldBlock(const vespalib::string &jsonInput)
+        : input(jsonInput), slime(), binary(1024), json()
+    {
+        size_t used = vespalib::slime::JsonFormat::decode(jsonInput, slime);
+        EXPECT_EQUAL(jsonInput.size(), used);
+        {
+            search::SlimeOutputRawBufAdapter adapter(binary);
+            vespalib::slime::JsonFormat::encode(slime, adapter, true);
+            json.assign(binary.GetDrainPos(), binary.GetUsedLen());
+            binary.reset();
+        }
+        search::SlimeOutputRawBufAdapter adapter(binary);
+        vespalib::slime::BinaryFormat::encode(slime, adapter);
+    }
+};
+
+class Test : public vespalib::TestApp {
+    std::unique_ptr<Schema> _schema;
+    std::unique_ptr<SummarymapConfigBuilder> _summarymap;
+    DocumentTypeRepo::SP      _documentRepo;
+    const DocumentType       *_documentType;
+    document::FixedTypeRepo   _fixedRepo;
+
+    void setUp();
+    void tearDown();
+
+    const DataType &getDataType(const string &name) const;
+
+    template <typename T>
+    T getValueAs(const string &field_name, const Document &doc);
+
+    template <typename T>
+    T
+    cvtValueAs(const FieldValue::UP &fv);
+
+    template <typename T>
+    T
+    cvtAttributeAs(const FieldValue::UP &fv);
+
+    template <typename T>
+    T
+    cvtSummaryAs(bool markup, const FieldValue::UP &fv);
+
+    void checkString(const string &str, const FieldValue *value);
+    void checkData(const search::RawBuf &data, const FieldValue *value);
+    void checkArray(const string &str, const FieldValue *value);
+    template <unsigned int N>
+    void checkArray(const char *(&str)[N], const FieldValue *value);
+    Document getDoc(const string &name, const Document *doc);
+    void setIndexField(const string &name);
+    void setSummaryField(const string &name);
+    void setAttributeField(const string &name);
+
+    void requireThatSummaryIsAnUnmodifiedString();
+    void requireThatAttributeIsAnUnmodifiedString();
+    void requireThatArrayIsFlattenedInSummaryField();
+    void requireThatWeightedSetIsFlattenedInSummaryField();
+    void requireThatPositionsAreTransformedInSummary();
+    void requireThatArrayIsPreservedInAttributeField();
+    void requireThatPositionsAreTransformedInAttributeField();
+    void requireThatPositionArrayIsTransformedInAttributeField();
+    void requireThatPositionWeightedSetIsTransformedInAttributeField();
+    void requireThatAttributeCanBePrimitiveTypes();
+    void requireThatSummaryCanBePrimitiveTypes();
+    void requireThatSummaryHandlesCjk();
+    void requireThatSearchDataTypeUsesDefaultDataTypes();
+    void requireThatLinguisticsAnnotationUsesDefaultDataTypes();
+    void requireThatPredicateIsPrinted();
+    void requireThatTensorIsPrinted();
+    const DocumentType &getDocType() const { return *_documentType; }
+    Document makeDocument();
+    StringFieldValue annotateTerm(const string &term);
+    StringFieldValue makeAnnotatedChineseString();
+    StringFieldValue makeAnnotatedString();
+    void setSpanTree(StringFieldValue & value, SpanTree::UP tree);
+public:
+    Test();
+    int Main();
+};
+
+DocumenttypesConfig getDocumenttypesConfig() {
+    using namespace document::config_builder;
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(42, "indexingdocument",
+                     Struct("indexingdocument.header")
+                     .addField("empty", DataType::T_STRING)
+                     .addField("string", DataType::T_STRING)
+                     .addField("plain_string", DataType::T_STRING)
+                     .addField("string_array", Array(DataType::T_STRING))
+                     .addField("string_wset", Wset(DataType::T_STRING))
+                     .addField("position1", DataType::T_INT)
+                     .addField("position2", DataType::T_LONG)
+                     .addField("position2_array", Array(DataType::T_LONG))
+                     .addField("position2_wset", Wset(DataType::T_LONG))
+                     .addField("uri", UrlDataType::getInstance().getId())
+                     .addField("uri_array",
+                               Array(UrlDataType::getInstance().getId()))
+                     .addField("int", DataType::T_INT)
+                     .addField("long", DataType::T_LONG)
+                     .addField("short", DataType::T_SHORT)
+                     .addField("byte", DataType::T_BYTE)
+                     .addField("double", DataType::T_DOUBLE)
+                     .addField("float", DataType::T_FLOAT)
+                     .addField("chinese", DataType::T_STRING)
+                     .addField("predicate", DataType::T_PREDICATE)
+                     .addField("tensor", DataType::T_TENSOR),
+                     Struct("indexingdocument.body"));
+    return builder.config();
+}
+
+Test::Test() :
+    _documentRepo(new DocumentTypeRepo(getDocumenttypesConfig())),
+    _documentType(_documentRepo->getDocumentType("indexingdocument")),
+    _fixedRepo(*_documentRepo, *_documentType)
+{
+    ASSERT_TRUE(_documentType);
+}
+
+#define TEST_CALL(func) \
+    TEST_DO(setUp()); \
+    TEST_DO(func); \
+    TEST_DO(tearDown())
+
+int
+Test::Main()
+{
+    TEST_INIT("summaryfieldconverter_test");
+
+    TEST_CALL(requireThatSummaryIsAnUnmodifiedString());
+    TEST_CALL(requireThatAttributeIsAnUnmodifiedString());
+    TEST_CALL(requireThatArrayIsFlattenedInSummaryField());
+    TEST_CALL(requireThatWeightedSetIsFlattenedInSummaryField());
+    TEST_CALL(requireThatPositionsAreTransformedInSummary());
+    TEST_CALL(requireThatArrayIsPreservedInAttributeField());
+    TEST_CALL(requireThatPositionsAreTransformedInAttributeField());
+    TEST_CALL(requireThatPositionArrayIsTransformedInAttributeField());
+    TEST_CALL(requireThatPositionWeightedSetIsTransformedInAttributeField());
+    TEST_CALL(requireThatAttributeCanBePrimitiveTypes());
+    TEST_CALL(requireThatSummaryCanBePrimitiveTypes());
+    TEST_CALL(requireThatSummaryHandlesCjk());
+    TEST_CALL(requireThatSearchDataTypeUsesDefaultDataTypes());
+    TEST_CALL(requireThatLinguisticsAnnotationUsesDefaultDataTypes());
+    TEST_CALL(requireThatPredicateIsPrinted());
+    TEST_CALL(requireThatTensorIsPrinted());
+
+    TEST_DONE();
+}
+
+void Test::setUp() {
+    _schema.reset(new Schema);
+    _summarymap.reset(new SummarymapConfigBuilder);
+}
+
+void Test::tearDown() {
+}
+
+const DataType &Test::getDataType(const string &name) const {
+    const DataType *type = _documentRepo->getDataType(*_documentType, name);
+    ASSERT_TRUE(type);
+    return *type;
+}
+
+template <typename T>
+std::unique_ptr<T> makeUP(T *p) { return std::unique_ptr<T>(p); }
+
+StringFieldValue Test::makeAnnotatedString() {
+    SpanList *span_list = new SpanList;
+    SpanTree::UP tree(new SpanTree(SPANTREE_NAME, makeUP(span_list)));
+    // Annotations don't have to be added sequentially.
+    tree->annotate(span_list->add(makeUP(new Span(8, 3))),
+                   makeUP(new Annotation(*TERM,
+                                         makeUP(new StringFieldValue(
+                                                         "Annotation")))));
+    tree->annotate(span_list->add(makeUP(new Span(0, 3))), *TERM);
+    tree->annotate(span_list->add(makeUP(new Span(4, 3))), *TERM);
+    tree->annotate(span_list->add(makeUP(new Span(4, 3))),
+                   makeUP(new Annotation(*TERM,
+                                         makeUP(new StringFieldValue(
+                                                         "Multiple")))));
+    tree->annotate(span_list->add(makeUP(new Span(1, 2))),
+                   makeUP(new Annotation(*TERM,
+                                         makeUP(new StringFieldValue(
+                                                         "Overlap")))));
+    StringFieldValue value("Foo Bar Baz");
+    setSpanTree(value, std::move(tree));
+    return value;
+}
+
+StringFieldValue Test::annotateTerm(const string &term) {
+    SpanTree::UP tree(new SpanTree(SPANTREE_NAME, makeUP(new Span(0, term.size()))));
+    tree->annotate(tree->getRoot(), *TERM);
+    StringFieldValue value(term);
+    setSpanTree(value, std::move(tree));
+    return value;
+}
+
+void Test::setSpanTree(StringFieldValue & value, SpanTree::UP tree) {
+    StringFieldValue::SpanTrees trees;
+    trees.push_back(std::move(tree));
+    value.setSpanTrees(trees, _fixedRepo);
+}
+
+StringFieldValue Test::makeAnnotatedChineseString() {
+    SpanList *span_list = new SpanList;
+    SpanTree::UP tree(new SpanTree(SPANTREE_NAME, makeUP(span_list)));
+    // These chinese characters each use 3 bytes in their UTF8 encoding.
+    tree->annotate(span_list->add(makeUP(new Span(0, 15))), *TERM);
+    tree->annotate(span_list->add(makeUP(new Span(15, 9))), *TERM);
+    StringFieldValue value("我就是那个大灰狼");
+    setSpanTree(value, std::move(tree));
+    return value;
+}
+
+Document Test::makeDocument() {
+    Document doc(getDocType(), DocumentId("doc:scheme:"));
+    doc.setRepo(*_documentRepo);
+    doc.setValue("string", makeAnnotatedString());
+
+    doc.setValue("plain_string", StringFieldValue("Plain"));
+
+    ArrayFieldValue array(getDataType("Array<String>"));
+    array.add(annotateTerm("\"foO\""));
+    array.add(annotateTerm("ba\\R"));
+    doc.setValue("string_array", array);
+
+    WeightedSetFieldValue wset(getDataType("WeightedSet<String>"));
+    wset.add(annotateTerm("\"foo\""), 2);
+    wset.add(annotateTerm("ba\\r"), 4);
+    doc.setValue("string_wset", wset);
+
+    doc.setValue("position1", IntFieldValue(5));
+
+    doc.setValue("position2", LongFieldValue(ZCurve::encode(4, 2)));
+
+    StructFieldValue uri(getDataType("url"));
+    uri.setValue("all", annotateTerm("http://www.yahoo.com:42/foobar?q#frag"));
+    uri.setValue("scheme", annotateTerm("http"));
+    uri.setValue("host", annotateTerm("www.yahoo.com"));
+    uri.setValue("port", annotateTerm("42"));
+    uri.setValue("path", annotateTerm("foobar"));
+    uri.setValue("query", annotateTerm("q"));
+    uri.setValue("fragment", annotateTerm("frag"));
+    doc.setValue("uri", uri);
+
+    ArrayFieldValue uri_array(getDataType("Array<url>"));
+    uri.setValue("all", annotateTerm("http://www.yahoo.com:80/foobar?q#frag"));
+    uri.setValue("port", annotateTerm("80"));
+    uri_array.add(uri);
+    uri.setValue("all", annotateTerm("https://www.yahoo.com:443/foo?q#frag"));
+    uri.setValue("scheme", annotateTerm("https"));
+    uri.setValue("path", annotateTerm("foo"));
+    uri.setValue("port", annotateTerm("443"));
+    uri_array.add(uri);
+    doc.setValue("uri_array", uri_array);
+
+    ArrayFieldValue position2_array(getDataType("Array<Long>"));
+    position2_array.add(LongFieldValue(ZCurve::encode(4, 2)));
+    position2_array.add(LongFieldValue(ZCurve::encode(4, 4)));
+    doc.setValue("position2_array", position2_array);
+
+    WeightedSetFieldValue position2_wset(getDataType("WeightedSet<Long>"));
+    position2_wset.add(LongFieldValue(ZCurve::encode(4, 2)), 4);
+    position2_wset.add(LongFieldValue(ZCurve::encode(4, 4)), 2);
+    doc.setValue("position2_wset", position2_wset);
+
+    doc.setValue("int", IntFieldValue(42));
+    doc.setValue("long", LongFieldValue(84));
+    doc.setValue("short", ShortFieldValue(21));
+    doc.setValue("byte", ByteFieldValue(11));
+    doc.setValue("double", DoubleFieldValue(0.4));
+    doc.setValue("float", FloatFieldValue(0.2f));
+
+    doc.setValue("chinese", makeAnnotatedChineseString());
+    return doc;
+}
+
+template <typename T>
+T Test::getValueAs(const string &field_name, const Document &doc) {
+    FieldValue::UP fv(doc.getValue(field_name));
+    const T *value = dynamic_cast<const T *>(fv.get());
+    ASSERT_TRUE(value);
+    return *value;
+}
+
+template <typename T>
+T
+Test::cvtValueAs(const FieldValue::UP &fv)
+{
+    ASSERT_TRUE(fv.get() != NULL);
+    const T *value = dynamic_cast<const T *>(fv.get());
+    ASSERT_TRUE(value);
+    return *value;
+}
+
+template <typename T>
+T
+Test::cvtAttributeAs(const FieldValue::UP &fv)
+{
+    ASSERT_TRUE(fv.get() != NULL);
+    return cvtValueAs<T>(fv);
+}
+
+template <typename T>
+T
+Test::cvtSummaryAs(bool markup, const FieldValue::UP &fv)
+{
+    ASSERT_TRUE(fv.get() != NULL);
+    FieldValue::UP r = SFC::convertSummaryField(markup, *fv, false);
+    return cvtValueAs<T>(r);
+}
+
+void Test::checkString(const string &str, const FieldValue *value) {
+    ASSERT_TRUE(value);
+    const StringFieldValue *s = dynamic_cast<const StringFieldValue *>(value);
+    ASSERT_TRUE(s);
+    // fprintf(stderr, ">>>%s<<<   >>>%s<<<\n", str.c_str(), s->getValue().c_str());
+    EXPECT_EQUAL(str, s->getValue());
+}
+
+void Test::checkData(const search::RawBuf &buf, const FieldValue *value) {
+    ASSERT_TRUE(value);
+    const RawFieldValue *s = dynamic_cast<const RawFieldValue *>(value);
+    ASSERT_TRUE(s);
+    auto got = s->getAsRaw();
+    EXPECT_EQUAL(buf.GetUsedLen(), got.second);
+    EXPECT_TRUE(memcmp(buf.GetDrainPos(), got.first, got.second) == 0);
+}
+
+void Test::checkArray(const string &str, const FieldValue *value) {
+    ASSERT_TRUE(value);
+    const ArrayFieldValue *a = dynamic_cast<const ArrayFieldValue *>(value);
+    ASSERT_TRUE(a);
+    EXPECT_EQUAL(1u, a->size());
+    checkString(str, &(*a)[0]);
+}
+
+template <unsigned int N>
+void Test::checkArray(const char *(&str)[N], const FieldValue *value) {
+    ASSERT_TRUE(value);
+    const ArrayFieldValue *a = dynamic_cast<const ArrayFieldValue *>(value);
+    ASSERT_TRUE(a);
+    EXPECT_EQUAL(N, a->size());
+    for (size_t i = 0; i < a->size() && i < N; ++i) {
+        checkString(str[i], &(*a)[i]);
+    }
+}
+
+Document Test::getDoc(const string &name, const Document *doc) {
+    ASSERT_TRUE(doc);
+    return getValueAs<Document>(name, *doc);
+}
+
+void Test::setIndexField(const string &field) {
+    _schema->addIndexField(
+            Schema::IndexField(field, Schema::STRING));
+}
+
+void Test::setSummaryField(const string &field) {
+    _schema->addSummaryField(Schema::Field(field, Schema::STRING));
+}
+
+void Test::setAttributeField(const string &field) {
+    _schema->addAttributeField(Schema::Field(field, Schema::STRING));
+}
+
+void Test::requireThatSummaryIsAnUnmodifiedString() {
+    setSummaryField("string");
+    Document summary = makeDocument();
+    checkString("Foo Bar Baz", SFC::convertSummaryField(false,
+                                                        *summary.getValue("string"),
+                                                        false).get());
+}
+
+void Test::requireThatAttributeIsAnUnmodifiedString() {
+    setAttributeField("string");
+    Document attribute = makeDocument();
+    checkString("Foo Bar Baz",
+                attribute.getValue("string").get());
+}
+
+void Test::requireThatArrayIsFlattenedInSummaryField() {
+    setSummaryField("string_array");
+    Document summary = makeDocument();
+    FieldBlock expect("[\"\\\"foO\\\"\",\"ba\\\\R\"]");
+    checkString(expect.json,
+                SFC::convertSummaryField(false,
+                                         *summary.getValue("string_array"),
+                                         false).get());
+    checkData(expect.binary,
+              SFC::convertSummaryField(false,
+                                       *summary.getValue("string_array"),
+                                       true).get());
+}
+
+void Test::requireThatWeightedSetIsFlattenedInSummaryField() {
+    setSummaryField("string_wset");
+    Document summary = makeDocument();
+    FieldBlock expect("[{\"item\":\"\\\"foo\\\"\",\"weight\":2},{\"item\":\"ba\\\\r\",\"weight\":4}]");
+    checkString(expect.json,
+                SFC::convertSummaryField(false,
+                                         *summary.getValue("string_wset"),
+                                         false).get());
+    checkData(expect.binary,
+              SFC::convertSummaryField(false,
+                                       *summary.getValue("string_wset"),
+                                       true).get());
+}
+
+void Test::requireThatPositionsAreTransformedInSummary() {
+    setSummaryField("position1");
+    setSummaryField("position2");
+    Document summary = makeDocument();
+    FieldValue::UP fv = summary.getValue("position1");
+    EXPECT_EQUAL(5, cvtSummaryAs<IntFieldValue>(false, fv).getValue());
+    FieldValue::UP fv2 = summary.getValue("position2");
+    EXPECT_EQUAL(24, cvtSummaryAs<LongFieldValue>(false, fv2).getValue());
+}
+
+void Test::requireThatArrayIsPreservedInAttributeField() {
+    setAttributeField("string_array");
+    Document attribute = makeDocument();
+    const char *array[] = { "\"foO\"", "ba\\R" };
+    checkArray(array,
+               attribute.getValue("string_array").get());
+}
+
+void Test::requireThatPositionsAreTransformedInAttributeField() {
+    setAttributeField("position1");
+    setAttributeField("position2");
+    Document attr = makeDocument();
+    FieldValue::UP fv = attr.getValue("position1");
+    EXPECT_EQUAL(5, cvtAttributeAs<IntFieldValue>(fv).getValue());
+    fv = attr.getValue("position2");
+    EXPECT_EQUAL(24, cvtAttributeAs<LongFieldValue>(fv).getValue());
+}
+
+void Test::requireThatPositionArrayIsTransformedInAttributeField() {
+    setAttributeField("position2_array");
+    Document attr = makeDocument();
+    FieldValue::UP fv = attr.getValue("position2_array");
+    ArrayFieldValue a = cvtAttributeAs<ArrayFieldValue>(fv);
+    EXPECT_EQUAL(2u, a.size());
+    EXPECT_EQUAL(24, dynamic_cast<LongFieldValue &>(a[0]).getValue());
+    EXPECT_EQUAL(48, dynamic_cast<LongFieldValue &>(a[1]).getValue());
+}
+
+void Test::requireThatPositionWeightedSetIsTransformedInAttributeField() {
+    setAttributeField("position2_wset");
+    Document attr = makeDocument();
+    FieldValue::UP fv = attr.getValue("position2_wset");
+    WeightedSetFieldValue w = cvtAttributeAs<WeightedSetFieldValue>(fv);
+    EXPECT_EQUAL(2u, w.size());
+    WeightedSetFieldValue::iterator it = w.begin();
+    EXPECT_EQUAL(24, dynamic_cast<const LongFieldValue&>(*it->first).getValue());
+    EXPECT_EQUAL(4, dynamic_cast<IntFieldValue &>(*it->second).getValue());
+    ++it;
+    EXPECT_EQUAL(48, dynamic_cast<const LongFieldValue&>(*it->first).getValue());
+    EXPECT_EQUAL(2, dynamic_cast<IntFieldValue &>(*it->second).getValue());
+}
+
+void Test::requireThatAttributeCanBePrimitiveTypes() {
+    setAttributeField("int");
+    setAttributeField("long");
+    setAttributeField("short");
+    setAttributeField("byte");
+    setAttributeField("double");
+    setAttributeField("float");
+    Document attribute = makeDocument();
+    FieldValue::UP fv = attribute.getValue("int");
+    EXPECT_EQUAL(42, cvtAttributeAs<IntFieldValue>(fv).getValue());
+    fv = attribute.getValue("long");
+    EXPECT_EQUAL(84, cvtAttributeAs<LongFieldValue>(fv).getValue());
+    fv = attribute.getValue("short");
+    EXPECT_EQUAL(21, cvtAttributeAs<ShortFieldValue>(fv).getValue());
+    fv = attribute.getValue("byte");
+    EXPECT_EQUAL(11, cvtAttributeAs<ByteFieldValue>(fv).getValue());
+    fv = attribute.getValue("double");
+    EXPECT_EQUAL(0.4, cvtAttributeAs<DoubleFieldValue>(fv).getValue());
+    fv = attribute.getValue("float");
+    EXPECT_EQUAL(0.2f, cvtAttributeAs<FloatFieldValue>(fv).getValue());
+}
+
+void Test::requireThatSummaryCanBePrimitiveTypes() {
+    setSummaryField("int");
+    setSummaryField("long");
+    setSummaryField("short");
+    setSummaryField("byte");
+    setSummaryField("double");
+    setSummaryField("float");
+    Document summary = makeDocument();
+    FieldValue::UP fv = summary.getValue("int");
+    EXPECT_EQUAL(42, cvtSummaryAs<IntFieldValue>(false, fv).getValue());
+    fv = summary.getValue("long");
+    EXPECT_EQUAL(84, cvtSummaryAs<LongFieldValue>(false, fv).getValue());
+    fv = summary.getValue("short");
+    EXPECT_EQUAL(21, cvtSummaryAs<ShortFieldValue>(false, fv).getValue());
+    fv = summary.getValue("byte");
+    EXPECT_EQUAL(11, cvtSummaryAs<ShortFieldValue>(false, fv).getValue());
+    fv = summary.getValue("double");
+    EXPECT_EQUAL(0.4, cvtSummaryAs<DoubleFieldValue>(false, fv).getValue());
+    fv = summary.getValue("float");
+    EXPECT_EQUAL(0.2f, cvtSummaryAs<FloatFieldValue>(false, fv).getValue());
+}
+
+void Test::requireThatSummaryHandlesCjk() {
+    Document summary = makeDocument();
+    FieldValue::UP fv = summary.getValue("chinese");
+    EXPECT_EQUAL("我就是那个\037大灰狼\037",
+                 cvtSummaryAs<StringFieldValue>(true, fv).getValue());
+}
+
+void Test::requireThatSearchDataTypeUsesDefaultDataTypes() {
+    const StructDataType *uri =
+        dynamic_cast<const StructDataType *>(SearchDataType::URI);
+    ASSERT_TRUE(uri);
+    ASSERT_TRUE(uri->hasField("all"));
+    ASSERT_TRUE(uri->hasField("scheme"));
+    ASSERT_TRUE(uri->hasField("host"));
+    ASSERT_TRUE(uri->hasField("port"));
+    ASSERT_TRUE(uri->hasField("path"));
+    ASSERT_TRUE(uri->hasField("query"));
+    ASSERT_TRUE(uri->hasField("fragment"));
+    EXPECT_EQUAL(*DataType::STRING, uri->getField("all").getDataType());
+    EXPECT_EQUAL(*DataType::STRING, uri->getField("scheme").getDataType());
+    EXPECT_EQUAL(*DataType::STRING, uri->getField("host").getDataType());
+    EXPECT_EQUAL(*DataType::STRING, uri->getField("port").getDataType());
+    EXPECT_EQUAL(*DataType::STRING, uri->getField("path").getDataType());
+    EXPECT_EQUAL(*DataType::STRING, uri->getField("query").getDataType());
+    EXPECT_EQUAL(*DataType::STRING, uri->getField("fragment").getDataType());
+}
+
+void Test::requireThatLinguisticsAnnotationUsesDefaultDataTypes() {
+    EXPECT_EQUAL(*AnnotationType::TERM, *linguistics::TERM);
+    ASSERT_TRUE(AnnotationType::TERM->getDataType());
+    ASSERT_TRUE(linguistics::TERM->getDataType());
+    EXPECT_EQUAL(*AnnotationType::TERM->getDataType(),
+                 *linguistics::TERM->getDataType());
+}
+
+void
+Test::requireThatPredicateIsPrinted()
+{
+    std::unique_ptr<Slime> input(new Slime());
+    Cursor &obj = input->setObject();
+    obj.setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_SET);
+    obj.setString(Predicate::KEY, "foo");
+    Cursor &arr = obj.setArray(Predicate::SET);
+    arr.addString("bar");
+
+    Document doc(getDocType(), DocumentId("doc:scheme:"));
+    doc.setRepo(*_documentRepo);
+    doc.setValue("predicate", PredicateFieldValue(std::move(input)));
+
+    checkString("'foo' in ['bar']\n",
+                SFC::convertSummaryField(false, *doc.getValue("predicate"), false).get());
+}
+
+
+Tensor::UP
+createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
+    vespalib::tensor::DefaultTensor::builder builder;
+    return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+}
+
+void
+Test::requireThatTensorIsPrinted()
+{
+    TensorFieldValue tensorFieldValue;
+    tensorFieldValue = createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} },
+                                    {"x", "y"});
+    Document doc(getDocType(), DocumentId("doc:scheme:"));
+    doc.setRepo(*_documentRepo);
+    doc.setValue("tensor", tensorFieldValue);
+
+    FieldBlock expect1("{ dimensions: [ 'x', 'y' ], cells: ["
+                       "{ address: { x:'4', y:'5' }, value: 7.0 }"
+                       "] }");
+
+    TEST_CALL(checkString(expect1.json,
+                          SFC::convertSummaryField(false,
+                                                   *doc.getValue("tensor"),
+                                                   false).get()));
+    TEST_CALL(checkData(expect1.binary,
+                        SFC::convertSummaryField(false,
+                                                 *doc.getValue("tensor"),
+                                                 true).get()));
+    doc.setValue("tensor", TensorFieldValue());
+
+    FieldBlock expect2("{ }");
+
+    TEST_CALL(checkString(expect2.json,
+                          SFC::convertSummaryField(false,
+                                                   *doc.getValue("tensor"),
+                                                   false).get()));
+    TEST_CALL(checkData(expect2.binary,
+                        SFC::convertSummaryField(false,
+                                                 *doc.getValue("tensor"),
+                                                 true).get()));
+}
+
+}  // namespace
+
+TEST_APPHOOK(Test);
diff --git a/searchcore/src/tests/proton/docsummary/summarymap.cfg b/searchcore/src/tests/proton/docsummary/summarymap.cfg
new file mode 100644
index 00000000000..f2d429b1412
--- /dev/null
+++ b/searchcore/src/tests/proton/docsummary/summarymap.cfg
@@ -0,0 +1,48 @@
+override[16]
+override[0].field "aa"
+override[0].command "copy"
+override[0].arguments "ab"
+override[1].field "ab"
+override[1].command "empty"
+override[2].field "ba"
+override[2].command "attribute"
+override[2].arguments "ba"
+override[3].field "bb"
+override[3].command "attribute"
+override[3].arguments "bb"
+override[4].field "bc"
+override[4].command "attribute"
+override[4].arguments "bc"
+override[5].field "bd"
+override[5].command "attribute"
+override[5].arguments "bd"
+override[6].field "be"
+override[6].command "attribute"
+override[6].arguments "be"
+override[7].field "bf"
+override[7].command "attribute"
+override[7].arguments "bf"
+override[8].field "bg"
+override[8].command "attribute"
+override[8].arguments "bg"
+override[9].field "bh"
+override[9].command "attribute"
+override[9].arguments "bh"
+override[10].field "bi"
+override[10].command "attribute"
+override[10].arguments "bi"
+override[11].field "dynamicstring"
+override[11].command "dynamicteaser"
+override[11].arguments "dynamicstring"
+override[12].field "sp2x"
+override[12].command "positions"
+override[12].arguments "sp2"
+override[13].field "ap2x"
+override[13].command "positions"
+override[13].arguments "ap2"
+override[14].field "wp2x"
+override[14].command "positions"
+override[14].arguments "wp2"
+override[15].field "bj"
+override[15].command "attribute"
+override[15].arguments "bj"
diff --git a/searchcore/src/tests/proton/document_iterator/.gitignore b/searchcore/src/tests/proton/document_iterator/.gitignore
new file mode 100644
index 00000000000..323a5d517ba
--- /dev/null
+++ b/searchcore/src/tests/proton/document_iterator/.gitignore
@@ -0,0 +1 @@
+searchcore_document_iterator_test_app
diff --git a/searchcore/src/tests/proton/document_iterator/CMakeLists.txt b/searchcore/src/tests/proton/document_iterator/CMakeLists.txt
new file mode 100644
index 00000000000..03d910ef02e
--- /dev/null
+++ b/searchcore/src/tests/proton/document_iterator/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_document_iterator_test_app
+    SOURCES
+    document_iterator_test.cpp
+    DEPENDS
+    searchcore_persistenceengine
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_document_iterator_test_app COMMAND searchcore_document_iterator_test_app)
diff --git a/searchcore/src/tests/proton/document_iterator/FILES b/searchcore/src/tests/proton/document_iterator/FILES
new file mode 100644
index 00000000000..351464d9f46
--- /dev/null
+++ b/searchcore/src/tests/proton/document_iterator/FILES
@@ -0,0 +1 @@
+document_iterator_test.cpp
diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp
new file mode 100644
index 00000000000..e0d92cd2a1a
--- /dev/null
+++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp
@@ -0,0 +1,888 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/fieldset/fieldsets.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/persistence/spi/bucket.h>
+#include <vespa/persistence/spi/docentry.h>
+#include <vespa/persistence/spi/result.h>
+#include <persistence/spi/types.h>
+#include <vespa/searchcore/proton/persistenceengine/document_iterator.h>
+#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchlib/attribute/attributecontext.h>
+#include <vespa/searchcore/proton/common/attrupdate.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchcore/proton/server/commit_and_wait_document_retriever.h>
+
+using document::DocumentType;
+using document::Field;
+using namespace proton;
+
+using document::DataType;
+using document::Document;
+using document::DocumentId;
+using document::BucketId;
+using document::IntFieldValue;
+using document::DoubleFieldValue;
+using document::StringFieldValue;
+using search::DocumentIdT;
+using search::DocumentMetaData;
+using search::AttributeContext;
+using search::AttributeEnumGuard;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::attribute::BasicType;
+using search::attribute::CollectionType;
+using search::attribute::Config;
+using search::attribute::IAttributeContext;
+using search::index::Schema;
+using storage::spi::Timestamp;
+using storage::spi::Bucket;
+using storage::spi::PartitionId;
+using storage::spi::IterateResult;
+using storage::spi::DocEntry;
+using storage::spi::Selection;
+using storage::spi::DocumentSelection;
+using storage::spi::IncludedVersions;
+
+const uint64_t largeNum = 10000000;
+
+Bucket bucket(size_t x) {
+    return Bucket(BucketId(x), PartitionId(0));
+}
+
+Selection selectAll() {
+    return Selection(DocumentSelection(""));
+}
+
+Selection selectTimestampRange(uint64_t min, uint64_t max) {
+    Selection sel(DocumentSelection(""));
+    sel.setFromTimestamp(Timestamp(min));
+    sel.setToTimestamp(Timestamp(max));
+    return sel;
+}
+
+Selection selectTimestampSet(uint64_t a, uint64_t b, uint64_t c) {
+    Selection sel(DocumentSelection(""));
+    Selection::TimestampSubset subset;
+    subset.push_back(Timestamp(a));
+    subset.push_back(Timestamp(b));
+    subset.push_back(Timestamp(c));
+    sel.setTimestampSubset(subset);
+    return sel;
+}
+
+Selection selectDocs(const std::string &docSel) {
+    return Selection(DocumentSelection(docSel));
+}
+
+Selection selectDocsWithinRange(const std::string &docSel, uint64_t min, uint64_t max) {
+    Selection sel((DocumentSelection(docSel)));
+    sel.setFromTimestamp(Timestamp(min));
+    sel.setToTimestamp(Timestamp(max));
+    return sel;
+}
+
+IncludedVersions docV() {
+    return storage::spi::NEWEST_DOCUMENT_ONLY;
+}
+
+IncludedVersions newestV() {
+    return storage::spi::NEWEST_DOCUMENT_OR_REMOVE;
+}
+
+IncludedVersions allV() {
+    return storage::spi::ALL_VERSIONS;
+}
+
+struct UnitDR : DocumentRetrieverBaseForTest {
+    static DocumentIdT _docidCnt;
+
+    document::DocumentTypeRepo repo;
+    document::Document::UP     document;
+    Timestamp                  timestamp;
+    Bucket                     bucket;
+    bool                       removed;
+    DocumentIdT                docid;
+
+    UnitDR()
+        : repo(), document(new Document(*DataType::DOCUMENT, DocumentId())),
+          timestamp(0), bucket(), removed(false), docid(0) {}
+    UnitDR(document::Document::UP d, Timestamp t, Bucket b, bool r)
+        : repo(), document(std::move(d)), timestamp(t), bucket(b), removed(r), docid(++_docidCnt) {}
+    UnitDR(const document::DocumentType &dt, document::Document::UP d, Timestamp t, Bucket b, bool r)
+        : repo(dt), document(std::move(d)), timestamp(t), bucket(b), removed(r), docid(++_docidCnt) {}
+
+    const document::DocumentTypeRepo &getDocumentTypeRepo() const override {
+        return repo;
+    }
+    void getBucketMetaData(const Bucket &b, DocumentMetaData::Vector &result) const override
+    {
+        if (b == bucket) {
+            result.push_back(DocumentMetaData(docid, timestamp, bucket, document->getId().getGlobalId(), removed));
+        }
+    }
+    DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const override {
+        if (document->getId() == id) {
+            return DocumentMetaData(docid, timestamp, bucket, document->getId().getGlobalId(), removed);
+        }
+        return DocumentMetaData();
+    }
+    document::Document::UP getDocument(DocumentIdT lid) const override {
+        return Document::UP((lid == docid) ? document->clone() : 0);
+    }
+
+    CachedSelect::SP parseSelect(const vespalib::string &selection) const override {
+        CachedSelect::SP res(new CachedSelect);
+        res->set(selection, repo);
+        return res;
+    }
+
+    static void reset() { _docidCnt = 2; }
+};
+
+struct VisitRecordingUnitDR : UnitDR {
+    using VisitedLIDs = std::unordered_set<DocumentIdT>;
+    VisitedLIDs& visited_lids;
+
+    VisitRecordingUnitDR(VisitedLIDs& visited, document::Document::UP d,
+                        Timestamp t, Bucket b, bool r)
+        : UnitDR(std::move(d), t, b, r),
+          visited_lids(visited)
+    {
+    }
+
+    document::Document::UP getDocument(DocumentIdT lid) const override {
+        if (lid == docid) {
+            visited_lids.insert(lid);
+        }
+        return UnitDR::getDocument(lid);
+    }
+};
+
+class MyAttributeManager : public search::IAttributeManager
+{
+public:
+    typedef std::map<string, AttributeVector::SP> AttributeMap;
+
+    AttributeMap _attributes;
+
+    AttributeVector::SP
+    findAttribute(const vespalib::string &name) const {
+        AttributeMap::const_iterator itr = _attributes.find(name);
+        if (itr != _attributes.end()) {
+            return itr->second;
+        }
+        return AttributeVector::SP();
+    }
+
+    AttributeGuard::UP getAttribute(const string &name) const override {
+        AttributeVector::SP attr = findAttribute(name);
+        return AttributeGuard::UP(new AttributeGuard(attr));
+    }
+
+    AttributeGuard::UP getAttributeStableEnum(const string & name) const override {
+        AttributeVector::SP attr = findAttribute(name);
+        return AttributeGuard::UP(new AttributeEnumGuard(attr));
+    }
+
+    void getAttributeList(std::vector<AttributeGuard> & list) const override {
+        list.reserve(_attributes.size());
+        for (AttributeMap::const_iterator itr = _attributes.begin();
+             itr != _attributes.end();
+             ++itr) {
+            list.push_back(AttributeGuard(itr->second));
+        }
+    }
+
+    IAttributeContext::UP createContext() const override {
+        return IAttributeContext::UP(new AttributeContext(*this));
+    }
+
+    MyAttributeManager() : _attributes() { }
+
+    void addAttribute(const string &name, const AttributeVector::SP &av) {
+        av->addReservedDoc();
+        _attributes[name] = av;
+    }
+};
+
+struct AttrUnitDR : public UnitDR
+{
+    MyAttributeManager _amgr;
+    Schema _schema;
+    AttributeVector::SP _aa;
+    AttributeVector::SP _dd;
+    AttributeVector::SP _ss;
+
+    AttrUnitDR(document::Document::UP d, Timestamp t, Bucket b, bool r)
+        : UnitDR(d->getType(), document::Document::UP(d->clone()), t, b, r),
+          _amgr(), _schema(), _aa(), _dd(), _ss()
+    {
+        createAttribute(_aa, BasicType::INT32, Schema::INT32, "aa");
+        createAttribute(_dd, BasicType::DOUBLE, Schema::DOUBLE, "dd");
+        createAttribute(_ss, BasicType::STRING, Schema::STRING, "ss");
+    }
+
+    AttrUnitDR(document::Document::UP d, Timestamp t, Bucket b, bool r,
+               int32_t aa, double dd, const vespalib::string &ss)
+        : UnitDR(d->getType(), document::Document::UP(d->clone()), t, b, r),
+          _amgr(), _schema(), _aa(), _dd(), _ss()
+    {
+        createAttribute(_aa, BasicType::INT32, Schema::INT32, "aa");
+        addAttribute<IntFieldValue, int32_t>(*_aa, aa);
+        createAttribute(_dd, BasicType::DOUBLE, Schema::DOUBLE, "dd");
+        addAttribute<DoubleFieldValue, double>(*_dd, dd);
+        createAttribute(_ss, BasicType::STRING, Schema::STRING, "ss");
+        addAttribute<StringFieldValue, vespalib::string>(*_ss, ss);
+    }
+
+    void createAttribute(AttributeVector::SP &av, BasicType basicType,
+                         Schema::DataType dataType, const vespalib::string &fieldName)
+    {
+        Config cfg(basicType, CollectionType::SINGLE);
+        cfg.setFastSearch(true);
+        av = search::AttributeFactory::createAttribute(fieldName, cfg);
+        _amgr.addAttribute(fieldName, av);
+        _schema.addAttributeField(Schema::AttributeField(fieldName, dataType));
+        while (docid >= av->getNumDocs()) {
+            AttributeVector::DocId checkDocId(0u);
+            ASSERT_TRUE(av->addDoc(checkDocId));
+            av->clearDoc(docid);
+        }
+        av->commit();
+    }
+
+    template <class FieldValType, typename FieldValArg>
+    void addAttribute(AttributeVector &av, const FieldValArg &val) {
+        search::AttrUpdate::handleValue(av, docid, FieldValType(val));
+        av.commit();
+    }
+
+    CachedSelect::SP parseSelect(const vespalib::string &selection) const override {
+        CachedSelect::SP res(new CachedSelect);
+        res->set(selection, "foo", Document(document->getType(), DocumentId()), repo, _schema, &_amgr, true);
+        return res;
+    }
+};
+
+DocumentIdT UnitDR::_docidCnt(2);
+
+struct PairDR : DocumentRetrieverBaseForTest {
+    IDocumentRetriever::SP first;
+    IDocumentRetriever::SP second;
+    PairDR(IDocumentRetriever::SP f, IDocumentRetriever::SP s)
+        : first(f), second(s) {}
+    const document::DocumentTypeRepo &getDocumentTypeRepo() const override {
+        return first->getDocumentTypeRepo();
+    }
+    void getBucketMetaData(const Bucket &b, DocumentMetaData::Vector &result) const override {
+        first->getBucketMetaData(b, result);
+        second->getBucketMetaData(b, result);
+    }
+    DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const override {
+        DocumentMetaData ret = first->getDocumentMetaData(id);
+        return (ret.valid()) ? ret : second->getDocumentMetaData(id);
+    }
+    document::Document::UP getDocument(DocumentIdT lid) const override {
+        Document::UP ret = first->getDocument(lid);
+        return (ret.get() != 0) ? std::move(ret) : second->getDocument(lid);
+    }
+
+    CachedSelect::SP parseSelect(const vespalib::string &selection) const override {
+        CachedSelect::SP res(new CachedSelect);
+        res->set(selection, getDocumentTypeRepo());
+        return res;
+    }
+};
+
+struct Committer : public ICommitable {
+    size_t _commitCount;
+    size_t _commitAndWaitCount;
+    Committer() : _commitCount(0), _commitAndWaitCount(0) { }
+    void commit() override { _commitCount++; }
+    void commitAndWait() override { _commitAndWaitCount++; }
+};
+
+size_t getSize() {
+    return sizeof(DocEntry);
+}
+
+size_t getSize(const document::Document &doc) {
+    vespalib::nbostream tmp;
+    doc.serialize(tmp);
+    return tmp.size() + getSize();
+}
+
+size_t getSize(const document::DocumentId &id) {
+    return id.getSerializedSize() + getSize();
+}
+
+IDocumentRetriever::SP nil() { return IDocumentRetriever::SP(new UnitDR()); }
+
+IDocumentRetriever::SP doc(const std::string &id, Timestamp t, Bucket b) {
+    Document::UP d(new Document(*DataType::DOCUMENT, DocumentId(id)));
+    return IDocumentRetriever::SP(new UnitDR(std::move(d), t, b, false));
+}
+
+IDocumentRetriever::SP rem(const std::string &id, Timestamp t, Bucket b) {
+    Document::UP d(new Document(*DataType::DOCUMENT, DocumentId(id)));
+    return IDocumentRetriever::SP(new UnitDR(std::move(d), t, b, true));
+}
+
+IDocumentRetriever::SP cat(IDocumentRetriever::SP first, IDocumentRetriever::SP second) {
+    return IDocumentRetriever::SP(new PairDR(first, second));
+}
+
+const DocumentType &getDocType() {
+    static DocumentType::UP doc_type;
+    if (!doc_type.get()) {
+        doc_type.reset(new DocumentType("foo", 42));
+        doc_type->addField(Field("header", 43, *DataType::STRING, true));
+        doc_type->addField(Field("body", 44, *DataType::STRING, false));
+    }
+    return *doc_type;
+}
+
+const DocumentType &getAttrDocType() {
+    static DocumentType::UP doc_type;
+    if (!doc_type.get()) {
+        doc_type.reset(new DocumentType("foo", 42));
+        doc_type->addField(Field("header", 43, *DataType::STRING, true));
+        doc_type->addField(Field("body", 44, *DataType::STRING, false));
+        doc_type->addField(Field("aa", 45, *DataType::INT, false));
+        doc_type->addField(Field("ab", 46, *DataType::INT, false));
+        doc_type->addField(Field("dd", 47, *DataType::DOUBLE, false));
+        doc_type->addField(Field("ss", 48, *DataType::STRING, false));
+    }
+    return *doc_type;
+}
+
+IDocumentRetriever::SP doc_with_fields(const std::string &id, Timestamp t, Bucket b) {
+    Document::UP d(new Document(getDocType(), DocumentId(id)));
+    d->set("header", "foo");
+    d->set("body", "bar");
+    return IDocumentRetriever::SP(new UnitDR(getDocType(), std::move(d), t, b, false));
+}
+
+IDocumentRetriever::SP doc_with_null_fields(const std::string &id, Timestamp t, Bucket b) {
+    Document::UP d(new Document(getAttrDocType(), DocumentId(id)));
+    return IDocumentRetriever::SP(new AttrUnitDR(std::move(d), t, b, false));
+}
+
+IDocumentRetriever::SP doc_with_attr_fields(const vespalib::string &id,
+                                            Timestamp t, Bucket b,
+                                            int32_t aa, int32_t ab, int32_t attr_aa,
+                                            double dd, double attr_dd,
+                                            const vespalib::string &ss,
+                                            const vespalib::string &attr_ss)
+{
+    Document::UP d(new Document(getAttrDocType(), DocumentId(id)));
+    d->set("header", "foo");
+    d->set("body", "bar");
+    d->set("aa", aa);
+    d->set("ab", ab);
+    d->set("dd", dd);
+    d->set("ss", ss);
+    return IDocumentRetriever::SP(new AttrUnitDR(std::move(d), t, b, false,
+                                                 attr_aa, attr_dd, attr_ss));
+}
+
+auto doc_rec(VisitRecordingUnitDR::VisitedLIDs& visited_lids,
+             const std::string &id, Timestamp t, Bucket b)
+{
+    Document::UP d(new Document(getDocType(), DocumentId(id)));
+    return std::make_shared<VisitRecordingUnitDR>(
+            visited_lids, std::move(d), t, b, false);
+}
+
+void checkDoc(const IDocumentRetriever &dr, const std::string &id,
+              size_t timestamp, size_t bucket, bool removed)
+{
+    DocumentMetaData dmd = dr.getDocumentMetaData(DocumentId(id));
+    EXPECT_TRUE(dmd.valid());
+    EXPECT_EQUAL(timestamp, dmd.timestamp);
+    EXPECT_EQUAL(bucket, dmd.bucketId.getId());
+    EXPECT_EQUAL(DocumentId(id).getGlobalId(), dmd.gid);
+    EXPECT_EQUAL(removed, dmd.removed);
+    Document::UP doc = dr.getDocument(dmd.lid);
+    ASSERT_TRUE(doc.get() != 0);
+    EXPECT_TRUE(DocumentId(id) == doc->getId());
+}
+
+void checkEntry(const IterateResult &res, size_t idx, const Timestamp &timestamp, int flags)
+{
+    ASSERT_LESS(idx, res.getEntries().size());
+    DocEntry expect(timestamp, flags);
+    EXPECT_EQUAL(expect, *res.getEntries()[idx]);
+    EXPECT_EQUAL(getSize(), res.getEntries()[idx]->getSize());
+}
+
+void checkEntry(const IterateResult &res, size_t idx, const DocumentId &id, const Timestamp &timestamp)
+{
+    ASSERT_LESS(idx, res.getEntries().size());
+    DocEntry expect(timestamp, storage::spi::REMOVE_ENTRY, id);
+    EXPECT_EQUAL(expect, *res.getEntries()[idx]);
+    EXPECT_EQUAL(getSize(id), res.getEntries()[idx]->getSize());
+    EXPECT_GREATER(getSize(id), 0u);
+}
+
+void checkEntry(const IterateResult &res, size_t idx, const Document &doc, const Timestamp &timestamp)
+{
+    ASSERT_LESS(idx, res.getEntries().size());
+    DocEntry expect(timestamp, storage::spi::NONE, Document::UP(doc.clone()));
+    EXPECT_EQUAL(expect, *res.getEntries()[idx]);
+    EXPECT_EQUAL(getSize(doc), res.getEntries()[idx]->getSize());
+    EXPECT_GREATER(getSize(doc), 0u);
+}
+
+TEST("require that custom retrievers work as expected") {
+    IDocumentRetriever::SP dr =
+        cat(cat(doc("doc:foo:1", Timestamp(2), bucket(5)),
+                rem("doc:foo:2", Timestamp(3), bucket(5))),
+            cat(doc("doc:foo:3", Timestamp(7), bucket(6)),
+                nil()));
+    EXPECT_FALSE(dr->getDocumentMetaData(DocumentId("doc:foo:bogus")).valid());
+    EXPECT_TRUE(dr->getDocument(1).get() == 0);
+    EXPECT_TRUE(dr->getDocument(2).get() == 0);
+    EXPECT_TRUE(dr->getDocument(3).get() != 0);
+    TEST_DO(checkDoc(*dr, "doc:foo:1", 2, 5, false));
+    TEST_DO(checkDoc(*dr, "doc:foo:2", 3, 5, true));
+    TEST_DO(checkDoc(*dr, "doc:foo:3", 7, 6, false));
+    DocumentMetaData::Vector b5;
+    DocumentMetaData::Vector b6;
+    dr->getBucketMetaData(bucket(5), b5);
+    dr->getBucketMetaData(bucket(6), b6);
+    ASSERT_EQUAL(2u, b5.size());
+    ASSERT_EQUAL(1u, b6.size());
+    EXPECT_EQUAL(5u, b5[0].timestamp + b5[1].timestamp);
+    EXPECT_EQUAL(7u, b6[0].timestamp);
+}
+
+TEST("require that an empty list of retrievers can be iterated") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false);
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_EQUAL(0u, res.getEntries().size());
+    EXPECT_TRUE(res.isCompleted());
+}
+
+TEST("require that a list of empty retrievers can be iterated") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false);
+    itr.add(nil());
+    itr.add(nil());
+    itr.add(nil());
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_EQUAL(0u, res.getEntries().size());
+    EXPECT_TRUE(res.isCompleted());
+}
+
+TEST("require that normal documents can be iterated") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false);
+    itr.add(doc("doc:foo:1", Timestamp(2), bucket(5)));
+    itr.add(cat(doc("doc:foo:2", Timestamp(3), bucket(5)),
+                doc("doc:foo:3", Timestamp(4), bucket(5))));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(3u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:1")), Timestamp(2)));
+    TEST_DO(checkEntry(res, 1, Document(*DataType::DOCUMENT, DocumentId("doc:foo:2")), Timestamp(3)));
+    TEST_DO(checkEntry(res, 2, Document(*DataType::DOCUMENT, DocumentId("doc:foo:3")), Timestamp(4)));
+}
+
+void verifyReadConsistency(DocumentIterator & itr, Committer & committer) {
+    IDocumentRetriever::SP retriever = doc("doc:foo:1", Timestamp(2), bucket(5));
+    IDocumentRetriever::SP commitAndWaitRetriever(new CommitAndWaitDocumentRetriever(retriever, committer));
+    itr.add(commitAndWaitRetriever);
+
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(1u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:1")), Timestamp(2)));
+    EXPECT_EQUAL(0u, committer._commitCount);
+}
+
+void verifyStrongReadConsistency(DocumentIterator & itr) {
+    Committer committer;
+    TEST_DO(verifyReadConsistency(itr, committer));
+    EXPECT_EQUAL(1u, committer._commitAndWaitCount);
+}
+
+void verifyWeakReadConsistency(DocumentIterator & itr) {
+    Committer committer;
+    TEST_DO(verifyReadConsistency(itr, committer));
+    EXPECT_EQUAL(0u, committer._commitAndWaitCount);
+}
+
+TEST("require that default readconsistency does commit") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false);
+    TEST_DO(verifyStrongReadConsistency(itr));
+}
+
+TEST("require that readconsistency::strong does commit") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false, storage::spi::ReadConsistency::STRONG);
+    TEST_DO(verifyStrongReadConsistency(itr));
+}
+
+TEST("require that readconsistency::weak does not commit") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false, storage::spi::ReadConsistency::WEAK);
+    TEST_DO(verifyWeakReadConsistency(itr));
+}
+
+TEST("require that remove entries can be iterated") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false);
+    itr.add(rem("doc:foo:1", Timestamp(2), bucket(5)));
+    itr.add(cat(rem("doc:foo:2", Timestamp(3), bucket(5)),
+                rem("doc:foo:3", Timestamp(4), bucket(5))));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(3u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, DocumentId("doc:foo:1"), Timestamp(2)));
+    TEST_DO(checkEntry(res, 1, DocumentId("doc:foo:2"), Timestamp(3)));
+    TEST_DO(checkEntry(res, 2, DocumentId("doc:foo:3"), Timestamp(4)));
+}
+
+TEST("require that remove entries can be ignored") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), docV(), -1, false);
+    itr.add(rem("doc:foo:1", Timestamp(2), bucket(5)));
+    itr.add(cat(doc("doc:foo:2", Timestamp(3), bucket(5)),
+                rem("doc:foo:3", Timestamp(4), bucket(5))));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(1u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:2")), Timestamp(3)));
+}
+
+TEST("require that iterating all versions returns both documents and removes") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), allV(), -1, false);
+    itr.add(rem("doc:foo:1", Timestamp(2), bucket(5)));
+    itr.add(cat(doc("doc:foo:2", Timestamp(3), bucket(5)),
+                rem("doc:foo:3", Timestamp(4), bucket(5))));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(3u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, DocumentId("doc:foo:1"), Timestamp(2)));
+    TEST_DO(checkEntry(res, 1, Document(*DataType::DOCUMENT, DocumentId("doc:foo:2")), Timestamp(3)));
+    TEST_DO(checkEntry(res, 2, DocumentId("doc:foo:3"), Timestamp(4)));
+}
+
+TEST("require that using an empty field set returns meta-data only") {
+    DocumentIterator itr(bucket(5), document::NoFields(), selectAll(), newestV(), -1, false);
+    itr.add(doc("doc:foo:1", Timestamp(2), bucket(5)));
+    itr.add(cat(doc("doc:foo:2", Timestamp(3), bucket(5)),
+                rem("doc:foo:3", Timestamp(4), bucket(5))));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(3u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Timestamp(2), storage::spi::NONE));
+    TEST_DO(checkEntry(res, 1, Timestamp(3), storage::spi::NONE));
+    TEST_DO(checkEntry(res, 2, Timestamp(4), storage::spi::REMOVE_ENTRY));
+}
+
+TEST("require that entries in other buckets are skipped") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false);
+    itr.add(rem("doc:foo:1", Timestamp(2), bucket(6)));
+    itr.add(cat(doc("doc:foo:2", Timestamp(3), bucket(5)),
+                doc("doc:foo:3", Timestamp(4), bucket(6))));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(1u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:2")), Timestamp(3)));
+}
+
+TEST("require that maxBytes splits iteration results") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false);
+    itr.add(doc("doc:foo:1", Timestamp(2), bucket(5)));
+    itr.add(cat(rem("doc:foo:2", Timestamp(3), bucket(5)),
+                doc("doc:foo:3", Timestamp(4), bucket(5))));
+    IterateResult res1 = itr.iterate(getSize(Document(*DataType::DOCUMENT, DocumentId("doc:foo:1"))) +
+                                     getSize(DocumentId("doc:foo:2")));
+    EXPECT_TRUE(!res1.isCompleted());
+    EXPECT_EQUAL(2u, res1.getEntries().size());
+    TEST_DO(checkEntry(res1, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:1")), Timestamp(2)));
+    TEST_DO(checkEntry(res1, 1, DocumentId("doc:foo:2"), Timestamp(3)));
+
+    IterateResult res2 = itr.iterate(largeNum);
+    EXPECT_TRUE(res2.isCompleted());
+    TEST_DO(checkEntry(res2, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:3")), Timestamp(4)));
+
+    IterateResult res3 = itr.iterate(largeNum);
+    EXPECT_TRUE(res3.isCompleted());
+    EXPECT_EQUAL(0u, res3.getEntries().size());
+}
+
+TEST("require that maxBytes splits iteration results for meta-data only iteration") {
+    DocumentIterator itr(bucket(5), document::NoFields(), selectAll(), newestV(), -1, false);
+    itr.add(doc("doc:foo:1", Timestamp(2), bucket(5)));
+    itr.add(cat(rem("doc:foo:2", Timestamp(3), bucket(5)),
+                doc("doc:foo:3", Timestamp(4), bucket(5))));
+    IterateResult res1 = itr.iterate(getSize() + getSize());
+    EXPECT_TRUE(!res1.isCompleted());
+    EXPECT_EQUAL(2u, res1.getEntries().size());
+    TEST_DO(checkEntry(res1, 0, Timestamp(2), storage::spi::NONE));
+    TEST_DO(checkEntry(res1, 1, Timestamp(3), storage::spi::REMOVE_ENTRY));
+
+    IterateResult res2 = itr.iterate(largeNum);
+    EXPECT_TRUE(res2.isCompleted());
+    TEST_DO(checkEntry(res2, 0, Timestamp(4), storage::spi::NONE));
+
+    IterateResult res3 = itr.iterate(largeNum);
+    EXPECT_TRUE(res3.isCompleted());
+    EXPECT_EQUAL(0u, res3.getEntries().size());
+}
+
+TEST("require that at least one document is returned by visit") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false);
+    itr.add(doc("doc:foo:1", Timestamp(2), bucket(5)));
+    itr.add(cat(rem("doc:foo:2", Timestamp(3), bucket(5)),
+                doc("doc:foo:3", Timestamp(4), bucket(5))));
+    IterateResult res1 = itr.iterate(0);
+    EXPECT_TRUE(1u <= res1.getEntries().size());
+    TEST_DO(checkEntry(res1, 0, Document(*DataType::DOCUMENT,DocumentId("doc:foo:1")), Timestamp(2)));
+}
+
+TEST("require that documents outside the timestamp limits are ignored") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectTimestampRange(100, 200), newestV(), -1, false);
+    itr.add(doc("doc:foo:1", Timestamp(99),  bucket(5)));
+    itr.add(doc("doc:foo:2", Timestamp(100), bucket(5)));
+    itr.add(doc("doc:foo:3", Timestamp(200), bucket(5)));
+    itr.add(doc("doc:foo:4", Timestamp(201), bucket(5)));
+    itr.add(rem("doc:foo:5", Timestamp(99),  bucket(5)));
+    itr.add(rem("doc:foo:6", Timestamp(100), bucket(5)));
+    itr.add(rem("doc:foo:7", Timestamp(200), bucket(5)));
+    itr.add(rem("doc:foo:8", Timestamp(201), bucket(5)));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(4u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:2")), Timestamp(100)));
+    TEST_DO(checkEntry(res, 1, Document(*DataType::DOCUMENT, DocumentId("doc:foo:3")), Timestamp(200)));
+    TEST_DO(checkEntry(res, 2, DocumentId("doc:foo:6"), Timestamp(100)));
+    TEST_DO(checkEntry(res, 3, DocumentId("doc:foo:7"), Timestamp(200)));
+}
+
+TEST("require that timestamp subset returns the appropriate documents") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectTimestampSet(200, 350, 400), newestV(), -1, false);
+    itr.add(doc("doc:foo:1", Timestamp(500),  bucket(5)));
+    itr.add(doc("doc:foo:2", Timestamp(400), bucket(5)));
+    itr.add(doc("doc:foo:3", Timestamp(300), bucket(5)));
+    itr.add(doc("doc:foo:4", Timestamp(200), bucket(5)));
+    itr.add(rem("doc:foo:5", Timestamp(250),  bucket(5)));
+    itr.add(rem("doc:foo:6", Timestamp(350), bucket(5)));
+    itr.add(rem("doc:foo:7", Timestamp(450), bucket(5)));
+    itr.add(rem("doc:foo:8", Timestamp(550), bucket(5)));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(3u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:2")), Timestamp(400)));
+    TEST_DO(checkEntry(res, 1, Document(*DataType::DOCUMENT, DocumentId("doc:foo:4")), Timestamp(200)));
+    TEST_DO(checkEntry(res, 2, DocumentId("doc:foo:6"), Timestamp(350)));
+}
+
+TEST("require that document selection will filter results") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectDocs("id=\"doc:foo:xxx*\""), newestV(), -1, false);
+    itr.add(doc("doc:foo:xxx1", Timestamp(99),  bucket(5)));
+    itr.add(doc("doc:foo:yyy1", Timestamp(100), bucket(5)));
+    itr.add(doc("doc:foo:xxx2", Timestamp(200), bucket(5)));
+    itr.add(doc("doc:foo:yyy2", Timestamp(201), bucket(5)));
+    itr.add(rem("doc:foo:xxx3", Timestamp(99),  bucket(5)));
+    itr.add(rem("doc:foo:yyy3", Timestamp(100), bucket(5)));
+    itr.add(rem("doc:foo:xxx4", Timestamp(200), bucket(5)));
+    itr.add(rem("doc:foo:yyy4", Timestamp(201), bucket(5)));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(4u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:xxx1")), Timestamp(99)));
+    TEST_DO(checkEntry(res, 1, Document(*DataType::DOCUMENT, DocumentId("doc:foo:xxx2")), Timestamp(200)));
+    TEST_DO(checkEntry(res, 2, DocumentId("doc:foo:xxx3"), Timestamp(99)));
+    TEST_DO(checkEntry(res, 3, DocumentId("doc:foo:xxx4"), Timestamp(200)));
+}
+
+TEST("require that document selection handles 'field == null'") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectDocs("foo.aa == null"), newestV(), -1, false);
+    itr.add(doc_with_null_fields("doc:foo:xxx1", Timestamp(99),  bucket(5)));
+    itr.add(doc_with_null_fields("doc:foo:xxx2", Timestamp(100),  bucket(5)));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    ASSERT_EQUAL(2u, res.getEntries().size());
+    Document expected1(getAttrDocType(), DocumentId("doc:foo:xxx1"));
+    TEST_DO(checkEntry(res, 0, expected1, Timestamp(99)));
+    Document expected2(getAttrDocType(), DocumentId("doc:foo:xxx2"));
+    TEST_DO(checkEntry(res, 1, expected2, Timestamp(100)));
+}
+
+TEST("require that invalid document selection returns no documents") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectDocs("=="), newestV(), -1, false);
+    itr.add(doc("doc:foo:xxx1", Timestamp(99),  bucket(5)));
+    itr.add(doc("doc:foo:yyy1", Timestamp(100), bucket(5)));
+    itr.add(doc("doc:foo:xxx2", Timestamp(200), bucket(5)));
+    itr.add(doc("doc:foo:yyy2", Timestamp(201), bucket(5)));
+    itr.add(rem("doc:foo:xxx3", Timestamp(99),  bucket(5)));
+    itr.add(rem("doc:foo:yyy3", Timestamp(100), bucket(5)));
+    itr.add(rem("doc:foo:xxx4", Timestamp(200), bucket(5)));
+    itr.add(rem("doc:foo:yyy4", Timestamp(201), bucket(5)));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(0u, res.getEntries().size());
+}
+
+TEST("require that document selection and timestamp range works together") {
+    DocumentIterator itr(bucket(5), document::AllFields(), selectDocsWithinRange("id=\"doc:foo:xxx*\"", 100, 200), newestV(), -1, false);
+    itr.add(doc("doc:foo:xxx1", Timestamp(99),  bucket(5)));
+    itr.add(doc("doc:foo:yyy1", Timestamp(100), bucket(5)));
+    itr.add(doc("doc:foo:xxx2", Timestamp(200), bucket(5)));
+    itr.add(doc("doc:foo:yyy2", Timestamp(201), bucket(5)));
+    itr.add(rem("doc:foo:xxx3", Timestamp(99),  bucket(5)));
+    itr.add(rem("doc:foo:yyy3", Timestamp(100), bucket(5)));
+    itr.add(rem("doc:foo:xxx4", Timestamp(200), bucket(5)));
+    itr.add(rem("doc:foo:yyy4", Timestamp(201), bucket(5)));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(2u, res.getEntries().size());
+    TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:xxx2")), Timestamp(200)));
+    TEST_DO(checkEntry(res, 1, DocumentId("doc:foo:xxx4"), Timestamp(200)));
+}
+
+TEST("require that fieldset limits fields returned") {
+    DocumentIterator itr(bucket(5), document::HeaderFields(), selectAll(), newestV(), -1, false);
+    itr.add(doc_with_fields("doc:foo:xxx1", Timestamp(1),  bucket(5)));
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(1u, res.getEntries().size());
+    Document expected(getDocType(), DocumentId("doc:foo:xxx1"));
+    expected.set("header", "foo");
+    TEST_DO(checkEntry(res, 0, expected, Timestamp(1)));
+}
+
+namespace {
+template <typename Container, typename T>
+bool contains(const Container& c, const T& value) {
+    return c.find(value) != c.end();
+}
+}
+
+TEST("require that userdoc-constrained selections pre-filter on GIDs") {
+    DocumentIterator itr(bucket(5), document::AllFields(),
+                         selectDocs("id.user=1234"), newestV(), -1, false);
+    VisitRecordingUnitDR::VisitedLIDs visited_lids;
+    // Even though GID filtering is probabilistic when it comes to filtering
+    // user IDs that cover the 64-bit range, it's fully deterministic when the
+    // user IDs are all 32 bits or less, which is the case for the below IDs.
+    auto wanted_dr_1   = doc_rec(visited_lids, "id::foo:n=1234:a",
+                                 Timestamp(99), bucket(5));
+    auto filtered_dr_1 = doc_rec(visited_lids, "id::foo:n=4321:b",
+                                 Timestamp(200), bucket(5));
+    auto filtered_dr_2 = doc_rec(visited_lids, "id::foo:n=5678:c",
+                                 Timestamp(201), bucket(5));
+    auto wanted_dr_2   = doc_rec(visited_lids, "id::foo:n=1234:d",
+                                 Timestamp(300), bucket(5));
+    auto wanted_dr_3   = doc_rec(visited_lids, "id::foo:n=1234:e",
+                                 Timestamp(301), bucket(5));
+    itr.add(wanted_dr_1);
+    itr.add(filtered_dr_1);
+    itr.add(cat(filtered_dr_2, wanted_dr_2));
+    itr.add(wanted_dr_3);
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(3u, visited_lids.size());
+    EXPECT_TRUE(contains(visited_lids, wanted_dr_1->docid));
+    EXPECT_TRUE(contains(visited_lids, wanted_dr_2->docid));
+    EXPECT_TRUE(contains(visited_lids, wanted_dr_3->docid));
+}
+
+TEST("require that attributes are used")
+{
+    UnitDR::reset();
+    DocumentIterator itr(bucket(5), document::AllFields(), selectDocs("foo.aa == 45"), docV(), -1, false);
+    itr.add(doc_with_attr_fields("doc:foo:xx1", Timestamp(1), bucket(5),
+                                 27, 28, 27, 2.7, 2.8, "x27", "x28"));
+    itr.add(doc_with_attr_fields("doc:foo:xx2", Timestamp(2), bucket(5),
+                                 27, 28, 45, 2.7, 4.5, "x27", "x45"));
+    itr.add(doc_with_attr_fields("doc:foo:xx3", Timestamp(3), bucket(5),
+                                 45, 46, 27, 4.5, 2.7, "x45", "x27"));
+    itr.add(doc_with_attr_fields("doc:foo:xx4", Timestamp(4), bucket(5),
+                                 45, 46, 45, 4.5, 4.5, "x45", "x45"));
+    
+    IterateResult res = itr.iterate(largeNum);
+    EXPECT_TRUE(res.isCompleted());
+    EXPECT_EQUAL(2u, res.getEntries().size());
+    Document expected1(getAttrDocType(), DocumentId("doc:foo:xx2"));
+    expected1.set("header", "foo");
+    expected1.set("body", "bar");
+    expected1.set("aa", 27);
+    expected1.set("ab", 28);
+    expected1.set("dd", 2.7);
+    expected1.set("ss", "x27");
+    Document expected2(getAttrDocType(), DocumentId("doc:foo:xx4"));
+    expected2.set("header", "foo");
+    expected2.set("body", "bar");
+    expected2.set("aa", 45);
+    expected2.set("ab", 46);
+    expected2.set("dd", 4.5);
+    expected2.set("ss", "x45");
+    TEST_DO(checkEntry(res, 0, expected1, Timestamp(2)));
+    TEST_DO(checkEntry(res, 1, expected2, Timestamp(4)));
+
+    DocumentIterator itr2(bucket(5), document::AllFields(), selectDocs("foo.dd == 4.5"), docV(), -1, false);
+    itr2.add(doc_with_attr_fields("doc:foo:xx5", Timestamp(5), bucket(5),
+                                  27, 28, 27, 2.7, 2.8, "x27", "x28"));
+    itr2.add(doc_with_attr_fields("doc:foo:xx6", Timestamp(6), bucket(5),
+                                  27, 28, 45, 2.7, 4.5, "x27", "x45"));
+    itr2.add(doc_with_attr_fields("doc:foo:xx7", Timestamp(7), bucket(5),
+                                  45, 46, 27, 4.5, 2.7, "x45", "x27"));
+    itr2.add(doc_with_attr_fields("doc:foo:xx8", Timestamp(8), bucket(5),
+                                  45, 46, 45, 4.5, 4.5, "x45", "x45"));
+    
+    IterateResult res2 = itr2.iterate(largeNum);
+    EXPECT_TRUE(res2.isCompleted());
+    EXPECT_EQUAL(2u, res2.getEntries().size());
+    Document expected3(getAttrDocType(), DocumentId("doc:foo:xx6"));
+    expected3.set("header", "foo");
+    expected3.set("body", "bar");
+    expected3.set("aa", 27);
+    expected3.set("ab", 28);
+    expected3.set("dd", 2.7);
+    expected3.set("ss", "x27");
+    Document expected4(getAttrDocType(), DocumentId("doc:foo:xx8"));
+    expected4.set("header", "foo");
+    expected4.set("body", "bar");
+    expected4.set("aa", 45);
+    expected4.set("ab", 46);
+    expected4.set("dd", 4.5);
+    expected4.set("ss", "x45");
+    TEST_DO(checkEntry(res2, 0, expected3, Timestamp(6)));
+    TEST_DO(checkEntry(res2, 1, expected4, Timestamp(8)));
+
+    DocumentIterator itr3(bucket(5), document::AllFields(), selectDocs("foo.ss == \"x45\""), docV(), -1, false);
+    itr3.add(doc_with_attr_fields("doc:foo:xx9", Timestamp(9), bucket(5),
+                                  27, 28, 27, 2.7, 2.8, "x27", "x28"));
+    itr3.add(doc_with_attr_fields("doc:foo:xx10", Timestamp(10), bucket(5),
+                                  27, 28, 45, 2.7, 4.5, "x27", "x45"));
+    itr3.add(doc_with_attr_fields("doc:foo:xx11", Timestamp(11), bucket(5),
+                                  45, 46, 27, 4.5, 2.7, "x45", "x27"));
+    itr3.add(doc_with_attr_fields("doc:foo:xx12", Timestamp(12), bucket(5),
+                                  45, 46, 45, 4.5, 4.5, "x45", "x45"));
+    
+    IterateResult res3 = itr3.iterate(largeNum);
+    EXPECT_TRUE(res3.isCompleted());
+    EXPECT_EQUAL(2u, res3.getEntries().size());
+    Document expected5(getAttrDocType(), DocumentId("doc:foo:xx10"));
+    expected5.set("header", "foo");
+    expected5.set("body", "bar");
+    expected5.set("aa", 27);
+    expected5.set("ab", 28);
+    expected5.set("dd", 2.7);
+    expected5.set("ss", "x27");
+    Document expected6(getAttrDocType(), DocumentId("doc:foo:xx12"));
+    expected6.set("header", "foo");
+    expected6.set("body", "bar");
+    expected6.set("aa", 45);
+    expected6.set("ab", 46);
+    expected6.set("dd", 4.5);
+    expected6.set("ss", "x45");
+    TEST_DO(checkEntry(res3, 0, expected5, Timestamp(10)));
+    TEST_DO(checkEntry(res3, 1, expected6, Timestamp(12)));
+} 
+
+TEST_MAIN() { TEST_RUN_ALL(); }
+
diff --git a/searchcore/src/tests/proton/documentdb/.gitignore b/searchcore/src/tests/proton/documentdb/.gitignore
new file mode 100644
index 00000000000..abcba544a6d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/.gitignore
@@ -0,0 +1,6 @@
+Makefile
+.depend
+documentdb_test
+tmp
+
+searchcore_documentdb_test_app
diff --git a/searchcore/src/tests/proton/documentdb/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/CMakeLists.txt
new file mode 100644
index 00000000000..9270a4b0b7c
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/CMakeLists.txt
@@ -0,0 +1,24 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_documentdb_test_app
+    SOURCES
+    documentdb_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_initializer
+    searchcore_reprocessing
+    searchcore_index
+    searchcore_docsummary
+    searchcore_persistenceengine
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_flushengine
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_fconfig
+    searchcore_util
+)
+vespa_add_test(NAME searchcore_documentdb_test_app COMMAND sh documentdb_test.sh)
diff --git a/searchcore/src/tests/proton/documentdb/DESC b/searchcore/src/tests/proton/documentdb/DESC
new file mode 100644
index 00000000000..0f8cbcb2eb0
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/DESC
@@ -0,0 +1 @@
+documentdb test. Take a look at documentdb_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/FILES b/searchcore/src/tests/proton/documentdb/FILES
new file mode 100644
index 00000000000..50fef46855d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/FILES
@@ -0,0 +1 @@
+documentdb_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/.gitignore b/searchcore/src/tests/proton/documentdb/buckethandler/.gitignore
new file mode 100644
index 00000000000..c159971ebc7
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/buckethandler/.gitignore
@@ -0,0 +1 @@
+searchcore_buckethandler_test_app
diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt
new file mode 100644
index 00000000000..3c1f5c79a57
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt
@@ -0,0 +1,18 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_buckethandler_test_app
+    SOURCES
+    buckethandler_test.cpp
+    DEPENDS
+    searchcore_test
+    searchcore_server
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_buckethandler_test_app COMMAND searchcore_buckethandler_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/DESC b/searchcore/src/tests/proton/documentdb/buckethandler/DESC
new file mode 100644
index 00000000000..f844b837422
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/buckethandler/DESC
@@ -0,0 +1 @@
+buckethandler test. Take a look at buckethandler_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/FILES b/searchcore/src/tests/proton/documentdb/buckethandler/FILES
new file mode 100644
index 00000000000..df0589a342b
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/buckethandler/FILES
@@ -0,0 +1 @@
+buckethandler_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp
new file mode 100644
index 00000000000..f139bf92e44
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp
@@ -0,0 +1,265 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("buckethandler_test");
+#include <vespa/searchcore/proton/server/buckethandler.h>
+#include <vespa/searchcore/proton/server/ibucketstatechangedhandler.h>
+#include <vespa/searchcore/proton/server/ibucketmodifiedhandler.h>
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace proton;
+using document::BucketId;
+using document::GlobalId;
+using storage::spi::Bucket;
+using storage::spi::BucketInfo;
+using storage::spi::PartitionId;
+using storage::spi::Timestamp;
+using vespalib::ThreadStackExecutor;
+using proton::test::BucketStateCalculator;
+
+const PartitionId PART_ID(0);
+const GlobalId GID_1("111111111111");
+const BucketId BUCKET_1(8, GID_1.convertToBucketId().getRawId());
+const Timestamp TIME_1(1u);
+
+struct MySubDb
+{
+    DocumentMetaStore   _metaStore;
+    test::UserDocuments _docs;
+    MySubDb(std::shared_ptr<BucketDBOwner> bucketDB, SubDbType subDbType)
+        : _metaStore(bucketDB,
+                     DocumentMetaStore::getFixedName(),
+                     search::GrowStrategy(),
+                     documentmetastore::IGidCompare::SP(
+                             new documentmetastore::DefaultGidCompare),
+                     subDbType),
+          _docs()
+    {
+    }
+    void insertDocs(const test::UserDocuments &docs_) {
+        _docs = docs_;
+        for (test::UserDocuments::Iterator itr = _docs.begin(); itr != _docs.end(); ++itr) {
+            const test::BucketDocuments &bucketDocs = itr->second;
+            for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) {
+                const test::Document &testDoc = bucketDocs.getDocs()[i];
+                _metaStore.put(testDoc.getGid(), testDoc.getBucket(),
+                               testDoc.getTimestamp(), testDoc.getLid());
+            }
+        }
+    }
+    BucketId bucket(uint32_t userId) {
+        return _docs.getBucket(userId);
+    }
+    test::DocumentVector docs(uint32_t userId) {
+        return _docs.getGidOrderDocs(userId);
+    }
+};
+
+
+struct MyChangedHandler : public IBucketStateChangedHandler
+{
+    BucketId _bucket;
+    BucketInfo::ActiveState _state;
+    MyChangedHandler() : _bucket(), _state(BucketInfo::NOT_ACTIVE) {}
+
+    virtual void notifyBucketStateChanged(const document::BucketId &bucketId,
+                                          storage::spi::BucketInfo::ActiveState newState) {
+        _bucket = bucketId;
+        _state = newState;
+    }
+};
+
+
+struct MyModifiedHandler : public IBucketModifiedHandler
+{
+    virtual void
+    notifyBucketModified(const BucketId &bucket)
+    {
+        (void) bucket;
+    }
+};
+
+
+bool
+expectEqual(uint32_t docCount, uint32_t metaCount, const BucketInfo &info)
+{
+    if (!EXPECT_EQUAL(docCount, info.getDocumentCount())) return false;
+    if (!EXPECT_EQUAL(metaCount, info.getEntryCount())) return false;
+    if (!EXPECT_EQUAL(docCount, info.getDocumentSize())) return false;
+    if (!EXPECT_EQUAL(metaCount, info.getUsedSize())) return false;
+    return true;
+}
+
+
+struct Fixture
+{
+    test::UserDocumentsBuilder      _builder;
+    std::shared_ptr<BucketDBOwner>  _bucketDB;
+    MySubDb                         _ready;
+    MySubDb                         _removed;
+    MySubDb                         _notReady;
+    ThreadStackExecutor             _exec;
+    BucketHandler                   _handler;
+    MyChangedHandler                _changedHandler;
+    MyModifiedHandler               _modifiedHandler;
+    BucketStateCalculator::SP       _calc;
+    test::BucketIdListResultHandler _bucketList;
+    test::BucketInfoResultHandler   _bucketInfo;
+    test::GenericResultHandler      _genResult;
+    Fixture()
+        : _builder(),
+          _bucketDB(std::make_shared<BucketDBOwner>()),
+          _ready(_bucketDB, SubDbType::READY),
+          _removed(_bucketDB, SubDbType::REMOVED),
+          _notReady(_bucketDB, SubDbType::NOTREADY),
+          _exec(1, 64000),
+          _handler(_exec),
+          _changedHandler(),
+          _modifiedHandler(),
+          _calc(new BucketStateCalculator()),
+          _bucketList(), _bucketInfo(), _genResult()
+    {
+        // bucket 2 & 3 & 4 & 7 in ready
+        _ready.insertDocs(_builder.createDocs(2, 1, 3).  // 2 docs
+                                   createDocs(3, 3, 6).  // 3 docs
+                                   createDocs(4, 6, 10). // 4 docs
+                                   createDocs(7, 10, 11). // 1 doc
+                                   getDocs());
+        // bucket 2 in removed
+        _removed.insertDocs(_builder.clearDocs().
+                                     createDocs(2, 16, 20). // 4 docs
+                                     getDocs());
+        // bucket 4 in not ready
+        _notReady.insertDocs(_builder.clearDocs().
+                                      createDocs(4, 22, 24). // 2 docs
+                                      getDocs());
+        _handler.setReadyBucketHandler(_ready._metaStore);
+        _handler.addBucketStateChangedHandler(&_changedHandler);
+        _handler.notifyClusterStateChanged(_calc);
+    }
+    ~Fixture()
+    {
+        _handler.removeBucketStateChangedHandler(&_changedHandler);
+    }
+    void sync() { _exec.sync(); }
+    void handleGetBucketInfo(const BucketId &bucket) {
+        _handler.handleGetBucketInfo(Bucket(bucket, PART_ID), _bucketInfo);
+    }
+    void
+    setNodeUp(bool value)
+    {
+        _calc->setNodeUp(value);
+        _handler.notifyClusterStateChanged(_calc);
+    }
+};
+
+
+TEST_F("require that handleListBuckets() returns buckets from all sub dbs", Fixture)
+{
+    f._handler.handleListBuckets(f._bucketList);
+    EXPECT_EQUAL(4u, f._bucketList.getList().size());
+    EXPECT_EQUAL(f._ready.bucket(2), f._bucketList.getList()[0]);
+    EXPECT_EQUAL(f._ready.bucket(3), f._bucketList.getList()[1]);
+    EXPECT_EQUAL(f._ready.bucket(4), f._bucketList.getList()[2]);
+    EXPECT_EQUAL(f._ready.bucket(7), f._bucketList.getList()[3]);
+    EXPECT_EQUAL(f._removed.bucket(2), f._bucketList.getList()[0]);
+    EXPECT_EQUAL(f._notReady.bucket(4), f._bucketList.getList()[2]);
+}
+
+
+TEST_F("require that bucket is reported in handleGetBucketInfo() and size faked", Fixture)
+{
+    f.handleGetBucketInfo(f._ready.bucket(3));
+    EXPECT_TRUE(expectEqual(3, 3, f._bucketInfo.getInfo()));
+
+    f.handleGetBucketInfo(f._ready.bucket(2)); // bucket 2 also in removed sub db
+    EXPECT_TRUE(expectEqual(2, 6, f._bucketInfo.getInfo()));
+}
+
+
+TEST_F("require that handleGetBucketInfo() can get cached bucket", Fixture)
+{
+    {
+        BucketDBOwner::Guard db = f._bucketDB->takeGuard();
+        db->add(GID_1, BUCKET_1, TIME_1, SubDbType::READY);
+        db->cacheBucket(BUCKET_1);
+        db->add(GID_1, BUCKET_1, TIME_1, SubDbType::NOTREADY);
+    }
+    f.handleGetBucketInfo(BUCKET_1);
+    EXPECT_TRUE(expectEqual(1, 1, f._bucketInfo.getInfo()));
+
+    f._bucketDB->takeGuard()->uncacheBucket();
+
+    f.handleGetBucketInfo(BUCKET_1);
+    EXPECT_TRUE(expectEqual(2, 2, f._bucketInfo.getInfo()));
+    {
+        // Must ensure empty bucket db before destruction.
+        BucketDBOwner::Guard db = f._bucketDB->takeGuard();
+        db->remove(GID_1, BUCKET_1, TIME_1, SubDbType::READY);
+        db->remove(GID_1, BUCKET_1, TIME_1, SubDbType::NOTREADY);
+    }
+}
+
+
+TEST_F("require that changed handlers are notified when bucket state changes", Fixture)
+{
+    f._handler.handleSetCurrentState(f._ready.bucket(2), BucketInfo::ACTIVE, f._genResult);
+    f.sync();
+    EXPECT_EQUAL(f._ready.bucket(2), f._changedHandler._bucket);
+    EXPECT_EQUAL(BucketInfo::ACTIVE, f._changedHandler._state);
+    f._handler.handleSetCurrentState(f._ready.bucket(3), BucketInfo::NOT_ACTIVE, f._genResult);
+    f.sync();
+    EXPECT_EQUAL(f._ready.bucket(3), f._changedHandler._bucket);
+    EXPECT_EQUAL(BucketInfo::NOT_ACTIVE, f._changedHandler._state);
+}
+
+
+TEST_F("require that unready bucket can be reported as active", Fixture)
+{
+    f._handler.handleSetCurrentState(f._ready.bucket(4),
+                                     BucketInfo::ACTIVE, f._genResult);
+    f.sync();
+    EXPECT_EQUAL(f._ready.bucket(4), f._changedHandler._bucket);
+    EXPECT_EQUAL(BucketInfo::ACTIVE, f._changedHandler._state);
+    f.handleGetBucketInfo(f._ready.bucket(4));
+    EXPECT_EQUAL(true, f._bucketInfo.getInfo().isActive());
+    EXPECT_EQUAL(false, f._bucketInfo.getInfo().isReady());
+}
+
+
+TEST_F("require that node being down deactivates buckets", Fixture)
+{
+    f._handler.handleSetCurrentState(f._ready.bucket(2),
+                                     BucketInfo::ACTIVE, f._genResult);
+    f.sync();
+    EXPECT_EQUAL(f._ready.bucket(2), f._changedHandler._bucket);
+    EXPECT_EQUAL(BucketInfo::ACTIVE, f._changedHandler._state);
+    f.handleGetBucketInfo(f._ready.bucket(2));
+    EXPECT_EQUAL(true, f._bucketInfo.getInfo().isActive());
+    f.setNodeUp(false);
+    f.sync();
+    f.handleGetBucketInfo(f._ready.bucket(2));
+    EXPECT_EQUAL(false, f._bucketInfo.getInfo().isActive());
+    f._handler.handleSetCurrentState(f._ready.bucket(2),
+                                     BucketInfo::ACTIVE, f._genResult);
+    f.sync();
+    f.handleGetBucketInfo(f._ready.bucket(2));
+    EXPECT_EQUAL(false, f._bucketInfo.getInfo().isActive());
+    f.setNodeUp(true);
+    f.sync();
+    f.handleGetBucketInfo(f._ready.bucket(2));
+    EXPECT_EQUAL(false, f._bucketInfo.getInfo().isActive());
+    f._handler.handleSetCurrentState(f._ready.bucket(2),
+                                     BucketInfo::ACTIVE, f._genResult);
+    f.sync();
+    f.handleGetBucketInfo(f._ready.bucket(2));
+    EXPECT_EQUAL(true, f._bucketInfo.getInfo().isActive());
+}
+
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
+
diff --git a/searchcore/src/tests/proton/documentdb/cfg/attributes.cfg b/searchcore/src/tests/proton/documentdb/cfg/attributes.cfg
new file mode 100644
index 00000000000..9d990996dd1
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/cfg/attributes.cfg
@@ -0,0 +1,3 @@
+attribute[1]
+attribute[0].name "attr1"
+attribute[0].datatype INT32
diff --git a/searchcore/src/tests/proton/documentdb/cfg/indexschema.cfg b/searchcore/src/tests/proton/documentdb/cfg/indexschema.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/cfg/indexschema.cfg
diff --git a/searchcore/src/tests/proton/documentdb/cfg/juniperrc.cfg b/searchcore/src/tests/proton/documentdb/cfg/juniperrc.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/cfg/juniperrc.cfg
diff --git a/searchcore/src/tests/proton/documentdb/cfg/rank-profiles.cfg b/searchcore/src/tests/proton/documentdb/cfg/rank-profiles.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/cfg/rank-profiles.cfg
diff --git a/searchcore/src/tests/proton/documentdb/cfg/summary.cfg b/searchcore/src/tests/proton/documentdb/cfg/summary.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/cfg/summary.cfg
diff --git a/searchcore/src/tests/proton/documentdb/cfg/summarymap.cfg b/searchcore/src/tests/proton/documentdb/cfg/summarymap.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/cfg/summarymap.cfg
diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/.gitignore b/searchcore/src/tests/proton/documentdb/clusterstatehandler/.gitignore
new file mode 100644
index 00000000000..bc38893db32
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/.gitignore
@@ -0,0 +1 @@
+searchcore_clusterstatehandler_test_app
diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt
new file mode 100644
index 00000000000..f107cddd103
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_clusterstatehandler_test_app
+    SOURCES
+    clusterstatehandler_test.cpp
+    DEPENDS
+    searchcore_test
+    searchcore_server
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_clusterstatehandler_test_app COMMAND searchcore_clusterstatehandler_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/DESC b/searchcore/src/tests/proton/documentdb/clusterstatehandler/DESC
new file mode 100644
index 00000000000..5d5921dea9a
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/DESC
@@ -0,0 +1 @@
+clusterstatehandler test. Take a look at clusterstatehandler_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/FILES b/searchcore/src/tests/proton/documentdb/clusterstatehandler/FILES
new file mode 100644
index 00000000000..92fc297c2a4
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/FILES
@@ -0,0 +1 @@
+clusterstatehandler_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp b/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp
new file mode 100644
index 00000000000..1b8bb37ac3a
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("clusterstatehandler_test");
+#include <vespa/searchcore/proton/server/clusterstatehandler.h>
+#include <vespa/searchcore/proton/server/iclusterstatechangedhandler.h>
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace proton;
+using document::BucketId;
+using storage::lib::Distribution;
+using storage::spi::BucketIdListResult;
+using storage::spi::ClusterState;
+using storage::spi::Result;
+
+struct MyClusterStateChangedHandler : public IClusterStateChangedHandler
+{
+    IBucketStateCalculator::SP _calc;
+    virtual void
+    notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) {
+        _calc = newCalc;
+    }
+};
+
+
+BucketId bucket1(1);
+BucketId bucket2(2);
+BucketId bucket3(3);
+Distribution distribution(Distribution::getDefaultDistributionConfig(3, 3));
+storage::lib::ClusterState rawClusterState("version:1 storage:3 distributor:3");
+ClusterState clusterState(rawClusterState, 0, distribution);
+
+
+struct Fixture
+{
+    vespalib::ThreadStackExecutor   _exec;
+    ClusterStateHandler             _stateHandler;
+    MyClusterStateChangedHandler    _changedHandler;
+    test::GenericResultHandler      _genericHandler;
+    test::BucketIdListResultHandler _bucketListHandler;
+    Fixture()
+        : _exec(1, 64000),
+          _stateHandler(_exec),
+          _changedHandler(),
+          _genericHandler(),
+          _bucketListHandler()
+    {
+        _stateHandler.addClusterStateChangedHandler(&_changedHandler);
+    }
+    ~Fixture()
+    {
+        _stateHandler.removeClusterStateChangedHandler(&_changedHandler);
+    }
+};
+
+
+TEST_F("require that cluster state change is notified", Fixture)
+{
+    f._stateHandler.handleSetClusterState(clusterState, f._genericHandler);
+    f._exec.sync();
+    EXPECT_TRUE(f._changedHandler._calc.get() != NULL);
+}
+
+
+TEST_F("require that modified buckets are returned", Fixture)
+{
+    f._stateHandler.handleSetClusterState(clusterState, f._genericHandler);
+    f._exec.sync();
+
+    // notify 2 buckets
+    IBucketModifiedHandler &bmh = f._stateHandler;
+    bmh.notifyBucketModified(bucket1);
+    bmh.notifyBucketModified(bucket2);
+    f._stateHandler.handleGetModifiedBuckets(f._bucketListHandler);
+    f._exec.sync();
+    EXPECT_EQUAL(2u, f._bucketListHandler.getList().size());
+    EXPECT_EQUAL(bucket1, f._bucketListHandler.getList()[0]);
+    EXPECT_EQUAL(bucket2, f._bucketListHandler.getList()[1]);
+
+    // notify 1 bucket, already reported buckets should be gone
+    bmh.notifyBucketModified(bucket3);
+    f._stateHandler.handleGetModifiedBuckets(f._bucketListHandler);
+    f._exec.sync();
+    EXPECT_EQUAL(1u, f._bucketListHandler.getList().size());
+    EXPECT_EQUAL(bucket3, f._bucketListHandler.getList()[0]);
+}
+
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
+
diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/.gitignore b/searchcore/src/tests/proton/documentdb/combiningfeedview/.gitignore
new file mode 100644
index 00000000000..3302e827c3e
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/.gitignore
@@ -0,0 +1 @@
+searchcore_combiningfeedview_test_app
diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt
new file mode 100644
index 00000000000..74f605d36d0
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_combiningfeedview_test_app
+    SOURCES
+    combiningfeedview_test.cpp
+    DEPENDS
+    searchcore_test
+    searchcore_server
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_combiningfeedview_test_app COMMAND searchcore_combiningfeedview_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/DESC b/searchcore/src/tests/proton/documentdb/combiningfeedview/DESC
new file mode 100644
index 00000000000..9882151634a
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/DESC
@@ -0,0 +1 @@
+combiningfeedview test. Take a look at combiningfeedview_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/FILES b/searchcore/src/tests/proton/documentdb/combiningfeedview/FILES
new file mode 100644
index 00000000000..791dc90442c
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/FILES
@@ -0,0 +1 @@
+combiningfeedview_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp
new file mode 100644
index 00000000000..d3d3aa4ac0d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp
@@ -0,0 +1,438 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("combiningfeedview_test");
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchcore/proton/server/combiningfeedview.h>
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using document::DocumentTypeRepo;
+using document::DocumentUpdate;
+using search::SerialNum;
+using storage::spi::Timestamp;
+using namespace proton;
+
+typedef std::vector<IFeedView::SP> FeedViewVector;
+
+struct MyStreamHandler : public NewConfigOperation::IStreamHandler
+{
+    virtual void serializeConfig(SerialNum, vespalib::nbostream &) {}
+    virtual void deserializeConfig(SerialNum, vespalib::nbostream &) {}
+};
+
+
+struct MyFeedView : public test::DummyFeedView
+{
+    typedef std::shared_ptr<MyFeedView> SP;
+    DocumentMetaStore    _metaStore;
+    MyStreamHandler      _streamHandler;
+    uint32_t             _preparePut;
+    uint32_t             _handlePut;
+    uint32_t             _prepareRemove;
+    uint32_t             _handleRemove;
+    uint32_t             _prepareUpdate;
+    uint32_t             _handleUpdate;
+    uint32_t             _prepareMove;
+    uint32_t             _handleMove;
+    uint32_t             _prepareDeleteBucket;
+    uint32_t             _handleDeleteBucket;
+    uint32_t             _heartBeat;
+    uint32_t             _handlePrune;
+    uint32_t             _wantedLidLimit;
+    MyFeedView(const DocumentTypeRepo::SP &repo,
+               std::shared_ptr<BucketDBOwner> bucketDB,
+               SubDbType subDbType) :
+        test::DummyFeedView(repo),
+        _metaStore(bucketDB,
+                   DocumentMetaStore::getFixedName(),
+                   search::GrowStrategy(),
+                   documentmetastore::IGidCompare::SP(
+                           new documentmetastore::DefaultGidCompare),
+                   subDbType),
+        _streamHandler(),
+        _preparePut(0),
+        _handlePut(0),
+        _prepareRemove(0),
+        _handleRemove(0),
+        _prepareUpdate(0),
+        _handleUpdate(0),
+        _prepareMove(0),
+        _handleMove(0),
+        _prepareDeleteBucket(0),
+        _handleDeleteBucket(0),
+        _heartBeat(0),
+        _handlePrune(0),
+        _wantedLidLimit(0)
+    {
+        _metaStore.constructFreeList();
+    }
+
+    // Implements IFeedView
+    virtual const DocumentMetaStore *getDocumentMetaStorePtr() const { return &_metaStore; }
+    virtual void preparePut(PutOperation &) { ++_preparePut; }
+    virtual void handlePut(FeedToken *, const PutOperation &) { ++_handlePut; }
+    virtual void prepareUpdate(UpdateOperation &) { ++_prepareUpdate; }
+    virtual void handleUpdate(FeedToken *, const UpdateOperation &) { ++_handleUpdate; }
+    virtual void prepareRemove(RemoveOperation &) { ++_prepareRemove; }
+    virtual void handleRemove(FeedToken *, const RemoveOperation &) { ++_handleRemove; }
+    virtual void prepareDeleteBucket(DeleteBucketOperation &) { ++_prepareDeleteBucket; }
+    virtual void handleDeleteBucket(const DeleteBucketOperation &)
+    { ++_handleDeleteBucket; }
+    virtual void prepareMove(MoveOperation &) { ++_prepareMove; }
+    virtual void handleMove(const MoveOperation &) { ++_handleMove; }
+    virtual void heartBeat(SerialNum) { ++_heartBeat; }
+    virtual void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &) { ++_handlePrune; }
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) {
+        _wantedLidLimit = op.getLidLimit();
+    }
+};
+
+
+struct MySubDb
+{
+    MyFeedView::SP _view;
+    MySubDb(const DocumentTypeRepo::SP &repo,
+            std::shared_ptr<BucketDBOwner> bucketDB,
+            SubDbType subDbType)
+        : _view(new MyFeedView(repo, bucketDB, subDbType))
+    {
+    }
+    void insertDocs(const test::BucketDocuments &docs) {
+        for (size_t i = 0; i < docs.getDocs().size(); ++i) {
+            const test::Document &testDoc = docs.getDocs()[i];
+            _view->_metaStore.put(testDoc.getGid(), testDoc.getBucket(),
+                                  testDoc.getTimestamp(), testDoc.getLid());
+        }
+    }
+};
+
+
+FeedViewVector
+getVector(const MySubDb &ready,
+          const MySubDb &removed,
+          const MySubDb &notReady)
+{
+    FeedViewVector retval;
+    retval.push_back(ready._view);
+    retval.push_back(removed._view);
+    retval.push_back(notReady._view);
+    return retval;
+}
+
+const uint32_t READY = 0;
+const uint32_t REMOVED = 1;
+const uint32_t NOT_READY = 2;
+
+struct Fixture
+{
+    test::UserDocumentsBuilder      _builder;
+    std::shared_ptr<BucketDBOwner>  _bucketDB;
+    MySubDb                         _ready;
+    MySubDb                         _removed;
+    MySubDb                         _notReady;
+    test::BucketStateCalculator::SP _calc;
+    CombiningFeedView               _view;
+    Fixture() :
+        _builder(),
+        _bucketDB(std::make_shared<BucketDBOwner>()),
+        _ready(_builder.getRepo(), _bucketDB, SubDbType::READY),
+        _removed(_builder.getRepo(), _bucketDB, SubDbType::REMOVED),
+        _notReady(_builder.getRepo(), _bucketDB, SubDbType::NOTREADY),
+        _calc(new test::BucketStateCalculator()),
+        _view(getVector(_ready, _removed, _notReady), _calc)
+    {
+        _builder.createDoc(1, 1);
+        _builder.createDoc(2, 2);
+    }
+    const test::UserDocuments &userDocs() const { return _builder.getDocs(); }
+    const test::BucketDocuments &userDocs(uint32_t userId) const { return userDocs().getUserDocs(userId); }
+    PutOperation put(uint32_t userId) {
+        const test::Document &doc = userDocs().getDocs(userId)[0];
+        return PutOperation(doc.getBucket(), doc.getTimestamp(), doc.getDoc());
+    }
+    RemoveOperation remove(uint32_t userId) {
+        const test::Document &doc = userDocs().getDocs(userId)[0];
+        return RemoveOperation(doc.getBucket(), doc.getTimestamp(), doc.getDoc()->getId());
+    }
+    UpdateOperation update(uint32_t userId) {
+        const test::Document &doc = userDocs().getDocs(userId)[0];
+        return UpdateOperation(doc.getBucket(), doc.getTimestamp(), DocumentUpdate::SP());
+    }
+    MoveOperation move(uint32_t userId, DbDocumentId sourceDbdId, DbDocumentId targetDbdId) {
+        const test::Document &doc = userDocs().getDocs(userId)[0];
+        MoveOperation retval(doc.getBucket(), doc.getTimestamp(), doc.getDoc(),
+                             sourceDbdId, targetDbdId.getSubDbId());
+        retval.setTargetLid(targetDbdId.getLid());
+        return retval;
+    }
+};
+
+
+TEST_F("require that preparePut() sends to ready view", Fixture)
+{
+    PutOperation op = f.put(1);
+    f._calc->addReady(f.userDocs().getBucket(1));
+    f._view.preparePut(op);
+    EXPECT_EQUAL(1u, f._ready._view->_preparePut);
+    EXPECT_EQUAL(0u, f._removed._view->_preparePut);
+    EXPECT_EQUAL(0u, f._notReady._view->_preparePut);
+    EXPECT_FALSE(op.getValidPrevDbdId());
+}
+
+
+TEST_F("require that preparePut() sends to not ready view", Fixture)
+{
+    PutOperation op = f.put(1);
+    f._view.preparePut(op);
+    EXPECT_EQUAL(0u, f._ready._view->_preparePut);
+    EXPECT_EQUAL(0u, f._removed._view->_preparePut);
+    EXPECT_EQUAL(1u, f._notReady._view->_preparePut);
+    EXPECT_FALSE(op.getValidPrevDbdId());
+}
+
+
+TEST_F("require that preparePut() can fill previous dbdId", Fixture)
+{
+    // insert bucket 1 in removed view
+    f._removed.insertDocs(f.userDocs(1));
+    PutOperation op = f.put(1);
+    f._view.preparePut(op);
+    EXPECT_EQUAL(1u, op.getPrevLid());
+    EXPECT_EQUAL(REMOVED, op.getPrevSubDbId());
+    EXPECT_EQUAL(Timestamp(1), op.getPrevTimestamp());
+    EXPECT_TRUE(op.getPrevMarkedAsRemoved());
+}
+
+
+TEST_F("require that handlePut() sends to 1 feed view", Fixture)
+{
+    PutOperation op = f.put(2);
+    op.setDbDocumentId(DbDocumentId(READY, 2));
+    f._view.handlePut(NULL, op);
+    EXPECT_EQUAL(1u, f._ready._view->_handlePut);
+    EXPECT_EQUAL(0u, f._removed._view->_handlePut);
+    EXPECT_EQUAL(0u, f._notReady._view->_handlePut);
+}
+
+
+TEST_F("require that handlePut() sends to 2 feed views", Fixture)
+{
+    PutOperation op = f.put(2);
+    op.setDbDocumentId(DbDocumentId(NOT_READY, 2));
+    op.setPrevDbDocumentId(DbDocumentId(REMOVED, 2));
+    f._view.handlePut(NULL, op);
+    EXPECT_EQUAL(0u, f._ready._view->_handlePut);
+    EXPECT_EQUAL(1u, f._removed._view->_handlePut);
+    EXPECT_EQUAL(1u, f._notReady._view->_handlePut);
+}
+
+
+TEST_F("require that prepareRemove() sends to removed view", Fixture)
+{
+    RemoveOperation op = f.remove(1);
+    f._view.prepareRemove(op);
+    EXPECT_EQUAL(0u, f._ready._view->_prepareRemove);
+    EXPECT_EQUAL(1u, f._removed._view->_prepareRemove);
+    EXPECT_EQUAL(0u, f._notReady._view->_prepareRemove);
+    EXPECT_FALSE(op.getValidPrevDbdId());
+}
+
+
+TEST_F("require that prepareRemove() can fill previous dbdId", Fixture)
+{
+    f._ready.insertDocs(f.userDocs(1));
+    RemoveOperation op = f.remove(1);
+    f._view.prepareRemove(op);
+    EXPECT_EQUAL(1u, op.getPrevLid());
+    EXPECT_EQUAL(READY, op.getPrevSubDbId());
+    EXPECT_EQUAL(Timestamp(1), op.getPrevTimestamp());
+    EXPECT_FALSE(op.getPrevMarkedAsRemoved());
+}
+
+
+TEST_F("require that handleRemove() sends op with valid dbdId to 1 feed view", Fixture)
+{
+    RemoveOperation op = f.remove(1);
+    op.setDbDocumentId(DbDocumentId(REMOVED, 1));
+    f._view.handleRemove(NULL, op);
+    EXPECT_EQUAL(0u, f._ready._view->_handleRemove);
+    EXPECT_EQUAL(1u, f._removed._view->_handleRemove);
+    EXPECT_EQUAL(0u, f._notReady._view->_handleRemove);
+}
+
+
+TEST_F("require that handleRemove() sends op with valid dbdId to 2 feed views", Fixture)
+{
+    RemoveOperation op = f.remove(1);
+    op.setDbDocumentId(DbDocumentId(REMOVED, 1));
+    op.setPrevDbDocumentId(DbDocumentId(READY, 1));
+    f._view.handleRemove(NULL, op);
+    EXPECT_EQUAL(1u, f._ready._view->_handleRemove);
+    EXPECT_EQUAL(1u, f._removed._view->_handleRemove);
+    EXPECT_EQUAL(0u, f._notReady._view->_handleRemove);
+}
+
+
+TEST_F("require that handleRemove() sends op with invalid dbdId to prev view", Fixture)
+{
+    RemoveOperation op = f.remove(1);
+    // can be used in the case where removed feed view does not remember removes.
+    op.setPrevDbDocumentId(DbDocumentId(READY, 1));
+    f._view.handleRemove(NULL, op);
+    EXPECT_EQUAL(1u, f._ready._view->_handleRemove);
+    EXPECT_EQUAL(0u, f._removed._view->_handleRemove);
+    EXPECT_EQUAL(0u, f._notReady._view->_handleRemove);
+}
+
+
+TEST_F("require that prepareUpdate() sends to ready view first", Fixture)
+{
+    UpdateOperation op = f.update(1);
+    // indicate that doc is in ready view
+    op.setPrevDbDocumentId(DbDocumentId(READY, 1));
+    f._view.prepareUpdate(op);
+    EXPECT_EQUAL(1u, f._ready._view->_prepareUpdate);
+    EXPECT_EQUAL(0u, f._removed._view->_prepareUpdate);
+    EXPECT_EQUAL(0u, f._notReady._view->_prepareUpdate);
+}
+
+
+TEST_F("require that prepareUpdate() sends to not ready view if not found in ready view", Fixture)
+{
+    UpdateOperation op = f.update(1);
+    f._view.prepareUpdate(op);
+    EXPECT_EQUAL(1u, f._ready._view->_prepareUpdate);
+    EXPECT_EQUAL(0u, f._removed._view->_prepareUpdate);
+    EXPECT_EQUAL(1u, f._notReady._view->_prepareUpdate);
+}
+
+
+TEST_F("require that handleUpdate() sends op to correct view", Fixture)
+{
+    UpdateOperation op = f.update(1);
+    op.setDbDocumentId(DbDocumentId(READY, 1));
+    op.setPrevDbDocumentId(DbDocumentId(READY, 1));
+    f._view.handleUpdate(NULL, op);
+    EXPECT_EQUAL(1u, f._ready._view->_handleUpdate);
+    EXPECT_EQUAL(0u, f._removed._view->_handleUpdate);
+    EXPECT_EQUAL(0u, f._notReady._view->_handleUpdate);
+}
+
+
+TEST_F("require that prepareMove() sends op to correct feed view", Fixture)
+{
+    MoveOperation op = f.move(1, DbDocumentId(READY, 1), DbDocumentId(NOT_READY, 1));
+    f._view.prepareMove(op);
+    EXPECT_EQUAL(0u, f._ready._view->_prepareMove);
+    EXPECT_EQUAL(0u, f._removed._view->_prepareMove);
+    EXPECT_EQUAL(1u, f._notReady._view->_prepareMove);
+}
+
+
+TEST_F("require that handleMove() sends op to 2 feed views", Fixture)
+{
+    MoveOperation op = f.move(1, DbDocumentId(READY, 1), DbDocumentId(NOT_READY, 1));
+    f._view.handleMove(op);
+    EXPECT_EQUAL(1u, f._ready._view->_handleMove);
+    EXPECT_EQUAL(0u, f._removed._view->_handleMove);
+    EXPECT_EQUAL(1u, f._notReady._view->_handleMove);
+}
+
+
+TEST_F("require that handleMove() sends op to 1 feed view", Fixture)
+{
+    // same source and target
+    MoveOperation op = f.move(1, DbDocumentId(READY, 1), DbDocumentId(READY, 1));
+    f._view.handleMove(op);
+    EXPECT_EQUAL(1u, f._ready._view->_handleMove);
+    EXPECT_EQUAL(0u, f._removed._view->_handleMove);
+    EXPECT_EQUAL(0u, f._notReady._view->_handleMove);
+}
+
+
+TEST_F("require that delete bucket is sent to all feed views", Fixture)
+{
+    DeleteBucketOperation op;
+    f._view.prepareDeleteBucket(op);
+    EXPECT_EQUAL(1u, f._ready._view->_prepareDeleteBucket);
+    EXPECT_EQUAL(1u, f._removed._view->_prepareDeleteBucket);
+    EXPECT_EQUAL(1u, f._notReady._view->_prepareDeleteBucket);
+    f._view.handleDeleteBucket(op);
+    EXPECT_EQUAL(1u, f._ready._view->_handleDeleteBucket);
+    EXPECT_EQUAL(1u, f._removed._view->_handleDeleteBucket);
+    EXPECT_EQUAL(1u, f._notReady._view->_handleDeleteBucket);
+}
+
+
+TEST_F("require that heart beat is sent to all feed views", Fixture)
+{
+    f._view.heartBeat(5);
+    EXPECT_EQUAL(1u, f._ready._view->_heartBeat);
+    EXPECT_EQUAL(1u, f._removed._view->_heartBeat);
+    EXPECT_EQUAL(1u, f._notReady._view->_heartBeat);
+}
+
+
+TEST_F("require that prune removed documents is sent to removed view", Fixture)
+{
+    PruneRemovedDocumentsOperation op;
+    f._view.handlePruneRemovedDocuments(op);
+    EXPECT_EQUAL(0u, f._ready._view->_handlePrune);
+    EXPECT_EQUAL(1u, f._removed._view->_handlePrune);
+    EXPECT_EQUAL(0u, f._notReady._view->_handlePrune);
+}
+
+
+TEST_F("require that calculator can be updated", Fixture)
+{
+    f._calc->addReady(f.userDocs().getBucket(1));
+    PutOperation op1 = f.put(1);
+    PutOperation op2 = f.put(2);
+    {
+        test::BucketStateCalculator::SP calc;
+        f._view.setCalculator(calc);
+        f._view.preparePut(op1);
+        EXPECT_EQUAL(1u, f._ready._view->_preparePut);
+        EXPECT_EQUAL(0u, f._notReady._view->_preparePut);
+        f._view.preparePut(op2);
+        EXPECT_EQUAL(2u, f._ready._view->_preparePut);
+        EXPECT_EQUAL(0u, f._notReady._view->_preparePut);
+    }
+    {
+        test::BucketStateCalculator::SP calc(new test::BucketStateCalculator());
+        calc->addReady(f.userDocs().getBucket(2));
+        f._view.setCalculator(calc);
+        f._view.preparePut(op1);
+        EXPECT_EQUAL(2u, f._ready._view->_preparePut);
+        EXPECT_EQUAL(1u, f._notReady._view->_preparePut);
+        f._view.preparePut(op2);
+        EXPECT_EQUAL(3u, f._ready._view->_preparePut);
+        EXPECT_EQUAL(1u, f._notReady._view->_preparePut);
+    }
+    {
+        test::BucketStateCalculator::SP calc(new test::BucketStateCalculator());
+        calc->setClusterUp(false);
+        f._view.setCalculator(calc);
+        f._view.preparePut(op1);
+        EXPECT_EQUAL(4u, f._ready._view->_preparePut);
+        EXPECT_EQUAL(1u, f._notReady._view->_preparePut);
+        f._view.preparePut(op2);
+        EXPECT_EQUAL(5u, f._ready._view->_preparePut);
+        EXPECT_EQUAL(1u, f._notReady._view->_preparePut);
+    }
+}
+
+TEST_F("require that compactLidSpace() is sent to correct feed view", Fixture)
+{
+    f._view.handleCompactLidSpace(CompactLidSpaceOperation(1, 99));
+    EXPECT_EQUAL(0u, f._ready._view->_wantedLidLimit);
+    EXPECT_EQUAL(99u, f._removed._view->_wantedLidLimit);
+    EXPECT_EQUAL(0u, f._notReady._view->_wantedLidLimit);
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
+
diff --git a/searchcore/src/tests/proton/documentdb/configurer/.gitignore b/searchcore/src/tests/proton/documentdb/configurer/.gitignore
new file mode 100644
index 00000000000..3714f1b204d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configurer/.gitignore
@@ -0,0 +1 @@
+searchcore_configurer_test_app
diff --git a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt
new file mode 100644
index 00000000000..ee18f0f6938
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt
@@ -0,0 +1,22 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_configurer_test_app
+    SOURCES
+    configurer_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_reprocessing
+    searchcore_index
+    searchcore_docsummary
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_flushengine
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_fconfig
+    searchcore_util
+)
+vespa_add_test(NAME searchcore_configurer_test_app COMMAND searchcore_configurer_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/configurer/DESC b/searchcore/src/tests/proton/documentdb/configurer/DESC
new file mode 100644
index 00000000000..5d7765db8d2
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configurer/DESC
@@ -0,0 +1 @@
+configurer test. Take a look at configurer_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/configurer/FILES b/searchcore/src/tests/proton/documentdb/configurer/FILES
new file mode 100644
index 00000000000..a7ff508edc0
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configurer/FILES
@@ -0,0 +1 @@
+configurer_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
new file mode 100644
index 00000000000..1764d6f2996
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -0,0 +1,611 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("configurer_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+#include <vespa/searchcore/proton/index/index_writer.h>
+#include <vespa/searchcore/proton/index/indexmanager.h>
+#include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h>
+#include <vespa/searchcore/proton/server/attributeadapterfactory.h>
+#include <vespa/searchcore/proton/server/documentdbconfigmanager.h>
+#include <vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+#include <vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h>
+#include <vespa/searchcore/proton/server/searchable_feed_view.h>
+#include <vespa/searchcore/proton/server/matchers.h>
+#include <vespa/searchcore/proton/server/summaryadapter.h>
+#include <vespa/searchcore/proton/common/commit_time_tracker.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/transactionlog/nosyncproxy.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+using namespace config;
+using namespace document;
+using namespace proton;
+using namespace search::grouping;
+using namespace search::index;
+using namespace search::queryeval;
+using namespace search;
+using namespace vespa::config::search::core;
+using namespace vespa::config::search::summary;
+using namespace vespa::config::search;
+using namespace vespalib;
+
+using document::DocumenttypesConfig;
+using fastos::TimeStamp;
+using proton::matching::SessionManager;
+using searchcorespi::IndexSearchable;
+using searchcorespi::index::IThreadingService;
+
+
+typedef DocumentDBConfig::ComparisonResult ConfigComparisonResult;
+typedef SearchableDocSubDBConfigurer Configurer;
+typedef std::unique_ptr<SearchableDocSubDBConfigurer> ConfigurerUP;
+typedef SummaryManager::SummarySetup SummarySetup;
+typedef proton::DocumentDBConfig::DocumenttypesConfigSP DocumenttypesConfigSP;
+
+const vespalib::string BASE_DIR("baseDir");
+const vespalib::string DOC_TYPE("invalid");
+
+class IndexManagerDummyReconfigurer : public searchcorespi::IIndexManager::Reconfigurer
+{
+    virtual bool reconfigure(vespalib::Closure0<bool>::UP closure) {
+        bool ret = true;
+        if (closure.get() != NULL)
+            ret = closure->call(); // Perform index manager reconfiguration now
+        return ret;
+    }
+};
+
+DocumentTypeRepo::SP
+createRepo()
+{
+    DocumentType docType(DOC_TYPE, 0);
+    return DocumentTypeRepo::SP(new DocumentTypeRepo(docType));
+}
+
+struct ViewPtrs
+{
+    SearchView::SP sv;
+    SearchableFeedView::SP fv;
+};
+
+struct ViewSet
+{
+    IndexManagerDummyReconfigurer _reconfigurer;
+    DummyFileHeaderContext _fileHeaderContext;
+    ExecutorThreadingService _writeService;
+    SearchableFeedView::SerialNum serialNum;
+    DocumentTypeRepo::SP repo;
+    DocTypeName _docTypeName;
+    DocIdLimit _docIdLimit;
+    search::transactionlog::NoSyncProxy _noTlSyncer;
+    ISummaryManager::SP _summaryMgr;
+    IDocumentMetaStoreContext::SP _dmsc;
+    std::unique_ptr<documentmetastore::ILidReuseDelayer> _lidReuseDelayer;
+    CommitTimeTracker _commitTimeTracker;
+    VarHolder<SearchView::SP> searchView;
+    VarHolder<SearchableFeedView::SP> feedView;
+    ViewSet()
+        : _reconfigurer(),
+          _fileHeaderContext(),
+          _writeService(),
+          serialNum(1),
+          repo(createRepo()),
+          _docTypeName(DOC_TYPE),
+          _docIdLimit(0u),
+          _noTlSyncer(),
+          _summaryMgr(),
+          _dmsc(),
+          _lidReuseDelayer(),
+          _commitTimeTracker(TimeStamp()),
+          searchView(),
+          feedView()
+    {
+    }
+
+    ViewPtrs getViewPtrs() {
+        ViewPtrs ptrs;
+        ptrs.sv = searchView.get();
+        ptrs.fv = feedView.get();
+        return ptrs;
+    }
+};
+
+struct Fixture
+{
+    vespalib::Clock _clock;
+    matching::QueryLimiter _queryLimiter;
+    vespalib::ThreadStackExecutor _summaryExecutor;
+    ViewSet _views;
+    ConfigurerUP _configurer;
+    Fixture()
+        : _clock(),
+          _queryLimiter(),
+          _summaryExecutor(8, 128*1024),
+          _views(),
+          _configurer()
+    {
+        vespalib::mkdir(BASE_DIR);
+        initViewSet(_views);
+        _configurer.reset(new Configurer(_views._summaryMgr,
+                                         _views.searchView,
+                                         _views.feedView,
+                                         _queryLimiter,
+                                         _clock,
+                                         "test",
+                                         0));
+    }
+    ~Fixture() {
+        vespalib::rmdir(BASE_DIR, true);
+    }
+    void initViewSet(ViewSet &views);
+};
+
+void
+Fixture::initViewSet(ViewSet &views)
+{
+    Matchers::SP matchers(new Matchers(_clock, _queryLimiter));
+    IndexManager::SP indexMgr(new IndexManager(BASE_DIR,
+                                      0.0, 2, 0, Schema(), Schema(), views._reconfigurer,
+                                      views._writeService, _summaryExecutor, TuneFileIndexManager(),
+                                      TuneFileAttributes(), views._fileHeaderContext));
+    AttributeManager::SP attrMgr(new AttributeManager(BASE_DIR,
+                                                      "test.subdb",
+                                                      TuneFileAttributes(),
+                                                      views._fileHeaderContext,
+                                                      views._writeService.
+                                                      attributeFieldWriter()));
+    ProtonConfig protonCfg;
+    SummaryManager::SP summaryMgr(
+            new SummaryManager(_summaryExecutor, ProtonConfig::Summary(),
+                               GrowStrategy(), BASE_DIR, views._docTypeName,
+                               TuneFileSummary(), views._fileHeaderContext,
+                               views._noTlSyncer, search::IBucketizer::SP()));
+    SessionManager::SP sesMgr(
+            new SessionManager(protonCfg.grouping.sessionmanager.maxentries));
+    DocumentMetaStoreContext::SP metaStore(
+            new DocumentMetaStoreContext(std::make_shared<BucketDBOwner>()));
+    IIndexWriter::SP indexWriter(new IndexWriter(indexMgr));
+    AttributeWriter::SP attrWriter(new AttributeWriter(attrMgr));
+    ISummaryAdapter::SP summaryAdapter(new SummaryAdapter(summaryMgr));
+    Schema::SP schema(new Schema());
+    views._summaryMgr = summaryMgr;
+    views._dmsc = metaStore;
+    views._lidReuseDelayer.reset(
+            new documentmetastore::LidReuseDelayer(views._writeService,
+                                                   metaStore->get()));
+    IndexSearchable::SP indexSearchable;
+    MatchView::SP matchView(new MatchView(matchers, indexSearchable, attrMgr,
+                                          sesMgr, metaStore, views._docIdLimit));
+    views.searchView.set(
+            SearchView::SP(
+                    new SearchView(
+                            summaryMgr->createSummarySetup(SummaryConfig(),
+                                    SummarymapConfig(),
+                                    JuniperrcConfig(),
+                                    views.repo,
+                                    attrMgr),
+                            matchView)));
+    PerDocTypeFeedMetrics metrics(0);
+    views.feedView.set(
+            SearchableFeedView::SP(
+                    new SearchableFeedView(StoreOnlyFeedView::Context(summaryAdapter,
+                            schema,
+                            views.searchView.get()->getDocumentMetaStore(),
+                            views.repo,
+                            views._writeService,
+                            *views._lidReuseDelayer,
+                            views._commitTimeTracker),
+                            SearchableFeedView::PersistentParams(
+                                    views.serialNum,
+                                    views.serialNum,
+                                    views._docTypeName,
+                                    metrics,
+                                    0u /* subDbId */,
+                                    SubDbType::READY),
+                            FastAccessFeedView::Context(attrWriter, views._docIdLimit),
+                            SearchableFeedView::Context(indexWriter))));
+}
+
+
+struct MySummaryAdapter : public ISummaryAdapter
+{
+    virtual void put(search::SerialNum, const document::Document &, const search::DocumentIdT) {}
+    virtual void remove(search::SerialNum, const search::DocumentIdT) {}
+    virtual void update(search::SerialNum, const document::DocumentUpdate &,
+            const search::DocumentIdT, const document::DocumentTypeRepo &) {}
+    virtual void heartBeat(search::SerialNum) {}
+    virtual const search::IDocumentStore &getDocumentStore() const {
+        const search::IDocumentStore *store = NULL;
+        return *store;
+    }
+    virtual std::unique_ptr<document::Document> get(const search::DocumentIdT,
+            const document::DocumentTypeRepo &) {
+        return std::unique_ptr<document::Document>();
+    }
+};
+
+struct MyFastAccessFeedView
+{
+    PerDocTypeFeedMetrics _metrics;
+    DummyFileHeaderContext _fileHeaderContext;
+    DocIdLimit _docIdLimit;
+    IThreadingService &_writeService;
+    IDocumentMetaStoreContext::SP _dmsc;
+    std::unique_ptr<documentmetastore::ILidReuseDelayer> _lidReuseDelayer;
+    CommitTimeTracker                 _commitTimeTracker;
+    VarHolder<FastAccessFeedView::SP> _feedView;
+
+    MyFastAccessFeedView(IThreadingService &writeService)
+        : _metrics(0),
+          _fileHeaderContext(),
+          _docIdLimit(0),
+          _writeService(writeService),
+          _dmsc(),
+          _lidReuseDelayer(),
+          _commitTimeTracker(TimeStamp()),
+          _feedView()
+    {
+        init();
+    }
+    void init() {
+        ISummaryAdapter::SP summaryAdapter(new MySummaryAdapter());
+        Schema::SP schema(new Schema());
+        DocumentMetaStoreContext::SP docMetaCtx(
+                new DocumentMetaStoreContext(std::make_shared<BucketDBOwner>()));
+        _dmsc = docMetaCtx;
+        _lidReuseDelayer.reset(
+                new documentmetastore::LidReuseDelayer(_writeService,
+                                                       docMetaCtx->get()));
+        DocumentTypeRepo::SP repo = createRepo();
+        StoreOnlyFeedView::Context storeOnlyCtx(summaryAdapter, schema, docMetaCtx, repo, _writeService, *_lidReuseDelayer, _commitTimeTracker);
+        StoreOnlyFeedView::PersistentParams params(1, 1, DocTypeName(DOC_TYPE), _metrics, 0, SubDbType::NOTREADY);
+        AttributeManager::SP mgr(new AttributeManager(BASE_DIR, "test.subdb",
+                                                      TuneFileAttributes(),
+                                                      _fileHeaderContext,
+                                                      _writeService.
+                                                      attributeFieldWriter()));
+        IAttributeWriter::SP writer(new AttributeWriter(mgr));
+        FastAccessFeedView::Context fastUpdateCtx(writer, _docIdLimit);
+        _feedView.set(FastAccessFeedView::SP(new FastAccessFeedView(storeOnlyCtx,
+                params, fastUpdateCtx)));;
+    }
+};
+
+struct FastAccessFixture
+{
+    ExecutorThreadingService _writeService;
+    MyFastAccessFeedView _view;
+    FastAccessDocSubDBConfigurer _configurer;
+    FastAccessFixture()
+        : _writeService(),
+          _view(_writeService),
+          _configurer(_view._feedView,
+                  IAttributeAdapterFactory::UP(new AttributeAdapterFactory), "test")
+    {
+        vespalib::mkdir(BASE_DIR);
+    }
+    ~FastAccessFixture() {
+        _writeService.sync();
+        vespalib::rmdir(BASE_DIR, true);
+    }
+};
+
+
+DocumentDBConfig::SP
+createConfig()
+{
+    DocumentDBConfig::SP config
+        (new DocumentDBConfig(
+                0,
+                DocumentDBConfig::RankProfilesConfigSP(
+                        new RankProfilesConfig()),
+                DocumentDBConfig::IndexschemaConfigSP(new IndexschemaConfig()),
+                DocumentDBConfig::AttributesConfigSP(new AttributesConfig()),
+                DocumentDBConfig::SummaryConfigSP(new SummaryConfig()),
+                DocumentDBConfig::SummarymapConfigSP(new SummarymapConfig()),
+                DocumentDBConfig::JuniperrcConfigSP(new JuniperrcConfig()),
+                DocumenttypesConfigSP(new DocumenttypesConfig()),
+                DocumentTypeRepo::SP(createRepo()),
+                TuneFileDocumentDB::SP(new TuneFileDocumentDB),
+                Schema::SP(new Schema),
+                DocumentDBMaintenanceConfig::SP(
+                        new DocumentDBMaintenanceConfig),
+                "client", DOC_TYPE));
+    return config;
+}
+
+DocumentDBConfig::SP
+createConfig(const Schema::SP &schema)
+{
+    DocumentDBConfig::SP config
+        (new DocumentDBConfig(
+                0,
+                DocumentDBConfig::RankProfilesConfigSP(new RankProfilesConfig()),
+                DocumentDBConfig::IndexschemaConfigSP(new IndexschemaConfig()),
+                DocumentDBConfig::AttributesConfigSP(new AttributesConfig()),
+                DocumentDBConfig::SummaryConfigSP(new SummaryConfig()),
+                DocumentDBConfig::SummarymapConfigSP(new SummarymapConfig()),
+                DocumentDBConfig::JuniperrcConfigSP(new JuniperrcConfig()),
+                DocumenttypesConfigSP(new DocumenttypesConfig()),
+                DocumentTypeRepo::SP(createRepo()),
+                TuneFileDocumentDB::SP(new TuneFileDocumentDB),
+                schema,
+                DocumentDBMaintenanceConfig::SP(
+                        new DocumentDBMaintenanceConfig),
+                "client", DOC_TYPE));
+    return config;
+}
+
+struct SearchViewComparer
+{
+    SearchView::SP _old;
+    SearchView::SP _new;
+    SearchViewComparer(SearchView::SP old, SearchView::SP new_) : _old(old), _new(new_) {}
+    void expect_equal() {
+        EXPECT_EQUAL(_old.get(), _new.get());
+    }
+    void expect_not_equal() {
+        EXPECT_NOT_EQUAL(_old.get(), _new.get());
+    }
+    void expect_equal_summary_setup() {
+        EXPECT_EQUAL(_old->getSummarySetup().get(), _new->getSummarySetup().get());
+    }
+    void expect_not_equal_summary_setup() {
+        EXPECT_NOT_EQUAL(_old->getSummarySetup().get(), _new->getSummarySetup().get());
+    }
+    void expect_equal_match_view() {
+        EXPECT_EQUAL(_old->getMatchView().get(), _new->getMatchView().get());
+    }
+    void expect_not_equal_match_view() {
+        EXPECT_NOT_EQUAL(_old->getMatchView().get(), _new->getMatchView().get());
+    }
+    void expect_equal_matchers() {
+        EXPECT_EQUAL(_old->getMatchers().get(), _new->getMatchers().get());
+    }
+    void expect_not_equal_matchers() {
+        EXPECT_NOT_EQUAL(_old->getMatchers().get(), _new->getMatchers().get());
+    }
+    void expect_equal_index_searchable() {
+        EXPECT_EQUAL(_old->getIndexSearchable().get(), _new->getIndexSearchable().get());
+    }
+    void expect_not_equal_index_searchable() {
+        EXPECT_NOT_EQUAL(_old->getIndexSearchable().get(), _new->getIndexSearchable().get());
+    }
+    void expect_equal_attribute_manager() {
+        EXPECT_EQUAL(_old->getAttributeManager().get(), _new->getAttributeManager().get());
+    }
+    void expect_not_equal_attribute_manager() {
+        EXPECT_NOT_EQUAL(_old->getAttributeManager().get(), _new->getAttributeManager().get());
+    }
+    void expect_equal_session_manager() {
+        EXPECT_EQUAL(_old->getSessionManager().get(), _new->getSessionManager().get());
+    }
+    void expect_equal_document_meta_store() {
+        EXPECT_EQUAL(_old->getDocumentMetaStore().get(), _new->getDocumentMetaStore().get());
+    }
+};
+
+struct FeedViewComparer
+{
+    SearchableFeedView::SP _old;
+    SearchableFeedView::SP _new;
+    FeedViewComparer(SearchableFeedView::SP old, SearchableFeedView::SP new_) : _old(old), _new(new_) {}
+    void expect_equal() {
+        EXPECT_EQUAL(_old.get(), _new.get());
+    }
+    void expect_not_equal() {
+        EXPECT_NOT_EQUAL(_old.get(), _new.get());
+    }
+    void expect_equal_index_adapter() {
+        EXPECT_EQUAL(_old->getIndexWriter().get(), _new->getIndexWriter().get());
+    }
+    void expect_equal_attribute_adapter() {
+        EXPECT_EQUAL(_old->getAttributeWriter().get(), _new->getAttributeWriter().get());
+    }
+    void expect_not_equal_attribute_adapter() {
+        EXPECT_NOT_EQUAL(_old->getAttributeWriter().get(), _new->getAttributeWriter().get());
+    }
+    void expect_equal_summary_adapter() {
+        EXPECT_EQUAL(_old->getSummaryAdapter().get(), _new->getSummaryAdapter().get());
+    }
+    void expect_equal_schema() {
+        EXPECT_EQUAL(_old->getSchema().get(), _new->getSchema().get());
+    }
+    void expect_not_equal_schema() {
+        EXPECT_NOT_EQUAL(_old->getSchema().get(), _new->getSchema().get());
+    }
+};
+
+struct FastAccessFeedViewComparer
+{
+    FastAccessFeedView::SP _old;
+    FastAccessFeedView::SP _new;
+    FastAccessFeedViewComparer(FastAccessFeedView::SP old, FastAccessFeedView::SP new_)
+        : _old(old), _new(new_)
+    {}
+    void expect_not_equal() {
+        EXPECT_NOT_EQUAL(_old.get(), _new.get());
+    }
+    void expect_not_equal_attribute_adapter() {
+        EXPECT_NOT_EQUAL(_old->getAttributeWriter().get(), _new->getAttributeWriter().get());
+    }
+    void expect_equal_summary_adapter() {
+        EXPECT_EQUAL(_old->getSummaryAdapter().get(), _new->getSummaryAdapter().get());
+    }
+    void expect_not_equal_schema() {
+        EXPECT_NOT_EQUAL(_old->getSchema().get(), _new->getSchema().get());
+    }
+};
+
+TEST_F("require that we can reconfigure index searchable", Fixture)
+{
+    ViewPtrs o = f._views.getViewPtrs();
+    f._configurer->reconfigureIndexSearchable();
+
+    ViewPtrs n = f._views.getViewPtrs();
+    { // verify search view
+        SearchViewComparer cmp(o.sv, n.sv);
+        cmp.expect_not_equal();
+        cmp.expect_equal_summary_setup();
+        cmp.expect_not_equal_match_view();
+        cmp.expect_equal_matchers();
+        cmp.expect_not_equal_index_searchable();
+        cmp.expect_equal_attribute_manager();
+        cmp.expect_equal_session_manager();
+        cmp.expect_equal_document_meta_store();
+    }
+    { // verify feed view
+        FeedViewComparer cmp(o.fv, n.fv);
+        cmp.expect_not_equal();
+        cmp.expect_equal_index_adapter();
+        cmp.expect_equal_attribute_adapter();
+        cmp.expect_equal_summary_adapter();
+        cmp.expect_equal_schema();
+    }
+}
+
+TEST_F("require that we can reconfigure attribute manager", Fixture)
+{
+    ViewPtrs o = f._views.getViewPtrs();
+    ConfigComparisonResult cmpres;
+    cmpres.attributesChanged = true;
+    cmpres._schemaChanged = true;
+    AttributeCollectionSpec::AttributeList specList;
+    AttributeCollectionSpec spec(specList, 1, 0);
+    ReconfigParams params(cmpres);
+    // Use new config snapshot == old config snapshot (only relevant for reprocessing)
+    f._configurer->reconfigure(*createConfig(), *createConfig(), spec, params);
+
+    ViewPtrs n = f._views.getViewPtrs();
+    { // verify search view
+        SearchViewComparer cmp(o.sv, n.sv);
+        cmp.expect_not_equal();
+        cmp.expect_not_equal_summary_setup();
+        cmp.expect_not_equal_match_view();
+        cmp.expect_not_equal_matchers();
+        cmp.expect_equal_index_searchable();
+        cmp.expect_not_equal_attribute_manager();
+        cmp.expect_equal_session_manager();
+        cmp.expect_equal_document_meta_store();
+    }
+    { // verify feed view
+        FeedViewComparer cmp(o.fv, n.fv);
+        cmp.expect_not_equal();
+        cmp.expect_equal_index_adapter();
+        cmp.expect_not_equal_attribute_adapter();
+        cmp.expect_equal_summary_adapter();
+        cmp.expect_not_equal_schema();
+    }
+}
+
+TEST_F("require that reconfigure returns reprocessing initializer when changing attributes", Fixture)
+{
+    ConfigComparisonResult cmpres;
+    cmpres.attributesChanged = true;
+    cmpres._schemaChanged = true;
+    AttributeCollectionSpec::AttributeList specList;
+    AttributeCollectionSpec spec(specList, 1, 0);
+    ReconfigParams params(cmpres);
+    IReprocessingInitializer::UP init =
+            f._configurer->reconfigure(*createConfig(), *createConfig(), spec, params);
+
+    EXPECT_TRUE(init.get() != nullptr);
+    EXPECT_TRUE((dynamic_cast<AttributeReprocessingInitializer *>(init.get())) != nullptr);
+    EXPECT_FALSE(init->hasReprocessors());
+}
+
+TEST_F("require that we can reconfigure attribute adapter", FastAccessFixture)
+{
+    AttributeCollectionSpec::AttributeList specList;
+    AttributeCollectionSpec spec(specList, 1, 0);
+    FastAccessFeedView::SP o = f._view._feedView.get();
+    f._configurer.reconfigure(*createConfig(), *createConfig(), spec);
+    FastAccessFeedView::SP n = f._view._feedView.get();
+
+    FastAccessFeedViewComparer cmp(o, n);
+    cmp.expect_not_equal();
+    cmp.expect_not_equal_attribute_adapter();
+    cmp.expect_equal_summary_adapter();
+    cmp.expect_not_equal_schema();
+}
+
+TEST_F("require that reconfigure returns reprocessing initializer", FastAccessFixture)
+{
+    AttributeCollectionSpec::AttributeList specList;
+    AttributeCollectionSpec spec(specList, 1, 0);
+    IReprocessingInitializer::UP init =
+            f._configurer.reconfigure(*createConfig(), *createConfig(), spec);
+
+    EXPECT_TRUE(init.get() != nullptr);
+    EXPECT_TRUE((dynamic_cast<AttributeReprocessingInitializer *>(init.get())) != nullptr);
+    EXPECT_FALSE(init->hasReprocessors());
+}
+
+TEST_F("require that we can reconfigure summary manager", Fixture)
+{
+    ViewPtrs o = f._views.getViewPtrs();
+    ConfigComparisonResult cmpres;
+    cmpres.summarymapChanged = true;
+    ReconfigParams params(cmpres);
+    // Use new config snapshot == old config snapshot (only relevant for reprocessing)
+    f._configurer->reconfigure(*createConfig(), *createConfig(), params);
+
+    ViewPtrs n = f._views.getViewPtrs();
+    { // verify search view
+        SearchViewComparer cmp(o.sv, n.sv);
+        cmp.expect_not_equal();
+        cmp.expect_not_equal_summary_setup();
+        cmp.expect_equal_match_view();
+    }
+    { // verify feed view
+        FeedViewComparer cmp(o.fv, n.fv);
+        cmp.expect_equal();
+    }
+}
+
+TEST_F("require that we can reconfigure matchers", Fixture)
+{
+    ViewPtrs o = f._views.getViewPtrs();
+    ConfigComparisonResult cmpres;
+    cmpres.rankProfilesChanged = true;
+    // Use new config snapshot == old config snapshot (only relevant for reprocessing)
+    f._configurer->reconfigure(*createConfig(o.fv->getSchema()), *createConfig(o.fv->getSchema()),
+            ReconfigParams(cmpres));
+
+    ViewPtrs n = f._views.getViewPtrs();
+    { // verify search view
+        SearchViewComparer cmp(o.sv, n.sv);
+        cmp.expect_not_equal();
+        cmp.expect_equal_summary_setup();
+        cmp.expect_not_equal_match_view();
+        cmp.expect_not_equal_matchers();
+        cmp.expect_equal_index_searchable();
+        cmp.expect_equal_attribute_manager();
+        cmp.expect_equal_session_manager();
+        cmp.expect_equal_document_meta_store();
+    }
+    { // verify feed view
+        FeedViewComparer cmp(o.fv, n.fv);
+        cmp.expect_not_equal();
+        cmp.expect_equal_index_adapter();
+        cmp.expect_equal_attribute_adapter();
+        cmp.expect_equal_summary_adapter();
+        cmp.expect_equal_schema();
+    }
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/documentdb/configvalidator/.gitignore b/searchcore/src/tests/proton/documentdb/configvalidator/.gitignore
new file mode 100644
index 00000000000..2a8675dad8d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configvalidator/.gitignore
@@ -0,0 +1 @@
+searchcore_configvalidator_test_app
diff --git a/searchcore/src/tests/proton/documentdb/configvalidator/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/configvalidator/CMakeLists.txt
new file mode 100644
index 00000000000..c7a3a6235cf
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configvalidator/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_configvalidator_test_app
+    SOURCES
+    configvalidator_test.cpp
+    DEPENDS
+    searchcore_server
+)
+vespa_add_test(NAME searchcore_configvalidator_test_app COMMAND searchcore_configvalidator_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/configvalidator/DESC b/searchcore/src/tests/proton/documentdb/configvalidator/DESC
new file mode 100644
index 00000000000..9263515a290
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configvalidator/DESC
@@ -0,0 +1 @@
+configvalidator test. Take a look at configvalidator_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/configvalidator/FILES b/searchcore/src/tests/proton/documentdb/configvalidator/FILES
new file mode 100644
index 00000000000..a7acf2f384c
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configvalidator/FILES
@@ -0,0 +1 @@
+configvalidator_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/configvalidator/configvalidator_test.cpp b/searchcore/src/tests/proton/documentdb/configvalidator/configvalidator_test.cpp
new file mode 100644
index 00000000000..cbcc97bdf68
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/configvalidator/configvalidator_test.cpp
@@ -0,0 +1,351 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("configvalidator_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/server/configvalidator.h>
+
+using namespace proton;
+using namespace search::index;
+using vespa::config::search::AttributesConfig;
+using vespa::config::search::AttributesConfigBuilder;
+
+typedef Schema::AttributeField AField;
+typedef Schema::IndexField     IField;
+typedef Schema::SummaryField   SField;
+
+const ConfigValidator::ResultType OK  = ConfigValidator::OK;
+const ConfigValidator::ResultType DTC = ConfigValidator::DATA_TYPE_CHANGED;
+const ConfigValidator::ResultType CTC = ConfigValidator::COLLECTION_TYPE_CHANGED;
+const ConfigValidator::ResultType IAA = ConfigValidator::INDEX_ASPECT_ADDED;
+const ConfigValidator::ResultType IAR = ConfigValidator::INDEX_ASPECT_REMOVED;
+const ConfigValidator::ResultType AAA = ConfigValidator::ATTRIBUTE_ASPECT_ADDED;
+const ConfigValidator::ResultType AAR = ConfigValidator::ATTRIBUTE_ASPECT_REMOVED;
+const ConfigValidator::ResultType AFAA = ConfigValidator::ATTRIBUTE_FAST_ACCESS_ADDED;
+const ConfigValidator::ResultType AFAR = ConfigValidator::ATTRIBUTE_FAST_ACCESS_REMOVED;
+
+enum FType {
+    INDEX,
+    ATTRIBUTE,
+    SUMMARY
+};
+
+struct SchemaBuilder
+{
+    Schema _schema;
+    SchemaBuilder() : _schema() {}
+    SchemaBuilder &add(const vespalib::string &name, FType ftype,
+                       Schema::DataType dtype, Schema::CollectionType ctype = Schema::SINGLE) {
+        switch (ftype) {
+        case INDEX:
+            _schema.addIndexField(IField(name, dtype, ctype));
+            break;
+        case ATTRIBUTE:
+            _schema.addAttributeField(AField(name, dtype, ctype));
+            break;
+        case SUMMARY:
+            _schema.addSummaryField(SField(name, dtype, ctype));
+            break;
+        }
+        return *this;
+    }
+    const Schema &schema() const { return _schema; }
+};
+
+Schema
+create(FType ftype, Schema::DataType dtype, Schema::CollectionType ctype)
+{
+    SchemaBuilder bld;
+    return bld.add("f1", ftype, dtype, ctype).schema();
+}
+
+Schema
+created(FType ftype, Schema::DataType dtype)
+{
+    return create(ftype, dtype, Schema::SINGLE);
+}
+
+Schema
+createc(FType ftype, Schema::CollectionType ctype)
+{
+    return create(ftype, Schema::STRING, ctype);
+}
+
+ConfigValidator::ResultType
+checkSchema(const Schema &newSchema,
+            const Schema &oldSchema,
+            const Schema &oldHistory)
+{
+    return ConfigValidator::validate(ConfigValidator::Config(newSchema, AttributesConfig()),
+            ConfigValidator::Config(oldSchema, AttributesConfig()), oldHistory).type();
+}
+
+ConfigValidator::ResultType
+checkAttribute(const AttributesConfig &newCfg,
+               const AttributesConfig &oldCfg)
+{
+    return ConfigValidator::validate(ConfigValidator::Config(Schema(), newCfg),
+            ConfigValidator::Config(Schema(), oldCfg), Schema()).type();
+}
+
+void
+requireThatChangedDataTypeIsDiscovered(FType ftype)
+{
+    EXPECT_EQUAL(DTC,
+                 checkSchema(created(ftype, Schema::INT32),
+                         created(ftype, Schema::STRING),
+                         Schema()));
+    EXPECT_EQUAL(DTC,
+                 checkSchema(created(ftype, Schema::INT32),
+                         Schema(),
+                         created(ftype, Schema::STRING)));
+}
+
+TEST("require that changed data type is discovered")
+{
+    requireThatChangedDataTypeIsDiscovered(INDEX);
+    requireThatChangedDataTypeIsDiscovered(ATTRIBUTE);
+    requireThatChangedDataTypeIsDiscovered(SUMMARY);
+}
+
+void
+requireThatChangedCollectionTypeIsDiscovered(FType ftype)
+{
+    EXPECT_EQUAL(CTC,
+                 checkSchema(createc(ftype, Schema::ARRAY),
+                         createc(ftype, Schema::SINGLE),
+                         Schema()));
+    EXPECT_EQUAL(CTC,
+                 checkSchema(createc(ftype, Schema::ARRAY),
+                         Schema(),
+                         createc(ftype, Schema::SINGLE)));
+}
+
+TEST("require that changed collection type is discovered")
+{
+    requireThatChangedCollectionTypeIsDiscovered(INDEX);
+    requireThatChangedCollectionTypeIsDiscovered(ATTRIBUTE);
+    requireThatChangedCollectionTypeIsDiscovered(SUMMARY);
+}
+
+TEST("require that changed index aspect is discovered")
+{
+    Schema s1 = created(SUMMARY, Schema::STRING);
+    s1.addIndexField(IField("f1", Schema::STRING));
+    Schema s2 = created(SUMMARY, Schema::STRING);
+    Schema s2h = created(INDEX, Schema::STRING);
+
+    Schema s3 = created(ATTRIBUTE, Schema::STRING);
+    s3.addIndexField(IField("f1", Schema::STRING));
+    Schema s4 = created(ATTRIBUTE, Schema::STRING);
+    Schema s4h = created(INDEX, Schema::STRING);
+    { // remove as index field
+        EXPECT_EQUAL(IAR, checkSchema(s2, s1, Schema()));
+        EXPECT_EQUAL(IAR, checkSchema(s2, Schema(), s1));
+        EXPECT_EQUAL(IAR, checkSchema(s4, s3, Schema()));
+        EXPECT_EQUAL(IAR, checkSchema(s4, Schema(), s3));
+    }
+    {
+        // undo field removal
+        EXPECT_EQUAL(OK, checkSchema(s1, Schema(), s1));
+        EXPECT_EQUAL(OK, checkSchema(s3, Schema(), s3));
+    }
+    { // add as index field
+        EXPECT_EQUAL(IAA, checkSchema(s1, s2, Schema()));
+        EXPECT_EQUAL(IAA, checkSchema(s1, s2, s2h));
+        EXPECT_EQUAL(IAA, checkSchema(s1, Schema(), s2));
+        EXPECT_EQUAL(IAA, checkSchema(s3, s4, Schema()));
+        EXPECT_EQUAL(IAA, checkSchema(s3, s4, s4h));
+        EXPECT_EQUAL(IAA, checkSchema(s3, Schema(), s4));
+    }
+}
+
+TEST("require that changed attribute aspect is discovered")
+{
+    Schema s1 = created(SUMMARY, Schema::STRING);
+    s1.addAttributeField(AField("f1", Schema::STRING));
+    Schema s2 = created(SUMMARY, Schema::STRING);
+    Schema s2h = created(ATTRIBUTE, Schema::STRING);
+
+    Schema s3 = created(INDEX, Schema::STRING);
+    s3.addAttributeField(AField("f1", Schema::STRING));
+    Schema s4 = created(INDEX, Schema::STRING);
+    Schema s4h = created(ATTRIBUTE, Schema::STRING);
+
+    Schema s5 = created(INDEX, Schema::STRING);
+    s5.addSummaryField(SField("f1", Schema::STRING));
+    s5.addAttributeField(AField("f1", Schema::STRING));
+    Schema s6 = created(INDEX, Schema::STRING);
+    s6.addSummaryField(SField("f1", Schema::STRING));
+    { // remove as attribute field
+        EXPECT_EQUAL(AAR, checkSchema(s2, s1, Schema()));
+        EXPECT_EQUAL(AAR, checkSchema(s2, Schema(), s1));
+        // remove as attribute is allowed when still existing as index.
+        EXPECT_EQUAL(OK, checkSchema(s4, s3, Schema()));
+        EXPECT_EQUAL(OK, checkSchema(s6, s5, Schema()));
+        EXPECT_EQUAL(IAA, checkSchema(s4, Schema(), s3));
+    }
+    {
+        // undo field removal
+        EXPECT_EQUAL(OK, checkSchema(s1, Schema(), s1));
+        EXPECT_EQUAL(OK, checkSchema(s3, Schema(), s3));
+    }
+    { // add as attribute field
+        EXPECT_EQUAL(AAA, checkSchema(s1, s2, Schema()));
+        EXPECT_EQUAL(AAA, checkSchema(s1, s2, s2h));
+        EXPECT_EQUAL(AAA, checkSchema(s1, Schema(), s2));
+        EXPECT_EQUAL(AAA, checkSchema(s3, s4, Schema()));
+        EXPECT_EQUAL(AAA, checkSchema(s3, s4, s4h));
+        EXPECT_EQUAL(AAA, checkSchema(s3, Schema(), s4));
+    }
+}
+
+TEST("require that changed summary aspect is allowed")
+{
+    Schema s1 = created(INDEX, Schema::STRING);
+    s1.addSummaryField(SField("f1", Schema::STRING));
+    Schema s2 = created(INDEX, Schema::STRING);
+    Schema s2h = created(SUMMARY, Schema::STRING);
+
+    Schema s3 = created(ATTRIBUTE, Schema::STRING);
+    s3.addSummaryField(SField("f1", Schema::STRING));
+    Schema s4 = created(ATTRIBUTE, Schema::STRING);
+    Schema s4h = created(SUMMARY, Schema::STRING);
+    { // remove as summary field
+        EXPECT_EQUAL(OK, checkSchema(s2, s1, Schema()));
+        EXPECT_EQUAL(IAA, checkSchema(s2, Schema(), s1));
+        EXPECT_EQUAL(OK, checkSchema(s4, s3, Schema()));
+        EXPECT_EQUAL(AAA, checkSchema(s4, Schema(), s3));
+    }
+    { // add as summary field
+        EXPECT_EQUAL(OK, checkSchema(s1, s2, Schema()));
+        EXPECT_EQUAL(OK, checkSchema(s1, s2, s2h));
+        EXPECT_EQUAL(OK, checkSchema(s1, Schema(), s2));
+        EXPECT_EQUAL(OK, checkSchema(s3, s4, Schema()));
+        EXPECT_EQUAL(OK, checkSchema(s3, s4, s4h));
+        EXPECT_EQUAL(OK, checkSchema(s3, Schema(), s4));
+    }
+}
+
+TEST("require that fields can be added and removed")
+{
+    Schema e;
+    Schema s1 = created(INDEX, Schema::STRING);
+    Schema s2 = created(ATTRIBUTE, Schema::STRING);
+    Schema s3 = created(SUMMARY, Schema::STRING);
+    Schema s4 = created(SUMMARY, Schema::STRING);
+    s4.addIndexField(IField("f1", Schema::STRING));
+    Schema s5 = created(SUMMARY, Schema::STRING);
+    s5.addAttributeField(AField("f1", Schema::STRING));
+    Schema s6 = created(SUMMARY, Schema::STRING);
+    s6.addIndexField(IField("f1", Schema::STRING));
+    s6.addAttributeField(AField("f1", Schema::STRING));
+    { // addition of field
+        EXPECT_EQUAL(OK, checkSchema(s1, e, e));
+        EXPECT_EQUAL(OK, checkSchema(s2, e, e));
+        EXPECT_EQUAL(OK, checkSchema(s3, e, e));
+        EXPECT_EQUAL(OK, checkSchema(s4, e, e));
+        EXPECT_EQUAL(OK, checkSchema(s5, e, e));
+        EXPECT_EQUAL(OK, checkSchema(s6, e, e));
+    }
+    { // removal of field
+        EXPECT_EQUAL(OK, checkSchema(e, s1, e));
+        EXPECT_EQUAL(OK, checkSchema(e, e, s1));
+        EXPECT_EQUAL(OK, checkSchema(e, s2, e));
+        EXPECT_EQUAL(OK, checkSchema(e, e, s2));
+        EXPECT_EQUAL(OK, checkSchema(e, s3, e));
+        EXPECT_EQUAL(OK, checkSchema(e, e, s3));
+        EXPECT_EQUAL(OK, checkSchema(e, s4, e));
+        EXPECT_EQUAL(OK, checkSchema(e, e, s4));
+        EXPECT_EQUAL(OK, checkSchema(e, s5, e));
+        EXPECT_EQUAL(OK, checkSchema(e, e, s5));
+        EXPECT_EQUAL(OK, checkSchema(e, s6, e));
+        EXPECT_EQUAL(OK, checkSchema(e, e, s6));
+    }
+}
+
+TEST("require that data type changed precedes collection type changed")
+{
+    Schema olds = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::INDEX, Schema::STRING).schema();
+    Schema news = SchemaBuilder().add("f1", FType::SUMMARY, Schema::INT32).
+            add("f2", FType::INDEX, Schema::STRING, Schema::ARRAY).schema();
+    EXPECT_EQUAL(DTC, checkSchema(news, olds, Schema()));
+}
+
+TEST("require that collection type change precedes index aspect added")
+{
+    Schema olds = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::SUMMARY, Schema::STRING).schema();
+    Schema news = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING, Schema::ARRAY).
+            add("f2", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::INDEX, Schema::STRING).schema();
+    EXPECT_EQUAL(CTC, checkSchema(news, olds, Schema()));
+}
+
+TEST("require that index aspect added precedes index aspect removed")
+{
+    Schema olds = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::INDEX, Schema::STRING).schema();
+    Schema news = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING).
+            add("f1", FType::INDEX, Schema::STRING).
+            add("f2", FType::SUMMARY, Schema::STRING).schema();
+    EXPECT_EQUAL(IAA, checkSchema(news, olds, Schema()));
+}
+
+TEST("require that index aspect removed precedes attribute aspect removed")
+{
+    Schema olds = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING).
+            add("f1", FType::INDEX, Schema::STRING).
+            add("f2", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::ATTRIBUTE, Schema::STRING).schema();
+    Schema news = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::SUMMARY, Schema::STRING).schema();
+    EXPECT_EQUAL(IAR, checkSchema(news, olds, Schema()));
+}
+
+TEST("require that attribute aspect removed precedes attribute aspect added")
+{
+    Schema olds = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING).
+            add("f1", FType::ATTRIBUTE, Schema::STRING).
+            add("f2", FType::SUMMARY, Schema::STRING).schema();
+    Schema news = SchemaBuilder().add("f1", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::SUMMARY, Schema::STRING).
+            add("f2", FType::ATTRIBUTE, Schema::STRING).schema();
+    EXPECT_EQUAL(AAR, checkSchema(news, olds, Schema()));
+}
+
+AttributesConfigBuilder::Attribute
+createAttribute(const vespalib::string &name, bool fastAccess)
+{
+    AttributesConfigBuilder::Attribute attr;
+    attr.name = name;
+    attr.fastaccess = fastAccess;
+    return attr;
+}
+
+TEST("require that adding attribute fast-access is discovered")
+{
+    AttributesConfigBuilder oldCfg;
+    oldCfg.attribute.push_back(createAttribute("a1", false));
+    AttributesConfigBuilder newCfg;
+    newCfg.attribute.push_back(createAttribute("a1", true));
+
+    EXPECT_EQUAL(AFAA, checkAttribute(newCfg, oldCfg));
+}
+
+TEST("require that removing attribute fast-access is discovered")
+{
+    AttributesConfigBuilder oldCfg;
+    oldCfg.attribute.push_back(createAttribute("a1", true));
+    AttributesConfigBuilder newCfg;
+    newCfg.attribute.push_back(createAttribute("a1", false));
+
+    EXPECT_EQUAL(AFAR, checkAttribute(newCfg, oldCfg));
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/.gitignore b/searchcore/src/tests/proton/documentdb/document_scan_iterator/.gitignore
new file mode 100644
index 00000000000..6c961d2f232
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/.gitignore
@@ -0,0 +1 @@
+searchcore_document_scan_iterator_test_app
diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt
new file mode 100644
index 00000000000..1a342660f7c
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt
@@ -0,0 +1,12 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_document_scan_iterator_test_app
+    SOURCES
+    document_scan_iterator_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_feedoperation
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_document_scan_iterator_test_app COMMAND searchcore_document_scan_iterator_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/DESC b/searchcore/src/tests/proton/documentdb/document_scan_iterator/DESC
new file mode 100644
index 00000000000..b5965bc2f2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/DESC
@@ -0,0 +1,2 @@
+Test for document scan iterator. Take a look at document_scan_iterator_test.cpp for details.
+
diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/FILES b/searchcore/src/tests/proton/documentdb/document_scan_iterator/FILES
new file mode 100644
index 00000000000..f1b6d86a774
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/FILES
@@ -0,0 +1 @@
+document_scan_iterator_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp
new file mode 100644
index 00000000000..8a05d46d22f
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("document_scan_iterator_test");
+
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+#include <vespa/searchcore/proton/server/document_scan_iterator.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+using namespace proton;
+using namespace search;
+
+using vespalib::make_string;
+
+typedef DocumentMetaStore::Result DMSResult;
+typedef DocumentMetaStore::Timestamp Timestamp;
+typedef std::set<uint32_t> LidSet;
+typedef std::vector<uint32_t> LidVector;
+
+struct Fixture
+{
+    DocumentMetaStore _metaStore;
+    DocumentScanIterator _itr;
+    Fixture()
+        : _metaStore(std::make_shared<BucketDBOwner>()),
+          _itr(_metaStore)
+    {
+        _metaStore.constructFreeList();
+    }
+    Fixture &add(const LidVector &lids) {
+        for (auto lid : lids) {
+            add(lid);
+        }
+        return *this;
+    }
+    Fixture &add(uint32_t lid) {
+        DocumentId docId(make_string("userdoc:test:%u:%u", 1, lid));
+        const GlobalId &gid = docId.getGlobalId();
+        DMSResult res = _metaStore.inspect(gid);
+        ASSERT_EQUAL(lid, res._lid);
+        _metaStore.put(gid, gid.convertToBucketId(), Timestamp(lid), lid);
+        return *this;
+    }
+    LidSet scan(uint32_t count, uint32_t compactLidLimit, uint32_t maxDocsToScan = 10) {
+        LidSet retval;
+        for (uint32_t i = 0; i < count; ++i) {
+            retval.insert(next(compactLidLimit, maxDocsToScan, false));
+            EXPECT_TRUE(_itr.valid());
+        }
+        EXPECT_EQUAL(0u, next(compactLidLimit, maxDocsToScan, false));
+        EXPECT_FALSE(_itr.valid());
+        return retval;
+    }
+    uint32_t next(uint32_t compactLidLimit, uint32_t maxDocsToScan = 10, bool retry = false) {
+        return _itr.next(compactLidLimit, maxDocsToScan, retry).lid;
+    }
+};
+
+void
+assertLidSet(const LidSet &exp, const LidSet &act)
+{
+    EXPECT_EQUAL(exp, act);
+}
+
+TEST_F("require that an empty document meta store don't return any thing", Fixture)
+{
+    assertLidSet({}, f.scan(0, 4));
+}
+
+TEST_F("require that only lids > lid limit are returned", Fixture)
+{
+    f.add({1,2,3,4,5,6,7,8});
+    assertLidSet({5,6,7,8}, f.scan(4, 4));
+}
+
+TEST_F("require that max docs to scan (1) are taken into consideration", Fixture)
+{
+    f.add({1,2,3,4,5,6,7,8});
+    assertLidSet({0,5,6,7,8}, f.scan(8, 4, 1));
+}
+
+TEST_F("require that max docs to scan (2) are taken into consideration", Fixture)
+{
+    f.add({1,2,3,4,5,6,7,8});
+    // scan order is: 8, {2,4}, 7, {5,3}, {1,6} (5 scans total)
+    assertLidSet({0,7,8}, f.scan(5, 6, 2));
+}
+
+TEST_F("require that we start scan at previous doc if retry is set", Fixture)
+{
+    f.add({1,2,3,4,5,6,7,8});
+    uint32_t lid1 = f.next(4, 10, false);
+    uint32_t lid2 = f.next(4, 10, true);
+    EXPECT_EQUAL(lid1, lid2);
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/.gitignore b/searchcore/src/tests/proton/documentdb/document_subdbs/.gitignore
new file mode 100644
index 00000000000..e47d2bafa0e
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/.gitignore
@@ -0,0 +1 @@
+searchcore_document_subdbs_test_app
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt
new file mode 100644
index 00000000000..d79b9ad92ae
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt
@@ -0,0 +1,24 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_document_subdbs_test_app
+    SOURCES
+    document_subdbs_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_initializer
+    searchcore_reprocessing
+    searchcore_index
+    searchcore_docsummary
+    searchcore_persistenceengine
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_flushengine
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_document_subdbs_test_app COMMAND searchcore_document_subdbs_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/DESC b/searchcore/src/tests/proton/documentdb/document_subdbs/DESC
new file mode 100644
index 00000000000..22718579d4c
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/DESC
@@ -0,0 +1 @@
+Test for document sub db implementations. Take a look at document_subdbs_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/FILES b/searchcore/src/tests/proton/documentdb/document_subdbs/FILES
new file mode 100644
index 00000000000..3d5222f3212
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/FILES
@@ -0,0 +1 @@
+document_subdbs_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/attributes.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/attributes.cfg
new file mode 100644
index 00000000000..9d990996dd1
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/attributes.cfg
@@ -0,0 +1,3 @@
+attribute[1]
+attribute[0].name "attr1"
+attribute[0].datatype INT32
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/indexschema.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/indexschema.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/indexschema.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/juniperrc.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/juniperrc.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/juniperrc.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/rank-profiles.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/rank-profiles.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/rank-profiles.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/summary.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/summary.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/summary.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/summarymap.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/summarymap.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg1/summarymap.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/attributes.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/attributes.cfg
new file mode 100644
index 00000000000..3e488bbd7d9
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/attributes.cfg
@@ -0,0 +1,5 @@
+attribute[2]
+attribute[0].name "attr1"
+attribute[0].datatype INT32
+attribute[1].name "attr2"
+attribute[1].datatype INT32
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/indexschema.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/indexschema.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/indexschema.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/juniperrc.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/juniperrc.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/juniperrc.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/rank-profiles.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/rank-profiles.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/rank-profiles.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/summary.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/summary.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/summary.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/summarymap.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/summarymap.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg2/summarymap.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/attributes.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/attributes.cfg
new file mode 100644
index 00000000000..deb4ddcf63c
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/attributes.cfg
@@ -0,0 +1,6 @@
+attribute[2]
+attribute[0].name "attr1"
+attribute[0].datatype INT32
+attribute[0].fastaccess true
+attribute[1].name "attr2"
+attribute[1].datatype INT32
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/indexschema.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/indexschema.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/indexschema.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/juniperrc.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/juniperrc.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/juniperrc.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/rank-profiles.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/rank-profiles.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/rank-profiles.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/summary.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/summary.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/summary.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/summarymap.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/summarymap.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg3/summarymap.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/attributes.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/attributes.cfg
new file mode 100644
index 00000000000..d4fc1468739
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/attributes.cfg
@@ -0,0 +1,7 @@
+attribute[2]
+attribute[0].name "attr1"
+attribute[0].datatype INT32
+attribute[0].fastaccess true
+attribute[1].name "attr2"
+attribute[1].datatype INT32
+attribute[1].fastaccess true
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/indexschema.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/indexschema.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/indexschema.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/juniperrc.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/juniperrc.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/juniperrc.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/rank-profiles.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/rank-profiles.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/rank-profiles.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/summary.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/summary.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/summary.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/summarymap.cfg b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/summarymap.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/cfg4/summarymap.cfg
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
new file mode 100644
index 00000000000..afe1253ed93
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -0,0 +1,978 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("document_subdbs_test");
+
+#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
+#include <vespa/searchcore/proton/initializer/task_runner.h>
+#include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h>
+#include <vespa/searchcore/proton/metrics/metricswireservice.h>
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h>
+#include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h>
+#include <vespa/searchcore/proton/server/document_subdb_explorer.h>
+#include <vespa/searchcore/proton/server/emptysearchview.h>
+#include <vespa/searchcore/proton/server/fast_access_document_retriever.h>
+#include <vespa/searchcore/proton/server/idocumentsubdb.h>
+#include <vespa/searchcore/proton/server/minimal_document_retriever.h>
+#include <vespa/searchcore/proton/server/searchable_document_retriever.h>
+#include <vespa/searchcore/proton/server/searchabledocsubdb.h>
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/searchcore/proton/test/thread_utils.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <iostream>
+
+using namespace document;
+using namespace proton;
+using namespace proton::matching;
+using namespace search;
+using namespace search::common;
+using namespace search::index;
+using namespace search::transactionlog;
+using namespace searchcorespi;
+using namespace vespalib;
+using proton::bucketdb::BucketDBHandler;
+using proton::bucketdb::IBucketDBHandler;
+using proton::bucketdb::IBucketDBHandlerInitializer;
+
+using searchcorespi::IFlushTarget;
+using searchcorespi::index::IThreadingService;
+using storage::spi::Timestamp;
+using vespa::config::search::core::ProtonConfig;
+using vespalib::mkdir;
+
+typedef StoreOnlyDocSubDB::Config StoreOnlyConfig;
+typedef StoreOnlyDocSubDB::Context StoreOnlyContext;
+typedef FastAccessDocSubDB::Config FastAccessConfig;
+typedef FastAccessDocSubDB::Context FastAccessContext;
+typedef SearchableDocSubDB::Config SearchableConfig;
+typedef SearchableDocSubDB::Context SearchableContext;
+typedef std::vector<AttributeGuard> AttributeGuardList;
+
+const std::string DOCTYPE_NAME = "searchdocument";
+const std::string SUB_NAME = "subdb";
+const std::string BASE_DIR = "basedir";
+const SerialNum CFG_SERIAL = 5;
+
+struct ConfigDir1 { static vespalib::string dir() { return "cfg1"; } };
+struct ConfigDir2 { static vespalib::string dir() { return "cfg2"; } };
+struct ConfigDir3 { static vespalib::string dir() { return "cfg3"; } };
+struct ConfigDir4 { static vespalib::string dir() { return "cfg4"; } };
+
+struct MySubDBOwner : public IDocumentSubDB::IOwner
+{
+    uint32_t _syncCnt;
+    MySubDBOwner() : _syncCnt(0) {}
+    void syncFeedView() override { ++_syncCnt; }
+    IIndexManagerFactory::SP
+    getIndexManagerFactory(const vespalib::stringref &name) const override {
+        (void) name;
+        return IIndexManagerFactory::SP();
+    }
+    vespalib::string getName() const override { return "owner"; }
+    uint32_t getDistributionKey() const override { return -1; }
+};
+
+struct MySyncProxy : public SyncProxy
+{
+	virtual void sync(SerialNum) {}
+};
+
+
+struct MyGetSerialNum : public IGetSerialNum
+{
+    virtual SerialNum getSerialNum() const { return 0u; }
+};
+
+struct MyFileHeaderContext : public FileHeaderContext
+{
+	virtual void addTags(vespalib::GenericHeader &, const vespalib::string &) const {}
+};
+
+struct MyMetricsWireService : public MetricsWireService
+{
+	std::set<vespalib::string> _attributes;
+	MyMetricsWireService() : _attributes() {}
+	virtual void addAttribute(AttributeMetrics &, AttributeMetrics *, const std::string &name) {
+		_attributes.insert(name);
+	}
+	virtual void removeAttribute(AttributeMetrics &, AttributeMetrics *, const std::string &) {}
+	virtual void cleanAttributes(AttributeMetrics &, AttributeMetrics *) {}
+	virtual void addRankProfile(LegacyDocumentDBMetrics &, const std::string &, size_t) {}
+	virtual void cleanRankProfiles(LegacyDocumentDBMetrics &) {}
+};
+
+struct MyStoreOnlyConfig
+{
+	StoreOnlyConfig _cfg;
+	MyStoreOnlyConfig()
+	    : _cfg(DocTypeName(DOCTYPE_NAME),
+			  SUB_NAME,
+			  BASE_DIR,
+			  GrowStrategy(),
+                   0, 0, SubDbType::READY)
+	{
+	}
+};
+
+struct MyStoreOnlyContext
+{
+	MySubDBOwner _owner;
+	MySyncProxy _syncProxy;
+	MyGetSerialNum _getSerialNum;
+	MyFileHeaderContext _fileHeader;
+	LegacyDocumentDBMetrics _metrics;
+	vespalib::Lock _configLock;
+	StoreOnlyContext _ctx;
+	MyStoreOnlyContext(IThreadingService &writeService,
+	                   ThreadStackExecutorBase &summaryExecutor,
+                           std::shared_ptr<BucketDBOwner> bucketDB,
+                           IBucketDBHandlerInitializer &
+                           bucketDBHandlerInitializer)
+	    : _owner(),
+	      _syncProxy(),
+	      _getSerialNum(),
+	      _fileHeader(),
+	      _metrics(DOCTYPE_NAME, 1),
+	      _configLock(),
+	      _ctx(_owner,
+               _syncProxy,
+               _getSerialNum,
+               _fileHeader,
+               writeService,
+               summaryExecutor,
+                   bucketDB,
+                   bucketDBHandlerInitializer,
+               _metrics,
+               _configLock)
+	{
+	}
+	const MySubDBOwner &getOwner() const {
+	    return _owner;
+	}
+};
+
+template <bool FastAccessAttributesOnly>
+struct MyFastAccessConfig
+{
+	FastAccessConfig _cfg;
+	MyFastAccessConfig()
+	    : _cfg(MyStoreOnlyConfig()._cfg, true, true, FastAccessAttributesOnly)
+	{
+	}
+};
+
+struct MyFastAccessContext
+{
+	MyStoreOnlyContext _storeOnlyCtx;
+	AttributeMetrics _attributeMetrics;
+	MyMetricsWireService _wireService;
+	FastAccessContext _ctx;
+	MyFastAccessContext(IThreadingService &writeService,
+	                    ThreadStackExecutorBase &summaryExecutor,
+                            std::shared_ptr<BucketDBOwner> bucketDB,
+                            IBucketDBHandlerInitializer &
+                            bucketDBHandlerInitializer)
+	    : _storeOnlyCtx(writeService, summaryExecutor, bucketDB,
+                            bucketDBHandlerInitializer),
+	      _attributeMetrics(NULL),
+	      _wireService(),
+	      _ctx(_storeOnlyCtx._ctx, _attributeMetrics, NULL, _wireService)
+	{
+	}
+	const MyMetricsWireService &getWireService() const {
+	    return _wireService;
+	}
+	const MySubDBOwner &getOwner() const {
+	    return _storeOnlyCtx.getOwner();
+	}
+};
+
+struct MySearchableConfig
+{
+	SearchableConfig _cfg;
+	MySearchableConfig()
+	    : _cfg(MyFastAccessConfig<false>()._cfg, 1)
+	{
+	}
+};
+
+struct MySearchableContext
+{
+	MyFastAccessContext _fastUpdCtx;
+	QueryLimiter _queryLimiter;
+	vespalib::Clock _clock;
+	SearchableContext _ctx;
+	MySearchableContext(IThreadingService &writeService,
+	                    ThreadStackExecutorBase &executor,
+                            std::shared_ptr<BucketDBOwner> bucketDB,
+                            IBucketDBHandlerInitializer &
+                            bucketDBHandlerInitializer)
+	    : _fastUpdCtx(writeService, executor, bucketDB,
+                          bucketDBHandlerInitializer),
+	      _queryLimiter(),
+	      _clock(),
+	      _ctx(_fastUpdCtx._ctx,
+	    	   _queryLimiter,
+	    	   _clock,
+	    	   executor)
+	{
+	}
+	const MyMetricsWireService &getWireService() const {
+	    return _fastUpdCtx.getWireService();
+	}
+	const MySubDBOwner &getOwner() const {
+	    return _fastUpdCtx.getOwner();
+	}
+};
+
+struct OneAttrSchema : public Schema
+{
+	OneAttrSchema() {
+		addAttributeField(Schema::AttributeField("attr1", Schema::DataType::INT32));
+	}
+};
+
+struct TwoAttrSchema : public OneAttrSchema
+{
+	TwoAttrSchema() {
+		addAttributeField(Schema::AttributeField("attr2", Schema::DataType::INT32));
+	}
+};
+
+struct MyConfigSnapshot
+{
+    typedef std::unique_ptr<MyConfigSnapshot> UP;
+    Schema _schema;
+	DocBuilder _builder;
+	DocumentDBConfig::SP _cfg;
+	MyConfigSnapshot(const Schema &schema,
+			         const vespalib::string &cfgDir)
+	    : _schema(schema),
+	      _builder(_schema),
+	      _cfg()
+	{
+		DocumentDBConfig::DocumenttypesConfigSP documenttypesConfig
+		    (new DocumenttypesConfig(_builder.getDocumenttypesConfig()));
+		TuneFileDocumentDB::SP tuneFileDocumentDB(new TuneFileDocumentDB());
+		BootstrapConfig::SP bootstrap
+		    (new BootstrapConfig(1,
+                                 documenttypesConfig,
+                                 _builder.getDocumentTypeRepo(),
+                                 BootstrapConfig::ProtonConfigSP(new ProtonConfig()),
+                                 tuneFileDocumentDB));
+		config::DirSpec spec(cfgDir);
+		DocumentDBConfigHelper mgr(spec, "searchdocument");
+		mgr.forwardConfig(bootstrap);
+		mgr.nextGeneration(1);
+		_cfg = mgr.getConfig();
+	}
+};
+
+template <typename Traits>
+struct FixtureBase
+{
+	ExecutorThreadingService _writeService;
+	ThreadStackExecutor _summaryExecutor;
+	typename Traits::Config _cfg;
+        std::shared_ptr<BucketDBOwner> _bucketDB;
+        BucketDBHandler _bucketDBHandler;
+	typename Traits::Context _ctx;
+	typename Traits::Schema _baseSchema;
+	MyConfigSnapshot::UP _snapshot;
+	test::DirectoryHandler _baseDir;
+	typename Traits::SubDB _subDb;
+	IFeedView::SP _tmpFeedView;
+	FixtureBase()
+	    : _writeService(),
+	      _summaryExecutor(1, 64 * 1024),
+	      _cfg(),
+              _bucketDB(std::make_shared<BucketDBOwner>()),
+              _bucketDBHandler(*_bucketDB),
+	      _ctx(_writeService, _summaryExecutor, _bucketDB,
+                   _bucketDBHandler),
+	      _baseSchema(),
+	      _snapshot(new MyConfigSnapshot(_baseSchema, Traits::ConfigDir::dir())),
+	      _baseDir(BASE_DIR + "/" + SUB_NAME, BASE_DIR),
+	      _subDb(_cfg._cfg, _ctx._ctx),
+              _tmpFeedView()
+	{
+	    init();
+	}
+	~FixtureBase() {
+	    _writeService.sync();
+	}
+    template <typename FunctionType>
+    void runInMaster(FunctionType func) {
+        test::runInMaster(_writeService, func);
+    }
+	void init() {
+		Schema::SP unionSchema(new Schema());
+                DocumentSubDbInitializer::SP task =
+                    _subDb.createInitializer(*_snapshot->_cfg,
+                                             Traits::configSerial(),
+                                             unionSchema,
+                                             ProtonConfig::Summary(),
+                                             ProtonConfig::Index());
+                vespalib::ThreadStackExecutor executor(1, 1024 * 1024);
+                initializer::TaskRunner taskRunner(executor);
+                taskRunner.runTask(task);
+		SessionManager::SP sessionMgr(new SessionManager(1));
+		runInMaster([&] () { _subDb.initViews(*_snapshot->_cfg, sessionMgr); });
+	}
+	void basicReconfig(SerialNum serialNum) {
+	    runInMaster([&] () { performReconfig(serialNum, TwoAttrSchema(), ConfigDir2::dir()); });
+	}
+	void reconfig(SerialNum serialNum,
+	              const Schema &reconfigSchema,
+	              const vespalib::string &reconfigConfigDir) {
+	    runInMaster([&] () { performReconfig(serialNum, reconfigSchema, reconfigConfigDir); });
+        }
+	void performReconfig(SerialNum serialNum,
+	              const Schema &reconfigSchema,
+	              const vespalib::string &reconfigConfigDir) {
+	    MyConfigSnapshot::UP newCfg(new MyConfigSnapshot(reconfigSchema, reconfigConfigDir));
+	    DocumentDBConfig::ComparisonResult cmpResult;
+	    cmpResult.attributesChanged = true;
+	    cmpResult._documenttypesChanged = true;
+	    cmpResult._documentTypeRepoChanged = true;
+	    IReprocessingTask::List tasks =
+	            _subDb.applyConfig(*newCfg->_cfg,
+	                    *_snapshot->_cfg,
+	                    serialNum,
+	                    ReconfigParams(cmpResult));
+	    _snapshot = std::move(newCfg);
+	    if (!tasks.empty()) {
+	        ReprocessingRunner runner;
+	        runner.addTasks(tasks);
+	        runner.run();
+	    }
+	    _subDb.onReprocessDone(serialNum);
+	}
+	void sync() {
+	    _writeService.master().sync();
+	}
+	proton::IAttributeManager::SP getAttributeManager() {
+		return _subDb.getAttributeManager();
+	}
+	const typename Traits::FeedView *getFeedView() {
+		_tmpFeedView = _subDb.getFeedView();
+		const typename Traits::FeedView *retval =
+		        dynamic_cast<typename Traits::FeedView *>(_tmpFeedView.get());
+		ASSERT_TRUE(retval != NULL);
+		return retval;
+	}
+	const MyMetricsWireService &getWireService() const {
+	    return _ctx.getWireService();
+	}
+	const MySubDBOwner &getOwner() const {
+	    return _ctx.getOwner();
+	}
+};
+
+template <typename SchemaT, typename ConfigDirT, uint32_t ConfigSerial = CFG_SERIAL>
+struct BaseTraitsT
+{
+    typedef SchemaT Schema;
+    typedef ConfigDirT ConfigDir;
+    static uint32_t configSerial() { return ConfigSerial; }
+};
+
+typedef BaseTraitsT<OneAttrSchema, ConfigDir1> BaseTraits;
+
+struct StoreOnlyTraits : public BaseTraits
+{
+    typedef MyStoreOnlyConfig Config;
+    typedef MyStoreOnlyContext Context;
+    typedef StoreOnlyDocSubDB SubDB;
+    typedef StoreOnlyFeedView FeedView;
+};
+
+typedef FixtureBase<StoreOnlyTraits> StoreOnlyFixture;
+
+struct FastAccessTraits : public BaseTraits
+{
+    typedef MyFastAccessConfig<false> Config;
+    typedef MyFastAccessContext Context;
+    typedef FastAccessDocSubDB SubDB;
+    typedef FastAccessFeedView FeedView;
+};
+
+typedef FixtureBase<FastAccessTraits> FastAccessFixture;
+
+template <typename ConfigDirT>
+struct FastAccessOnlyTraitsBase : public BaseTraitsT<TwoAttrSchema, ConfigDirT>
+{
+    typedef MyFastAccessConfig<true> Config;
+    typedef MyFastAccessContext Context;
+    typedef FastAccessDocSubDB SubDB;
+    typedef FastAccessFeedView FeedView;
+};
+
+// Setup with 1 fast-access attribute
+typedef FastAccessOnlyTraitsBase<ConfigDir3> FastAccessOnlyTraits;
+typedef FixtureBase<FastAccessOnlyTraits> FastAccessOnlyFixture;
+
+template <typename SchemaT, typename ConfigDirT>
+struct SearchableTraitsBase : public BaseTraitsT<SchemaT, ConfigDirT>
+{
+    typedef MySearchableConfig Config;
+    typedef MySearchableContext Context;
+    typedef SearchableDocSubDB SubDB;
+    typedef proton::SearchableFeedView FeedView;
+};
+
+typedef SearchableTraitsBase<OneAttrSchema, ConfigDir1> SearchableTraits;
+typedef FixtureBase<SearchableTraits> SearchableFixture;
+
+void
+assertAttributes1(const AttributeGuardList &attributes)
+{
+	EXPECT_EQUAL(1u, attributes.size());
+	EXPECT_EQUAL("attr1", attributes[0].get().getName());
+}
+
+void
+assertAttributes1(const std::vector<search::AttributeVector *> &attributes)
+{
+	EXPECT_EQUAL(1u, attributes.size());
+	EXPECT_EQUAL("attr1", attributes[0]->getName());
+}
+
+void
+assertAttributes2(const AttributeGuardList &attributes)
+{
+	EXPECT_EQUAL(2u, attributes.size());
+	EXPECT_EQUAL("attr1", attributes[0].get().getName());
+	EXPECT_EQUAL("attr2", attributes[1].get().getName());
+}
+
+void
+assertAttributes2(const std::vector<search::AttributeVector *> &attributes)
+{
+	EXPECT_EQUAL(2u, attributes.size());
+	EXPECT_EQUAL("attr1", attributes[0]->getName());
+	EXPECT_EQUAL("attr2", attributes[1]->getName());
+}
+
+TEST_F("require that managers and components are instantiated", StoreOnlyFixture)
+{
+	EXPECT_TRUE(f._subDb.getSummaryManager().get() != NULL);
+	EXPECT_TRUE(f._subDb.getSummaryAdapter().get() != NULL);
+	EXPECT_TRUE(f._subDb.getAttributeManager().get() == NULL);
+	EXPECT_TRUE(f._subDb.getIndexManager().get() == NULL);
+	EXPECT_TRUE(f._subDb.getIndexWriter().get() == NULL);
+	EXPECT_TRUE(f._subDb.getFeedView().get() != NULL);
+	EXPECT_TRUE(f._subDb.getSearchView().get() != NULL);
+	EXPECT_TRUE(dynamic_cast<StoreOnlyFeedView *>(f._subDb.getFeedView().get()) != NULL);
+	EXPECT_TRUE(dynamic_cast<EmptySearchView *>(f._subDb.getSearchView().get()) != NULL);
+	EXPECT_TRUE(dynamic_cast<MinimalDocumentRetriever *>(f._subDb.getDocumentRetriever().get()) != NULL);
+}
+
+TEST_F("require that managers and components are instantiated", FastAccessFixture)
+{
+	EXPECT_TRUE(f._subDb.getSummaryManager().get() != NULL);
+	EXPECT_TRUE(f._subDb.getSummaryAdapter().get() != NULL);
+	EXPECT_TRUE(f._subDb.getAttributeManager().get() != NULL);
+	EXPECT_TRUE(f._subDb.getIndexManager().get() == NULL);
+	EXPECT_TRUE(f._subDb.getIndexWriter().get() == NULL);
+	EXPECT_TRUE(f._subDb.getFeedView().get() != NULL);
+	EXPECT_TRUE(f._subDb.getSearchView().get() != NULL);
+	EXPECT_TRUE(dynamic_cast<FastAccessFeedView *>(f._subDb.getFeedView().get()) != NULL);
+	EXPECT_TRUE(dynamic_cast<EmptySearchView *>(f._subDb.getSearchView().get()) != NULL);
+	EXPECT_TRUE(dynamic_cast<FastAccessDocumentRetriever *>(f._subDb.getDocumentRetriever().get()) != NULL);
+}
+
+TEST_F("require that managers and components are instantiated", SearchableFixture)
+{
+	EXPECT_TRUE(f._subDb.getSummaryManager().get() != NULL);
+	EXPECT_TRUE(f._subDb.getSummaryAdapter().get() != NULL);
+	EXPECT_TRUE(f._subDb.getAttributeManager().get() != NULL);
+	EXPECT_TRUE(f._subDb.getIndexManager().get() != NULL);
+	EXPECT_TRUE(f._subDb.getIndexWriter().get() != NULL);
+	EXPECT_TRUE(f._subDb.getFeedView().get() != NULL);
+	EXPECT_TRUE(f._subDb.getSearchView().get() != NULL);
+	EXPECT_TRUE(dynamic_cast<SearchableFeedView *>(f._subDb.getFeedView().get()) != NULL);
+	EXPECT_TRUE(dynamic_cast<SearchView *>(f._subDb.getSearchView().get()) != NULL);
+	EXPECT_TRUE(dynamic_cast<SearchableDocumentRetriever *>(f._subDb.getDocumentRetriever().get()) != NULL);
+}
+
+template<typename Fixture>
+void
+requireThatAttributeManagerIsInstantiated(Fixture &f)
+{
+	std::vector<AttributeGuard> attributes;
+	f.getAttributeManager()->getAttributeList(attributes);
+	assertAttributes1(attributes);
+}
+
+TEST_F("require that attribute manager is instantiated", FastAccessFixture)
+{
+    requireThatAttributeManagerIsInstantiated(f);
+}
+
+TEST_F("require that attribute manager is instantiated", SearchableFixture)
+{
+    requireThatAttributeManagerIsInstantiated(f);
+}
+
+template <typename Fixture>
+void
+requireThatAttributesAreAccessibleViaFeedView(Fixture &f)
+{
+	assertAttributes1(f.getFeedView()->getAttributeWriter()->getWritableAttributes());
+}
+
+TEST_F("require that attributes are accessible via feed view", FastAccessFixture)
+{
+    requireThatAttributesAreAccessibleViaFeedView(f);
+}
+
+TEST_F("require that attributes are accessible via feed view", SearchableFixture)
+{
+    requireThatAttributesAreAccessibleViaFeedView(f);
+}
+
+template <typename Fixture>
+void
+requireThatAttributeManagerCanBeReconfigured(Fixture &f)
+{
+	f.basicReconfig(10);
+	std::vector<AttributeGuard> attributes;
+	f.getAttributeManager()->getAttributeList(attributes);
+	assertAttributes2(attributes);
+}
+
+TEST_F("require that attribute manager can be reconfigured", FastAccessFixture)
+{
+    requireThatAttributeManagerCanBeReconfigured(f);
+}
+
+TEST_F("require that attribute manager can be reconfigured", SearchableFixture)
+{
+    requireThatAttributeManagerCanBeReconfigured(f);
+}
+
+template <typename Fixture>
+void
+requireThatReconfiguredAttributesAreAccessibleViaFeedView(Fixture &f)
+{
+	f.basicReconfig(10);
+	assertAttributes2(f.getFeedView()->getAttributeWriter()->getWritableAttributes());
+}
+
+TEST_F("require that reconfigured attributes are accessible via feed view", FastAccessFixture)
+{
+    requireThatReconfiguredAttributesAreAccessibleViaFeedView(f);
+}
+
+TEST_F("require that reconfigured attributes are accessible via feed view", SearchableFixture)
+{
+    requireThatReconfiguredAttributesAreAccessibleViaFeedView(f);
+}
+
+template <typename Fixture>
+void
+requireThatOwnerIsNotifiedWhenFeedViewChanges(Fixture &f)
+{
+	EXPECT_EQUAL(0u, f.getOwner()._syncCnt);
+	f.basicReconfig(10);
+	EXPECT_EQUAL(1u, f.getOwner()._syncCnt);
+}
+
+TEST_F("require that owner is noticed when feed view changes", StoreOnlyFixture)
+{
+    requireThatOwnerIsNotifiedWhenFeedViewChanges(f);
+}
+
+TEST_F("require that owner is noticed when feed view changes", FastAccessFixture)
+{
+    requireThatOwnerIsNotifiedWhenFeedViewChanges(f);
+}
+
+TEST_F("require that owner is noticed when feed view changes", SearchableFixture)
+{
+	EXPECT_EQUAL(1u, f.getOwner()._syncCnt); // NOTE: init also notifies owner
+	f.basicReconfig(10);
+	EXPECT_EQUAL(2u, f.getOwner()._syncCnt);
+}
+
+template <typename Fixture>
+void
+requireThatAttributeMetricsAreRegistered(Fixture &f)
+{
+	EXPECT_EQUAL(2u, f.getWireService()._attributes.size());
+	auto itr = f.getWireService()._attributes.begin();
+	EXPECT_EQUAL("[documentmetastore]", *itr++);
+	EXPECT_EQUAL("attr1", *itr);
+}
+
+TEST_F("require that attribute metrics are registered", FastAccessFixture)
+{
+    requireThatAttributeMetricsAreRegistered(f);
+}
+
+TEST_F("require that attribute metrics are registered", SearchableFixture)
+{
+    requireThatAttributeMetricsAreRegistered(f);
+}
+
+template <typename Fixture>
+void
+requireThatAttributeMetricsCanBeReconfigured(Fixture &f)
+{
+	f.basicReconfig(10);
+	EXPECT_EQUAL(3u, f.getWireService()._attributes.size());
+	auto itr = f.getWireService()._attributes.begin();
+	EXPECT_EQUAL("[documentmetastore]", *itr++);
+	EXPECT_EQUAL("attr1", *itr++);
+	EXPECT_EQUAL("attr2", *itr);
+}
+
+TEST_F("require that attribute metrics can be reconfigured", FastAccessFixture)
+{
+    requireThatAttributeMetricsCanBeReconfigured(f);
+}
+
+TEST_F("require that attribute metrics can be reconfigured", SearchableFixture)
+{
+    requireThatAttributeMetricsCanBeReconfigured(f);
+}
+
+template <typename Fixture>
+IFlushTarget::List
+getFlushTargets(Fixture &f)
+{
+	IFlushTarget::List targets = (static_cast<IDocumentSubDB &>(f._subDb)).getFlushTargets();
+	std::sort(targets.begin(), targets.end(),
+	        [](const IFlushTarget::SP &lhs, const IFlushTarget::SP &rhs) {
+	    return lhs->getName() < rhs->getName(); });
+	return targets;
+}
+
+typedef IFlushTarget::Type FType;
+typedef IFlushTarget::Component FComponent;
+
+bool
+assertTarget(const vespalib::string &name,
+             FType type,
+             FComponent component,
+             const IFlushTarget &target)
+{
+    if (!EXPECT_EQUAL(name, target.getName())) return false;
+    if (!EXPECT_TRUE(type == target.getType())) return false;
+    if (!EXPECT_TRUE(component == target.getComponent())) return false;
+    return true;
+}
+
+TEST_F("require that flush targets can be retrieved", FastAccessFixture)
+{
+	IFlushTarget::List targets = getFlushTargets(f);
+	EXPECT_EQUAL(4u, targets.size());
+	EXPECT_EQUAL("subdb.attribute.attr1", targets[0]->getName());
+	EXPECT_EQUAL("subdb.documentmetastore", targets[1]->getName());
+	EXPECT_EQUAL("subdb.summary.compact", targets[2]->getName());
+	EXPECT_EQUAL("subdb.summary.flush", targets[3]->getName());
+}
+
+TEST_F("require that flush targets can be retrieved", SearchableFixture)
+{
+	IFlushTarget::List targets = getFlushTargets(f);
+	EXPECT_EQUAL(6u, targets.size());
+	EXPECT_TRUE(assertTarget("subdb.attribute.attr1", FType::SYNC, FComponent::ATTRIBUTE, *targets[0]));
+	EXPECT_TRUE(assertTarget("subdb.documentmetastore", FType::SYNC, FComponent::ATTRIBUTE, *targets[1]));
+	EXPECT_TRUE(assertTarget("subdb.memoryindex.flush", FType::FLUSH, FComponent::INDEX, *targets[2]));
+	EXPECT_TRUE(assertTarget("subdb.memoryindex.fusion", FType::GC, FComponent::INDEX, *targets[3]));
+	EXPECT_TRUE(assertTarget("subdb.summary.compact", FType::GC, FComponent::DOCUMENT_STORE, *targets[4]));
+	EXPECT_TRUE(assertTarget("subdb.summary.flush", FType::SYNC, FComponent::DOCUMENT_STORE, *targets[5]));
+}
+
+TEST_F("require that only fast-access attributes are instantiated", FastAccessOnlyFixture)
+{
+	std::vector<AttributeGuard> attrs;
+	f.getAttributeManager()->getAttributeList(attrs);
+	EXPECT_EQUAL(1u, attrs.size());
+	EXPECT_EQUAL("attr1", attrs[0].get().getName());
+}
+
+template <typename FixtureType>
+struct DocumentHandler
+{
+    FixtureType &_f;
+    DocBuilder _builder;
+    DocumentHandler(FixtureType &f) : _f(f), _builder(f._baseSchema) {}
+    static constexpr uint32_t BUCKET_USED_BITS = 8;
+    static DocumentId createDocId(uint32_t docId)
+    {
+        return DocumentId(vespalib::make_string("id:searchdocument:"
+                                                "searchdocument::%u", docId));
+    }
+    Document::UP createEmptyDoc(uint32_t docId) {
+        return _builder.startDocument
+            (vespalib::make_string("id:searchdocument:searchdocument::%u",
+                                   docId)).
+            endDocument();
+    }
+    Document::UP createDoc(uint32_t docId, int64_t attr1Value, int64_t attr2Value) {
+        return _builder.startDocument
+                (vespalib::make_string("id:searchdocument:searchdocument::%u", docId)).
+                startAttributeField("attr1").addInt(attr1Value).endField().
+                startAttributeField("attr2").addInt(attr2Value).endField().endDocument();
+    }
+    PutOperation createPut(Document::UP doc, Timestamp timestamp, SerialNum serialNum) {
+        test::Document testDoc(Document::SP(doc.release()), 0, timestamp);
+        PutOperation op(testDoc.getBucket(), testDoc.getTimestamp(), testDoc.getDoc());
+        op.setSerialNum(serialNum);
+        return op;
+    }
+    MoveOperation createMove(Document::UP doc, Timestamp timestamp,
+                             DbDocumentId sourceDbdId,
+                             uint32_t targetSubDbId,
+                             SerialNum serialNum)
+    {
+        test::Document testDoc(Document::SP(doc.release()), 0, timestamp);
+        MoveOperation op(testDoc.getBucket(), testDoc.getTimestamp(), testDoc.getDoc(), sourceDbdId, targetSubDbId);
+        op.setSerialNum(serialNum);
+        return op;
+    }
+    RemoveOperation createRemove(const DocumentId &docId, Timestamp timestamp,
+                                 SerialNum serialNum)
+    {
+        const document::GlobalId &gid = docId.getGlobalId();
+        BucketId bucket = gid.convertToBucketId();
+        bucket.setUsedBits(BUCKET_USED_BITS);
+        bucket = bucket.stripUnused();
+        RemoveOperation op(bucket, timestamp, docId);
+        op.setSerialNum(serialNum);
+        return op;
+    }
+    void putDoc(PutOperation &op) {
+        IFeedView::SP feedView = _f._subDb.getFeedView();
+        _f.runInMaster([&]() {    feedView->preparePut(op);
+                                  feedView->handlePut(NULL, op); } );
+    }
+    void moveDoc(MoveOperation &op) {
+        IFeedView::SP feedView = _f._subDb.getFeedView();
+        _f.runInMaster([&]() {    feedView->handleMove(op); } );
+    }
+    void removeDoc(RemoveOperation &op)
+    {
+        IFeedView::SP feedView = _f._subDb.getFeedView();
+        _f.runInMaster([&]() {    feedView->prepareRemove(op);
+                                  feedView->handleRemove(NULL, op); } );
+    }
+    void putDocs() {
+        PutOperation putOp = createPut(std::move(createDoc(1, 22, 33)),
+                                       Timestamp(10), 10);
+        putDoc(putOp);
+        putOp = createPut(std::move(createDoc(2, 44, 55)), Timestamp(20), 20);
+        putDoc(putOp);
+    }
+};
+
+void
+assertAttribute(const AttributeGuard &attr,
+                const vespalib::string &name,
+                uint32_t numDocs,
+                int64_t doc1Value,
+                int64_t doc2Value,
+                SerialNum createSerialNum,
+                SerialNum lastSerialNum)
+{
+	EXPECT_EQUAL(name, attr.get().getName());
+	EXPECT_EQUAL(numDocs, attr.get().getNumDocs());
+	EXPECT_EQUAL(doc1Value, attr.get().getInt(1));
+	EXPECT_EQUAL(doc2Value, attr.get().getInt(2));
+	EXPECT_EQUAL(createSerialNum, attr.get().getCreateSerialNum());
+	EXPECT_EQUAL(lastSerialNum, attr.get().getStatus().getLastSyncToken());
+}
+
+void
+assertAttribute1(const AttributeGuard &attr,
+                           SerialNum createSerialNum,
+                           SerialNum lastSerialNum)
+{
+    assertAttribute(attr, "attr1", 3, 22, 44, createSerialNum, lastSerialNum);
+}
+
+void
+assertAttribute2(const AttributeGuard &attr,
+                           SerialNum createSerialNum,
+                           SerialNum lastSerialNum)
+{
+    assertAttribute(attr, "attr2", 3, 33, 55, createSerialNum, lastSerialNum);
+}
+
+TEST_F("require that fast-access attributes are populated during feed", FastAccessOnlyFixture)
+{
+    f._subDb.onReplayDone();
+    DocumentHandler<FastAccessOnlyFixture> handler(f);
+    handler.putDocs();
+
+	std::vector<AttributeGuard> attrs;
+	f.getAttributeManager()->getAttributeList(attrs);
+	EXPECT_EQUAL(1u, attrs.size());
+    assertAttribute1(attrs[0], CFG_SERIAL, 20);
+}
+
+template <typename FixtureType, typename ConfigDirT>
+void
+requireThatAttributesArePopulatedDuringReprocessing(FixtureType &f)
+{
+    f._subDb.onReplayDone();
+    DocumentHandler<FixtureType> handler(f);
+    handler.putDocs();
+
+    {
+        std::vector<AttributeGuard> attrs;
+        f.getAttributeManager()->getAttributeList(attrs);
+        EXPECT_EQUAL(1u, attrs.size());
+    }
+
+    // Reconfig to 2 attribute fields
+    f.reconfig(40u, TwoAttrSchema(), ConfigDirT::dir());
+
+    {
+        std::vector<AttributeGuard> attrs;
+        f.getAttributeManager()->getAttributeList(attrs);
+        EXPECT_EQUAL(2u, attrs.size());
+        assertAttribute1(attrs[0], CFG_SERIAL, 40);
+        assertAttribute2(attrs[1], 40, 40);
+    }
+}
+
+TEST_F("require that fast-access attributes are populated during reprocessing",
+        FastAccessOnlyFixture)
+{
+    requireThatAttributesArePopulatedDuringReprocessing<FastAccessOnlyFixture, ConfigDir4>(f);
+}
+
+// Setup with 2 fields (1 attribute according to config in dir)
+typedef SearchableTraitsBase<TwoAttrSchema, ConfigDir1> SearchableTraitsTwoField;
+typedef FixtureBase<SearchableTraitsTwoField> SearchableFixtureTwoField;
+
+TEST_F("require that regular attributes are populated during reprocessing",
+        SearchableFixtureTwoField)
+{
+    requireThatAttributesArePopulatedDuringReprocessing<SearchableFixtureTwoField, ConfigDir2>(f);
+}
+
+namespace
+{
+
+bool
+assertOperation(DocumentOperation &op,
+                uint32_t expPrevSubDbId, uint32_t expPrevLid,
+                uint32_t expSubDbId, uint32_t expLid)
+{
+    if (!EXPECT_EQUAL(expPrevSubDbId, op.getPrevSubDbId())) {
+        return false;
+    }
+    if (!EXPECT_EQUAL(expPrevLid, op.getPrevLid())) {
+        return false;
+    }
+    if (!EXPECT_EQUAL(expSubDbId, op.getSubDbId())) {
+        return false;
+    }
+    if (!EXPECT_EQUAL(expLid, op.getLid())) {
+        return false;
+    }
+    return true;
+}
+
+}
+
+TEST_F("require that lid allocation uses lowest free lid", StoreOnlyFixture)
+{
+    f._subDb.onReplayDone();
+    DocumentHandler<StoreOnlyFixture> handler(f);
+    Document::UP doc;
+    PutOperation putOp;
+    RemoveOperation rmOp;
+    MoveOperation moveOp;
+
+    doc = handler.createEmptyDoc(1);
+    putOp = handler.createPut(std::move(doc), Timestamp(10), 10);
+    handler.putDoc(putOp);
+    EXPECT_TRUE(assertOperation(putOp, 0, 0, 0, 1));
+    doc = handler.createEmptyDoc(2);
+    putOp = handler.createPut(std::move(doc), Timestamp(20), 20);
+    handler.putDoc(putOp);
+    EXPECT_TRUE(assertOperation(putOp, 0, 0, 0, 2));
+    rmOp = handler.createRemove(handler.createDocId(1), Timestamp(30), 30);
+    handler.removeDoc(rmOp);
+    EXPECT_TRUE(assertOperation(rmOp, 0, 1, 0, 0));
+    doc = handler.createEmptyDoc(3);
+    putOp = handler.createPut(std::move(doc), Timestamp(40), 40);
+    handler.putDoc(putOp);
+    EXPECT_TRUE(assertOperation(putOp, 0, 0, 0, 1));
+    rmOp = handler.createRemove(handler.createDocId(3), Timestamp(50), 50);
+    handler.removeDoc(rmOp);
+    EXPECT_TRUE(assertOperation(rmOp, 0, 1, 0, 0));
+    doc = handler.createEmptyDoc(2);
+    moveOp = handler.createMove(std::move(doc), Timestamp(20),
+                                DbDocumentId(0, 2), 0, 60);
+    moveOp.setTargetLid(1);
+    handler.moveDoc(moveOp);
+    EXPECT_TRUE(assertOperation(moveOp, 0, 2, 0, 1));
+    doc = handler.createEmptyDoc(3);
+    putOp = handler.createPut(std::move(doc), Timestamp(70), 70);
+    handler.putDoc(putOp);
+    EXPECT_TRUE(assertOperation(putOp, 0, 0, 0, 2));
+}
+
+template <typename FixtureType>
+struct ExplorerFixture : public FixtureType
+{
+    DocumentSubDBExplorer _explorer;
+    ExplorerFixture()
+        : FixtureType(),
+          _explorer(this->_subDb)
+    {
+    }
+};
+
+typedef ExplorerFixture<StoreOnlyFixture> StoreOnlyExplorerFixture;
+typedef ExplorerFixture<FastAccessFixture> FastAccessExplorerFixture;
+typedef ExplorerFixture<SearchableFixture> SearchableExplorerFixture;
+typedef std::vector<vespalib::string> StringVector;
+
+void
+assertExplorer(const StringVector &extraNames, const vespalib::StateExplorer &explorer)
+{
+    StringVector allNames = {"documentmetastore", "documentstore"};
+    allNames.insert(allNames.end(), extraNames.begin(), extraNames.end());
+    EXPECT_EQUAL(allNames, explorer.get_children_names());
+    EXPECT_TRUE(explorer.get_child("documentmetastore").get() != nullptr);
+    EXPECT_TRUE(explorer.get_child("documentstore").get() != nullptr);
+}
+
+TEST_F("require that underlying components are explorable", StoreOnlyExplorerFixture)
+{
+    assertExplorer({}, f._explorer);
+    EXPECT_TRUE(f._explorer.get_child("attribute").get() == nullptr);
+    EXPECT_TRUE(f._explorer.get_child("index").get() == nullptr);
+}
+
+TEST_F("require that underlying components are explorable", FastAccessExplorerFixture)
+{
+    assertExplorer({"attribute"}, f._explorer);
+    EXPECT_TRUE(f._explorer.get_child("attribute").get() != nullptr);
+    EXPECT_TRUE(f._explorer.get_child("index").get() == nullptr);
+}
+
+TEST_F("require that underlying components are explorable", SearchableExplorerFixture)
+{
+    assertExplorer({"attribute", "index"}, f._explorer);
+    EXPECT_TRUE(f._explorer.get_child("attribute").get() != nullptr);
+    EXPECT_TRUE(f._explorer.get_child("index").get() != nullptr);
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
+
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/.gitignore b/searchcore/src/tests/proton/documentdb/documentbucketmover/.gitignore
new file mode 100644
index 00000000000..4c7bc43b278
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/.gitignore
@@ -0,0 +1 @@
+searchcore_documentbucketmover_test_app
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt
new file mode 100644
index 00000000000..97a9ae1516b
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_documentbucketmover_test_app
+    SOURCES
+    documentbucketmover_test.cpp
+    DEPENDS
+    searchcore_test
+    searchcore_server
+    searchcore_persistenceengine
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_documentbucketmover_test_app COMMAND searchcore_documentbucketmover_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/DESC b/searchcore/src/tests/proton/documentdb/documentbucketmover/DESC
new file mode 100644
index 00000000000..7746f5e2222
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/DESC
@@ -0,0 +1 @@
+documentbucketmover test. Take a look at documentbucketmover_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/FILES b/searchcore/src/tests/proton/documentdb/documentbucketmover/FILES
new file mode 100644
index 00000000000..b035525acff
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/FILES
@@ -0,0 +1 @@
+documentbucketmover_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp
new file mode 100644
index 00000000000..9062067132c
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp
@@ -0,0 +1,1182 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("documentbucketmover_test");
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchcore/proton/server/bucketmovejob.h>
+#include <vespa/searchcore/proton/server/documentbucketmover.h>
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchcore/proton/server/idocumentmovehandler.h>
+#include <vespa/searchcore/proton/test/clusterstatehandler.h>
+#include <vespa/searchcore/proton/test/buckethandler.h>
+#include <vespa/searchcore/proton/server/maintenancedocumentsubdb.h>
+#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
+
+using namespace proton;
+using document::BucketId;
+using document::Document;
+using document::DocumentId;
+using document::DocumentTypeRepo;
+using document::GlobalId;
+using search::DocumentIdT;
+using search::DocumentMetaData;
+using search::index::DocBuilder;
+using search::index::Schema;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+using storage::spi::BucketInfo;
+
+typedef std::vector<MoveOperation> MoveOperationVector;
+typedef std::vector<Document::SP> DocumentVector;
+typedef BucketId::List BucketIdVector;
+typedef std::set<BucketId> BucketIdSet;
+typedef BucketMoveJob::ScanPosition ScanPos;
+typedef BucketMoveJob::ScanIterator ScanItr;
+
+namespace {
+
+const uint32_t FIRST_SCAN_PASS = 1;
+const uint32_t SECOND_SCAN_PASS = 2;
+
+}
+
+struct MyMoveHandler : public IDocumentMoveHandler
+{
+    BucketDBOwner &_bucketDb;
+    MoveOperationVector _moves;
+    size_t _numCachedBuckets;
+    MyMoveHandler(BucketDBOwner &bucketDb)
+        : _bucketDb(bucketDb),
+          _moves(),
+          _numCachedBuckets()
+    {}
+    virtual void handleMove(MoveOperation &op) {
+        _moves.push_back(op);
+        if (_bucketDb.takeGuard()->isCachedBucket(op.getBucketId())) {
+            ++_numCachedBuckets;
+        }
+    }
+    void reset() {
+        _moves.clear();
+        _numCachedBuckets = 0;
+    }
+};
+
+
+struct MyDocumentRetriever : public DocumentRetrieverBaseForTest
+{
+    DocumentTypeRepo::SP _repo;
+    DocumentVector       _docs;
+    MyDocumentRetriever(DocumentTypeRepo::SP repo) : _repo(repo), _docs() {
+        _docs.push_back(Document::SP()); // lid 0 invalid
+    }
+    virtual const document::DocumentTypeRepo &getDocumentTypeRepo() const { return *_repo; }
+    virtual void getBucketMetaData(const storage::spi::Bucket &,
+                                   DocumentMetaData::Vector &) const {}
+    virtual DocumentMetaData getDocumentMetaData(const DocumentId &) const { return DocumentMetaData(); }
+    virtual Document::UP getDocument(DocumentIdT lid) const {
+        return Document::UP(_docs[lid]->clone());
+    }
+
+    virtual CachedSelect::SP
+    parseSelect(const vespalib::string &) const
+    {
+        return CachedSelect::SP();
+    }
+};
+
+
+struct MyBucketModifiedHandler : public IBucketModifiedHandler
+{
+    BucketIdVector _modified;
+    virtual void notifyBucketModified(const BucketId &bucket) {
+        BucketIdVector::const_iterator itr = std::find(_modified.begin(), _modified.end(), bucket);
+        ASSERT_TRUE(itr == _modified.end());
+        _modified.push_back(bucket);
+    }
+    void reset() { _modified.clear(); }
+};
+
+
+struct MySubDb
+{
+    DocumentMetaStore::SP _metaStoreSP;
+    DocumentMetaStore &   _metaStore;
+    std::shared_ptr<MyDocumentRetriever> _realRetriever;
+    std::shared_ptr<IDocumentRetriever> _retriever;
+    MaintenanceDocumentSubDB _subDb;
+    test::UserDocuments        _docs;
+    bucketdb::BucketDBHandler _bucketDBHandler;
+    MySubDb(const DocumentTypeRepo::SP &repo,
+            std::shared_ptr<BucketDBOwner> bucketDB,
+            uint32_t subDbId,
+            SubDbType subDbType)
+        : _metaStoreSP(std::make_shared<DocumentMetaStore>(bucketDB,
+                       DocumentMetaStore::getFixedName(),
+                       search::GrowStrategy(),
+                       documentmetastore::IGidCompare::SP(new documentmetastore::DefaultGidCompare),
+                       subDbType)),
+          _metaStore(*_metaStoreSP),
+          _realRetriever(std::make_shared<MyDocumentRetriever>(repo)),
+          _retriever(_realRetriever),
+          _subDb(_metaStoreSP, _retriever, subDbId), _docs(),
+          _bucketDBHandler(*bucketDB)
+    {
+        _bucketDBHandler.addDocumentMetaStore(_metaStoreSP.get(), 0);
+    }
+    void insertDocs(const test::UserDocuments &docs_) {
+        for (test::UserDocuments::Iterator itr = docs_.begin(); itr != docs_.end(); ++itr) {
+            const test::BucketDocuments &bucketDocs = itr->second;
+            for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) {
+                const test::Document &testDoc = bucketDocs.getDocs()[i];
+                _metaStore.put(testDoc.getGid(), testDoc.getBucket(),
+                               testDoc.getTimestamp(), testDoc.getLid());
+                _realRetriever->_docs.push_back(testDoc.getDoc());
+                ASSERT_EQUAL(testDoc.getLid() + 1,
+                             _realRetriever->_docs.size());
+            }
+        }
+        _docs.merge(docs_);
+    }
+
+    BucketId bucket(uint32_t userId) const {
+        return _docs.getBucket(userId);
+    }
+
+    test::DocumentVector docs(uint32_t userId) {
+        return _docs.getGidOrderDocs(userId);
+    }
+
+    void setBucketState(const BucketId &bucketId, bool active) {
+        _metaStore.setBucketState(bucketId, active);
+    }
+
+    void removeBucket(uint32_t userId) {
+        const test::DocumentVector &userDocs = _docs.getDocs(userId);
+        for (size_t i = 0; i < userDocs.size(); ++i) {
+            _metaStore.remove(userDocs[i].getLid());
+            if (_metaStore.getFreeListActive()) {
+                _metaStore.removeComplete(userDocs[i].getLid());
+            }
+        }
+        BucketId b(bucket(userId));
+        EXPECT_EQUAL(0u, _metaStore.getBucketDB().takeGuard()->get(b).getEntryCount());
+        _bucketDBHandler.handleDeleteBucket(b);
+    }
+
+};
+
+
+struct MySubDbTwoBuckets : public MySubDb
+{
+    MySubDbTwoBuckets(test::UserDocumentsBuilder &builder,
+                      std::shared_ptr<BucketDBOwner> bucketDB,
+                      uint32_t subDbId,
+                      SubDbType subDbType)
+        : MySubDb(builder.getRepo(), bucketDB, subDbId, subDbType)
+    {
+        builder.createDocs(1, 1, 6);
+        builder.createDocs(2, 6, 9);
+        insertDocs(builder.getDocs());
+        ASSERT_NOT_EQUAL(bucket(1), bucket(2));
+        ASSERT_EQUAL(5u, docs(1).size());
+        ASSERT_EQUAL(3u, docs(2).size());
+        ASSERT_EQUAL(9u, _realRetriever->_docs.size());
+    }
+};
+
+
+struct MoveFixture
+{
+    test::UserDocumentsBuilder _builder;
+    std::shared_ptr<BucketDBOwner> _bucketDB;
+    DocumentBucketMover        _mover;
+    MySubDbTwoBuckets          _source;
+    BucketDBOwner              _bucketDb;
+    MyMoveHandler              _handler;
+    MoveFixture()
+        : _builder(),
+          _bucketDB(std::make_shared<BucketDBOwner>()),
+          _mover(),
+          _source(_builder, _bucketDB, 0u, SubDbType::READY),
+          _bucketDb(),
+          _handler(_bucketDb)
+    {
+    }
+    void setupForBucket(const BucketId &bucket,
+                        uint32_t sourceSubDbId,
+                        uint32_t targetSubDbId) {
+        _source._subDb._subDbId = sourceSubDbId;
+        _mover.setupForBucket(bucket, &_source._subDb, targetSubDbId, _handler, _bucketDb);
+    }
+    void moveDocuments(size_t maxDocsToMove) {
+        _mover.moveDocuments(maxDocsToMove);
+    }
+};
+
+
+TEST("require that initial bucket mover is done")
+{
+    DocumentBucketMover dbm;
+    EXPECT_TRUE(dbm.bucketDone());
+    dbm.moveDocuments(2);
+    EXPECT_TRUE(dbm.bucketDone());
+}
+
+
+bool
+assertEqual(const BucketId &bucket, const test::Document &doc,
+            uint32_t sourceSubDbId, uint32_t targetSubDbId, const MoveOperation &op)
+{
+    if (!EXPECT_EQUAL(bucket, op.getBucketId())) return false;
+    if (!EXPECT_EQUAL(doc.getTimestamp(), op.getTimestamp())) return false;
+    if (!EXPECT_EQUAL(doc.getDocId(), op.getDocument()->getId())) return false;
+    if (!EXPECT_EQUAL(doc.getLid(), op.getSourceDbdId().getLid())) return false;
+    if (!EXPECT_EQUAL(sourceSubDbId, op.getSourceDbdId().getSubDbId())) return false;
+    if (!EXPECT_EQUAL(0u, op.getTargetDbdId().getLid())) return false;
+    if (!EXPECT_EQUAL(targetSubDbId, op.getTargetDbdId().getSubDbId())) return false;
+    return true;
+}
+
+
+TEST_F("require that we can move all documents", MoveFixture)
+{
+    f.setupForBucket(f._source.bucket(1), 6, 9);
+    f.moveDocuments(5);
+    EXPECT_TRUE(f._mover.bucketDone());
+    EXPECT_EQUAL(5u, f._handler._moves.size());
+    for (size_t i = 0; i < 5u; ++i) {
+        assertEqual(f._source.bucket(1), f._source.docs(1)[0], 6, 9, f._handler._moves[0]);
+    }
+}
+
+TEST_F("require that bucket is cached when IDocumentMoveHandler handles move operation",
+        MoveFixture)
+{
+    f.setupForBucket(f._source.bucket(1), 6, 9);
+    f.moveDocuments(5);
+    EXPECT_TRUE(f._mover.bucketDone());
+    EXPECT_EQUAL(5u, f._handler._moves.size());
+    EXPECT_EQUAL(5u, f._handler._numCachedBuckets);
+    EXPECT_FALSE(f._bucketDb.takeGuard()->isCachedBucket(f._source.bucket(1)));
+}
+
+TEST_F("require that we can move documents in several steps", MoveFixture)
+{
+    f.setupForBucket(f._source.bucket(1), 6, 9);
+    f.moveDocuments(2);
+    EXPECT_FALSE(f._mover.bucketDone());
+    EXPECT_EQUAL(2u, f._handler._moves.size());
+    assertEqual(f._source.bucket(1), f._source.docs(1)[0], 6, 9, f._handler._moves[0]);
+    assertEqual(f._source.bucket(1), f._source.docs(1)[1], 6, 9, f._handler._moves[1]);
+    f.moveDocuments(2);
+    EXPECT_FALSE(f._mover.bucketDone());
+    EXPECT_EQUAL(4u, f._handler._moves.size());
+    assertEqual(f._source.bucket(1), f._source.docs(1)[2], 6, 9, f._handler._moves[2]);
+    assertEqual(f._source.bucket(1), f._source.docs(1)[3], 6, 9, f._handler._moves[3]);
+    f.moveDocuments(2);
+    EXPECT_TRUE(f._mover.bucketDone());
+    EXPECT_EQUAL(5u, f._handler._moves.size());
+    assertEqual(f._source.bucket(1), f._source.docs(1)[4], 6, 9, f._handler._moves[4]);
+    f.moveDocuments(2);
+    EXPECT_TRUE(f._mover.bucketDone());
+    EXPECT_EQUAL(5u, f._handler._moves.size());
+}
+
+
+struct ScanFixtureBase
+{
+    test::UserDocumentsBuilder _builder;
+    std::shared_ptr<BucketDBOwner> _bucketDB;
+    MySubDb                    _ready;
+    MySubDb                    _notReady;
+    ScanFixtureBase()
+        : _builder(),
+          _bucketDB(std::make_shared<BucketDBOwner>()),
+          _ready(_builder.getRepo(), _bucketDB, 1, SubDbType::READY),
+          _notReady(_builder.getRepo(), _bucketDB, 2, SubDbType::NOTREADY)
+    {
+    }
+
+    ScanItr
+    getItr(void)
+    {
+        return ScanItr(_bucketDB->takeGuard(), BucketId());
+    }
+
+    ScanItr
+    getItr(BucketId bucket,
+             BucketId endBucket = BucketId(),
+             uint32_t pass = FIRST_SCAN_PASS)
+    {
+        return ScanItr(_bucketDB->takeGuard(), pass,
+                       bucket, endBucket);
+    }
+};
+
+
+struct ScanFixture : public ScanFixtureBase
+{
+    ScanFixture() : ScanFixtureBase()
+    {
+        _builder.createDocs(6, 1, 2);
+        _builder.createDocs(8, 2, 3);
+        _ready.insertDocs(_builder.getDocs());
+        _builder.clearDocs();
+        _builder.createDocs(2, 1, 2);
+        _builder.createDocs(4, 2, 3);
+        _notReady.insertDocs(_builder.getDocs());
+        _builder.clearDocs();
+    }
+};
+
+
+struct OnlyNotReadyScanFixture : public ScanFixtureBase
+{
+    OnlyNotReadyScanFixture() : ScanFixtureBase()
+    {
+        _builder.createDocs(2, 1, 2);
+        _builder.createDocs(4, 2, 3);
+        _notReady.insertDocs(_builder.getDocs());
+    }
+};
+
+
+struct OnlyReadyScanFixture : public ScanFixtureBase
+{
+    OnlyReadyScanFixture() : ScanFixtureBase()
+    {
+        _builder.createDocs(6, 1, 2);
+        _builder.createDocs(8, 2, 3);
+        _ready.insertDocs(_builder.getDocs());
+    }
+};
+
+
+struct BucketVector : public BucketId::List
+{
+    BucketVector() : BucketId::List() {}
+    BucketVector &add(const BucketId &bucket) {
+        push_back(bucket);
+        return *this;
+    }
+};
+
+
+void
+advanceToFirstBucketWithDocs(ScanItr &itr, SubDbType subDbType)
+{
+    while (itr.valid()) {
+        if (subDbType == SubDbType::READY) {
+            if (itr.hasReadyBucketDocs())
+                return;
+        } else {
+            if (itr.hasNotReadyBucketDocs())
+                return;
+        }
+        ++itr;
+    }
+}
+
+
+void assertEquals(const BucketVector &exp, ScanItr &itr, SubDbType subDbType)
+{
+    for (size_t i = 0; i < exp.size(); ++i) {
+        advanceToFirstBucketWithDocs(itr, subDbType);
+        EXPECT_TRUE(itr.valid());
+        EXPECT_EQUAL(exp[i], itr.getBucket());
+        ++itr;
+    }
+    advanceToFirstBucketWithDocs(itr, subDbType);
+    EXPECT_FALSE(itr.valid());
+}
+
+
+TEST_F("require that we can iterate all buckets from start to end", ScanFixture)
+{
+    {
+        ScanItr itr = f.getItr();
+        assertEquals(BucketVector().
+                     add(f._notReady.bucket(2)).
+                     add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY);
+    }
+    {
+        ScanItr itr = f.getItr();
+        assertEquals(BucketVector().
+                     add(f._ready.bucket(6)).
+                     add(f._ready.bucket(8)), itr, SubDbType::READY);
+    }
+}
+
+
+TEST_F("require that we can iterate from the middle of not ready buckets", ScanFixture)
+{
+    BucketId bucket = f._notReady.bucket(2);
+    {
+        ScanItr itr = f.getItr(bucket, bucket, FIRST_SCAN_PASS);
+        assertEquals(BucketVector().
+                     add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY);
+    }
+    {
+        ScanItr itr = f.getItr(BucketId(), bucket, SECOND_SCAN_PASS);
+        assertEquals(BucketVector().
+                     add(f._notReady.bucket(2)), itr, SubDbType::NOTREADY);
+    }
+    {
+        ScanItr itr = f.getItr();
+        assertEquals(BucketVector().
+                     add(f._ready.bucket(6)).
+                     add(f._ready.bucket(8)), itr, SubDbType::READY);
+    }
+}
+
+
+TEST_F("require that we can iterate from the middle of ready buckets", ScanFixture)
+{
+    BucketId bucket = f._ready.bucket(6);
+    {
+        ScanItr itr = f.getItr();
+        assertEquals(BucketVector().
+                     add(f._notReady.bucket(2)).
+                     add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY);
+    }
+    {
+        ScanItr itr = f.getItr(bucket, bucket, FIRST_SCAN_PASS);
+        assertEquals(BucketVector().
+                     add(f._ready.bucket(8)), itr, SubDbType::READY);
+    }
+    {
+        ScanItr itr = f.getItr(BucketId(), bucket, SECOND_SCAN_PASS);
+        assertEquals(BucketVector().
+                     add(f._ready.bucket(6)), itr, SubDbType::READY);
+    }
+}
+
+
+TEST_F("require that we can iterate only not ready buckets", OnlyNotReadyScanFixture)
+{
+    ScanItr itr = f.getItr();
+    assertEquals(BucketVector().
+                 add(f._notReady.bucket(2)).
+                 add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY);
+}
+
+
+TEST_F("require that we can iterate only ready buckets", OnlyReadyScanFixture)
+{
+    ScanItr itr = f.getItr();
+    assertEquals(BucketVector().
+                 add(f._ready.bucket(6)).
+                 add(f._ready.bucket(8)), itr, SubDbType::READY);
+}
+
+
+TEST_F("require that we can iterate zero buckets", ScanFixtureBase)
+{
+    ScanItr itr = f.getItr();
+    EXPECT_FALSE(itr.valid());
+}
+
+
+struct MyFrozenBucketHandler : public IFrozenBucketHandler
+{
+    BucketIdSet _frozen;
+    std::set<IBucketFreezeListener *> _listeners;
+
+    MyFrozenBucketHandler()
+        : IFrozenBucketHandler(),
+          _frozen(),
+          _listeners()
+    {
+    }
+
+    virtual ~MyFrozenBucketHandler()
+    {
+        assert(_listeners.empty());
+    }
+
+    MyFrozenBucketHandler &addFrozen(const BucketId &bucket) {
+        _frozen.insert(bucket);
+        return *this;
+    }
+    MyFrozenBucketHandler &remFrozen(const BucketId &bucket) {
+        _frozen.erase(bucket);
+        for (auto &listener : _listeners) {
+            listener->notifyThawedBucket(bucket);
+        }
+        return *this;
+    }
+    virtual void addListener(IBucketFreezeListener *listener) override {
+        _listeners.insert(listener);
+    }
+    virtual void removeListener(IBucketFreezeListener *listener) override {
+        _listeners.erase(listener);
+    }
+
+    virtual ExclusiveBucketGuard::UP acquireExclusiveBucket(BucketId bucket) override {
+        return (_frozen.count(bucket) != 0)
+            ? ExclusiveBucketGuard::UP()
+            : std::make_unique<ExclusiveBucketGuard>(bucket);
+    }
+};
+
+struct ControllerFixtureBase
+{
+    test::UserDocumentsBuilder  _builder;
+    test::BucketStateCalculator::SP _calc;
+    test::ClusterStateHandler   _clusterStateHandler;
+    test::BucketHandler         _bucketHandler;
+    MyBucketModifiedHandler     _modifiedHandler;
+    std::shared_ptr<BucketDBOwner> _bucketDB;
+    MyMoveHandler               _moveHandler;
+    MySubDb                     _ready;
+    MySubDb                     _notReady;
+    MyFrozenBucketHandler       _fbh;
+    BucketMoveJob               _bmj;
+    ControllerFixtureBase()
+        : _builder(),
+          _calc(new test::BucketStateCalculator),
+          _bucketHandler(),
+          _modifiedHandler(),
+          _bucketDB(std::make_shared<BucketDBOwner>()),
+          _moveHandler(*_bucketDB),
+          _ready(_builder.getRepo(), _bucketDB, 1, SubDbType::READY),
+          _notReady(_builder.getRepo(), _bucketDB, 2, SubDbType::NOTREADY),
+          _fbh(),
+          _bmj(_calc, _moveHandler, _modifiedHandler, _ready._subDb,
+               _notReady._subDb, _fbh, _clusterStateHandler, _bucketHandler,
+               "test")
+    {
+    }
+    ControllerFixtureBase &addReady(const BucketId &bucket) {
+        _calc->addReady(bucket);
+        return *this;
+    }
+    ControllerFixtureBase &remReady(const BucketId &bucket) {
+        _calc->remReady(bucket);
+        return *this;
+    }
+    ControllerFixtureBase &changeCalc() {
+        _calc->resetAsked();
+        _moveHandler.reset();
+        _modifiedHandler.reset();
+        _clusterStateHandler.notifyClusterStateChanged(_calc);
+        return *this;
+    }
+    ControllerFixtureBase &addFrozen(const BucketId &bucket) {
+        _fbh.addFrozen(bucket);
+        return *this;
+    }
+    ControllerFixtureBase &remFrozen(const BucketId &bucket) {
+        _fbh.remFrozen(bucket);
+        _bmj.notifyThawedBucket(bucket);
+        return *this;
+    }
+    ControllerFixtureBase &activateBucket(const BucketId &bucket) {
+        _ready.setBucketState(bucket, true);
+        _bucketHandler.notifyBucketStateChanged(bucket,
+                                                BucketInfo::ActiveState::
+                                                ACTIVE);
+        return *this;
+    }
+    ControllerFixtureBase &deactivateBucket(const BucketId &bucket) {
+        _ready.setBucketState(bucket, false);
+        _bucketHandler.notifyBucketStateChanged(bucket,
+                                                BucketInfo::ActiveState::
+                                                NOT_ACTIVE);
+        return *this;
+    }
+    const MoveOperationVector &docsMoved() const {
+        return _moveHandler._moves;
+    }
+    const BucketIdVector &bucketsModified() const {
+        return _modifiedHandler._modified;
+    }
+    const BucketIdVector &calcAsked() const {
+        return _calc->asked();
+    }
+};
+
+
+struct ControllerFixture : public ControllerFixtureBase
+{
+    ControllerFixture() : ControllerFixtureBase()
+    {
+        _builder.createDocs(1, 1, 4); // 3 docs
+        _builder.createDocs(2, 4, 6); // 2 docs
+        _ready.insertDocs(_builder.getDocs());
+        _builder.clearDocs();
+        _builder.createDocs(3, 1, 3); // 2 docs
+        _builder.createDocs(4, 3, 6); // 3 docs
+        _notReady.insertDocs(_builder.getDocs());
+    }
+};
+
+
+struct OnlyReadyControllerFixture : public ControllerFixtureBase
+{
+    OnlyReadyControllerFixture() : ControllerFixtureBase()
+    {
+        _builder.createDocs(1, 1, 2); // 1 docs
+        _builder.createDocs(2, 2, 4); // 2 docs
+        _builder.createDocs(3, 4, 7); // 3 docs
+        _builder.createDocs(4, 7, 11); // 4 docs
+        _ready.insertDocs(_builder.getDocs());
+    }
+};
+
+
+TEST_F("require that nothing is moved if bucket state says so", ControllerFixture)
+{
+    EXPECT_FALSE(f._bmj.done());
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(2));
+    f._bmj.scanAndMove(4, 3);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_TRUE(f.docsMoved().empty());
+    EXPECT_TRUE(f.bucketsModified().empty());
+}
+
+
+TEST_F("require that not ready bucket is moved to ready if bucket state says so", ControllerFixture)
+{
+    // bucket 4 should be moved
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(2));
+    f.addReady(f._notReady.bucket(4));
+    f._bmj.scanAndMove(4, 3);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[0], 2, 1, f.docsMoved()[0]);
+    assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[1], 2, 1, f.docsMoved()[1]);
+    assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[2], 2, 1, f.docsMoved()[2]);
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._notReady.bucket(4), f.bucketsModified()[0]);
+}
+
+
+TEST_F("require that ready bucket is moved to not ready if bucket state says so", ControllerFixture)
+{
+    // bucket 2 should be moved
+    f.addReady(f._ready.bucket(1));
+    f._bmj.scanAndMove(4, 3);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(2u, f.docsMoved().size());
+    assertEqual(f._ready.bucket(2), f._ready.docs(2)[0], 1, 2, f.docsMoved()[0]);
+    assertEqual(f._ready.bucket(2), f._ready.docs(2)[1], 1, 2, f.docsMoved()[1]);
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(2), f.bucketsModified()[0]);
+}
+
+
+TEST_F("require that maxBucketsToScan is taken into consideration between not ready and ready scanning",
+       ControllerFixture)
+{
+    // bucket 4 should moved (last bucket)
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(2));
+    f.addReady(f._notReady.bucket(4));
+
+    // buckets 1, 2, and 3 considered
+    f._bmj.scanAndMove(3, 3);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+
+    // move bucket 4
+    f._bmj.scanAndMove(1, 4);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[0], 2, 1, f.docsMoved()[0]);
+    assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[1], 2, 1, f.docsMoved()[1]);
+    assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[2], 2, 1, f.docsMoved()[2]);
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._notReady.bucket(4), f.bucketsModified()[0]);
+}
+
+
+TEST_F("require that we move buckets in several steps", ControllerFixture)
+{
+    // bucket 2, 3, and 4 should be moved
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._notReady.bucket(3));
+    f.addReady(f._notReady.bucket(4));
+
+    // consider move bucket 1
+    f._bmj.scanAndMove(1, 2);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+
+    // move bucket 2, docs 1,2
+    f._bmj.scanAndMove(1, 2);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(2u, f.docsMoved().size());
+    EXPECT_TRUE(assertEqual(f._ready.bucket(2), f._ready.docs(2)[0], 1, 2, f.docsMoved()[0]));
+    EXPECT_TRUE(assertEqual(f._ready.bucket(2), f._ready.docs(2)[1], 1, 2, f.docsMoved()[1]));
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(2), f.bucketsModified()[0]);
+
+    // move bucket 3, docs 1,2
+    f._bmj.scanAndMove(1, 2);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(4u, f.docsMoved().size());
+    EXPECT_TRUE(assertEqual(f._notReady.bucket(3), f._notReady.docs(3)[0], 2, 1, f.docsMoved()[2]));
+    EXPECT_TRUE(assertEqual(f._notReady.bucket(3), f._notReady.docs(3)[1], 2, 1, f.docsMoved()[3]));
+    EXPECT_EQUAL(2u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._notReady.bucket(3), f.bucketsModified()[1]);
+
+    // move bucket 4, docs 1,2
+    f._bmj.scanAndMove(1, 2);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(6u, f.docsMoved().size());
+    EXPECT_TRUE(assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[0], 2, 1, f.docsMoved()[4]));
+    EXPECT_TRUE(assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[1], 2, 1, f.docsMoved()[5]));
+    EXPECT_EQUAL(2u, f.bucketsModified().size());
+
+    // move bucket 4, docs 3
+    f._bmj.scanAndMove(1, 2);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(7u, f.docsMoved().size());
+    EXPECT_TRUE(assertEqual(f._notReady.bucket(4), f._notReady.docs(4)[2], 2, 1, f.docsMoved()[6]));
+    EXPECT_EQUAL(3u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._notReady.bucket(4), f.bucketsModified()[2]);
+}
+
+
+TEST_F("require that we can change calculator and continue scanning where we left off", ControllerFixture)
+{
+    // no buckets should move
+    // original scan sequence is bucket1, bucket2, bucket3, bucket4
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(2));
+
+    // start with bucket2
+    f._bmj.scanAndMove(1, 0);
+    f.changeCalc();
+    f._bmj.scanAndMove(5, 0);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(2),    f.calcAsked()[0]);
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[1]);
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[2]);
+    EXPECT_EQUAL(f._ready.bucket(1),    f.calcAsked()[3]);
+
+    // start with bucket3
+    f.changeCalc();
+    f._bmj.scanAndMove(2, 0);
+    f.changeCalc();
+    f._bmj.scanAndMove(5, 0);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[0]);
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[1]);
+    EXPECT_EQUAL(f._ready.bucket(1),    f.calcAsked()[2]);
+    EXPECT_EQUAL(f._ready.bucket(2),    f.calcAsked()[3]);
+
+    // start with bucket4
+    f.changeCalc();
+    f._bmj.scanAndMove(3, 0);
+    f.changeCalc();
+    f._bmj.scanAndMove(5, 0);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[0]);
+    EXPECT_EQUAL(f._ready.bucket(1),    f.calcAsked()[1]);
+    EXPECT_EQUAL(f._ready.bucket(2),    f.calcAsked()[2]);
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[3]);
+
+    // start with bucket1
+    f.changeCalc();
+    f._bmj.scanAndMove(5, 0);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1),    f.calcAsked()[0]);
+    EXPECT_EQUAL(f._ready.bucket(2),    f.calcAsked()[1]);
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[2]);
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[3]);
+
+    // change calc in second pass
+    f.changeCalc();
+    f._bmj.scanAndMove(3, 0);
+    f.changeCalc();
+    f._bmj.scanAndMove(2, 0);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(2u, f.calcAsked().size());
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[0]);
+    EXPECT_EQUAL(f._ready.bucket(1),    f.calcAsked()[1]);
+    f.changeCalc();
+    f._bmj.scanAndMove(5, 0);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(2),    f.calcAsked()[0]);
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[1]);
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[2]);
+    EXPECT_EQUAL(f._ready.bucket(1),    f.calcAsked()[3]);
+
+    // check 1 bucket at a time, start with bucket2
+    f.changeCalc();
+    f._bmj.scanAndMove(1, 0);
+    f.changeCalc();
+    f._bmj.scanAndMove(1, 0);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(1u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(2), f.calcAsked()[0]);
+    f._bmj.scanAndMove(1, 0);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(2u, f.calcAsked().size());
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[1]);
+    f._bmj.scanAndMove(1, 0);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.calcAsked().size());
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[2]);
+    f._bmj.scanAndMove(1, 0);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[3]);
+}
+
+
+TEST_F("require that current bucket moving is cancelled when we change calculator", ControllerFixture)
+{
+    // bucket 1 should be moved
+    f.addReady(f._ready.bucket(2));
+    f._bmj.scanAndMove(3, 1);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(1u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.calcAsked().size());
+    f.changeCalc(); // Not cancelled, bucket 1 still moving to notReady
+    EXPECT_EQUAL(1u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[0]);
+    f._calc->resetAsked();
+    f._bmj.scanAndMove(2, 1);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(1u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.calcAsked().size());
+    f.addReady(f._ready.bucket(1));
+    f.changeCalc(); // cancelled, bucket 1 no longer moving to notReady
+    EXPECT_EQUAL(1u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[0]);
+    f._calc->resetAsked();
+    f.remReady(f._ready.bucket(1));
+    f.changeCalc(); // not cancelled.  No active bucket move
+    EXPECT_EQUAL(0u, f.calcAsked().size());
+    f._calc->resetAsked();
+    f._bmj.scanAndMove(2, 1);
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(2u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(2), f.calcAsked()[0]);
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[1]);
+    f._bmj.scanAndMove(2, 3);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[2]);
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[3]);
+}
+
+
+TEST_F("require that last bucket is moved before reporting done", ControllerFixture)
+{
+    // bucket 4 should be moved
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(2));
+    f.addReady(f._notReady.bucket(4));
+    f._bmj.scanAndMove(4, 1);
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(1u, f.docsMoved().size());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    f._bmj.scanAndMove(0, 2);
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+}
+
+
+TEST_F("require that frozen bucket is not moved until thawed", ControllerFixture)
+{
+    // bucket 1 should be moved but is frozen
+    f.addReady(f._ready.bucket(2));
+    f.addFrozen(f._ready.bucket(1));
+    f._bmj.scanAndMove(4, 3); // scan all, delay frozen bucket 1
+    f.remFrozen(f._ready.bucket(1));
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    f._bmj.scanAndMove(0, 3); // move delayed and thawed bucket 1
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.bucketsModified()[0]);
+}
+
+
+TEST_F("require that thawed bucket is moved before other buckets", ControllerFixture)
+{
+    // bucket 2 should be moved but is frozen.
+    // bucket 3 & 4 should also be moved
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._notReady.bucket(3));
+    f.addReady(f._notReady.bucket(4));
+    f.addFrozen(f._ready.bucket(2));
+    f._bmj.scanAndMove(3, 2); // delay bucket 2, move bucket 3
+    f.remFrozen(f._ready.bucket(2));
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(2u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._notReady.bucket(3), f.bucketsModified()[0]);
+    f._bmj.scanAndMove(2, 2); // move thawed bucket 2
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(4u, f.docsMoved().size());
+    EXPECT_EQUAL(2u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(2), f.bucketsModified()[1]);
+    f._bmj.scanAndMove(1, 4); // move bucket 4
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(7u, f.docsMoved().size());
+    EXPECT_EQUAL(3u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._notReady.bucket(4), f.bucketsModified()[2]);
+}
+
+
+TEST_F("require that re-frozen thawed bucket is not moved until re-thawed", ControllerFixture)
+{
+    // bucket 1 should be moved but is re-frozen
+    f.addReady(f._ready.bucket(2));
+    f.addFrozen(f._ready.bucket(1));
+    f._bmj.scanAndMove(1, 0); // scan, delay frozen bucket 1
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(1u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[0]);
+    f.remFrozen(f._ready.bucket(1));
+    f.addFrozen(f._ready.bucket(1));
+    f._bmj.scanAndMove(1, 0); // scan, but nothing to move
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(3u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[1]);
+    EXPECT_EQUAL(f._ready.bucket(2), f.calcAsked()[2]);
+    f.remFrozen(f._ready.bucket(1));
+    f._bmj.scanAndMove(3, 4); // move delayed and thawed bucket 1
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.bucketsModified()[0]);
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[3]);
+    f._bmj.scanAndMove(2, 0); // scan the rest
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(6u, f.calcAsked().size());
+}
+
+
+TEST_F("require that thawed bucket is not moved if new calculator does not say so", ControllerFixture)
+{
+    // bucket 3 should be moved
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(2));
+    f.addReady(f._notReady.bucket(3));
+    f.addFrozen(f._notReady.bucket(3));
+    f._bmj.scanAndMove(4, 3); // scan all, delay frozen bucket 3
+    f.remFrozen(f._notReady.bucket(3));
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+    f.changeCalc();
+    f.remReady(f._notReady.bucket(3));
+    f._bmj.scanAndMove(0, 3); // consider delayed bucket 3
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(1u, f.calcAsked().size());
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[0]);
+}
+
+
+TEST_F("require that current bucket mover is cancelled if bucket is frozen", ControllerFixture)
+{
+    // bucket 3 should be moved
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(2));
+    f.addReady(f._notReady.bucket(3));
+    f._bmj.scanAndMove(3, 1); // move 1 doc from bucket 3
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(1u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(3u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[0]);
+    EXPECT_EQUAL(f._ready.bucket(2), f.calcAsked()[1]);
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[2]);
+
+    f.addFrozen(f._notReady.bucket(3));
+    f._bmj.scanAndMove(1, 3); // done scanning
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(1u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(3u, f.calcAsked().size());
+
+    f._bmj.scanAndMove(1, 3); // done scanning
+    f.remFrozen(f._notReady.bucket(3));
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(1u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(4u, f.calcAsked().size());
+
+    EXPECT_EQUAL(f._notReady.bucket(4), f.calcAsked()[3]);
+    f._bmj.scanAndMove(0, 2); // move all docs from bucket 3 again
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._notReady.bucket(3), f.bucketsModified()[0]);
+    EXPECT_EQUAL(5u, f.calcAsked().size());
+    EXPECT_EQUAL(f._notReady.bucket(3), f.calcAsked()[4]);
+}
+
+
+TEST_F("require that current bucket mover is not cancelled if another bucket is frozen", ControllerFixture)
+{
+    // bucket 3 and 4 should be moved
+    f.addReady(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(2));
+    f.addReady(f._notReady.bucket(3));
+    f.addReady(f._notReady.bucket(4));
+    f._bmj.scanAndMove(3, 1); // move 1 doc from bucket 3
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(1u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(3u, f.calcAsked().size());
+    f.addFrozen(f._notReady.bucket(4));
+    f._bmj.scanAndMove(1, 2); // move rest of docs from bucket 3
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(2u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._notReady.bucket(3), f.bucketsModified()[0]);
+    EXPECT_EQUAL(3u, f.calcAsked().size());
+}
+
+
+TEST_F("require that active bucket is not moved from ready to not ready until being not active", ControllerFixture)
+{
+    // bucket 1 should be moved but is active
+    f.addReady(f._ready.bucket(2));
+    f.activateBucket(f._ready.bucket(1));
+    f._bmj.scanAndMove(4, 3); // scan all, delay active bucket 1
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+
+    f.deactivateBucket(f._ready.bucket(1));
+    EXPECT_FALSE(f._bmj.done());
+    f._bmj.scanAndMove(0, 3); // move delayed and de-activated bucket 1
+    EXPECT_TRUE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.bucketsModified()[0]);
+}
+
+
+TEST_F("require that de-activated bucket is moved before other buckets", OnlyReadyControllerFixture)
+{
+    // bucket 1, 2, 3 should be moved (but bucket 1 is active)
+    f.addReady(f._ready.bucket(4));
+    f.activateBucket(f._ready.bucket(1));
+    f._bmj.scanAndMove(2, 4); // delay bucket 1, move bucket 2
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(2u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(2), f.bucketsModified()[0]);
+
+    f.deactivateBucket(f._ready.bucket(1));
+    f._bmj.scanAndMove(2, 4); // move de-activated bucket 1
+    EXPECT_FALSE(f._bmj.done());
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(2u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.bucketsModified()[1]);
+
+    f._bmj.scanAndMove(2, 4); // move bucket 3
+    // EXPECT_TRUE(f._bmj.done()); // TODO(geirst): fix this
+    EXPECT_EQUAL(6u, f.docsMoved().size());
+    EXPECT_EQUAL(3u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(3), f.bucketsModified()[2]);
+}
+
+
+TEST_F("require that de-activated bucket is not moved if new calculator does not say so", ControllerFixture)
+{
+    // bucket 1 should be moved
+    f.addReady(f._ready.bucket(2));
+    f.activateBucket(f._ready.bucket(1));
+    f._bmj.scanAndMove(4, 3); // scan all, delay active bucket 1
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+
+    f.deactivateBucket(f._ready.bucket(1));
+    f.addReady(f._ready.bucket(1));
+    f.changeCalc();
+    f._bmj.scanAndMove(0, 3); // consider delayed bucket 3
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+    EXPECT_EQUAL(1u, f.calcAsked().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.calcAsked()[0]);
+}
+
+
+TEST_F("require that de-activated bucket is not moved if frozen as well", ControllerFixture)
+{
+    // bucket 1 should be moved
+    f.addReady(f._ready.bucket(2));
+    f.activateBucket(f._ready.bucket(1));
+    f._bmj.scanAndMove(4, 3); // scan all, delay active bucket 1
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+
+    f.addFrozen(f._ready.bucket(1));
+    f.deactivateBucket(f._ready.bucket(1));
+    f._bmj.scanAndMove(0, 3); // bucket 1 de-activated but frozen
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+
+    f.remFrozen(f._ready.bucket(1));
+    f._bmj.scanAndMove(0, 3); // handle thawed bucket 1
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.bucketsModified()[0]);
+}
+
+
+TEST_F("require that thawed bucket is not moved if active as well", ControllerFixture)
+{
+    // bucket 1 should be moved
+    f.addReady(f._ready.bucket(2));
+    f.addFrozen(f._ready.bucket(1));
+    f._bmj.scanAndMove(4, 3); // scan all, delay frozen bucket 1
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+
+    f.activateBucket(f._ready.bucket(1));
+    f.remFrozen(f._ready.bucket(1));
+    f._bmj.scanAndMove(0, 3); // bucket 1 thawed but active
+    EXPECT_EQUAL(0u, f.docsMoved().size());
+    EXPECT_EQUAL(0u, f.bucketsModified().size());
+
+    f.deactivateBucket(f._ready.bucket(1));
+    f._bmj.scanAndMove(0, 3); // handle de-activated bucket 1
+    EXPECT_EQUAL(3u, f.docsMoved().size());
+    EXPECT_EQUAL(1u, f.bucketsModified().size());
+    EXPECT_EQUAL(f._ready.bucket(1), f.bucketsModified()[0]);
+}
+
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
+
diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
new file mode 100644
index 00000000000..cba08197b56
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
@@ -0,0 +1,218 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("documentdb_test");
+
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/messagebus/emptyreply.h>
+#include <vespa/messagebus/testlib/receptor.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/attribute/flushableattribute.h>
+#include <vespa/searchcore/proton/docsummary/summaryflushtarget.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h>
+#include <vespa/searchcore/proton/flushengine/threadedflushtarget.h>
+#include <vespa/searchcore/proton/server/document_db_explorer.h>
+#include <vespa/searchcore/proton/server/documentdb.h>
+#include <vespa/searchcore/proton/server/memoryconfigstore.h>
+#include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h>
+#include <vespa/searchcore/proton/metrics/metricswireservice.h>
+#include <vespa/searchcorespi/index/indexflushtarget.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/transactionlog/translogserver.h>
+#include <tests/proton/common/dummydbowner.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using document::DocumentType;
+using document::DocumentTypeRepo;
+using search::index::Schema;
+using search::transactionlog::TransLogServer;
+using namespace proton;
+using namespace vespalib::slime;
+using search::TuneFileDocumentDB;
+using document::DocumenttypesConfig;
+using search::index::DummyFileHeaderContext;
+using searchcorespi::index::IndexFlushTarget;
+using vespa::config::search::core::ProtonConfig;
+using vespalib::Slime;
+
+namespace {
+
+class LocalTransport : public FeedToken::ITransport {
+    mbus::Receptor _receptor;
+
+public:
+    void send(mbus::Reply::UP reply) {
+        fprintf(stderr, "in local transport.");
+        _receptor.handleReply(std::move(reply));
+    }
+
+    mbus::Reply::UP getReply() {
+        return _receptor.getReply(10000);
+    }
+};
+
+struct Fixture {
+    DummyWireService _dummy;
+    DummyDBOwner _dummyDBOwner;
+    vespalib::ThreadStackExecutor _summaryExecutor;
+    DocumentDB::SP _db;
+    DummyFileHeaderContext _fileHeaderContext;
+    TransLogServer _tls;
+    matching::QueryLimiter _queryLimiter;
+    vespalib::Clock _clock;
+
+    Fixture();
+};
+
+Fixture::Fixture()
+    : _summaryExecutor(8, 128*1024),
+      _tls("tmp", 9014, ".", _fileHeaderContext) {
+
+    DocumentDBConfig::DocumenttypesConfigSP documenttypesConfig(new DocumenttypesConfig());
+    DocumentType docType("typea", 0);
+    DocumentTypeRepo::SP repo(new DocumentTypeRepo(docType));
+    TuneFileDocumentDB::SP tuneFileDocumentDB(new TuneFileDocumentDB);
+    config::DirSpec spec("cfg");
+    DocumentDBConfigHelper mgr(spec, "typea");
+    BootstrapConfig::SP
+        b(new BootstrapConfig(1,
+                              documenttypesConfig,
+                              repo,
+                              BootstrapConfig::ProtonConfigSP(new ProtonConfig()),
+                              tuneFileDocumentDB));
+    mgr.forwardConfig(b);
+    mgr.nextGeneration(0);
+    _db.reset(new DocumentDB(".", mgr.getConfig(), "tcp/localhost:9014",
+                             _queryLimiter, _clock, DocTypeName("typea"),
+                             ProtonConfig(),
+                             _dummyDBOwner, _summaryExecutor, _summaryExecutor, NULL, _dummy, _fileHeaderContext,
+                             ConfigStore::UP(new MemoryConfigStore),
+                             std::make_shared<vespalib::ThreadStackExecutor>
+                             (16, 128 * 1024)));
+    _db->start();
+    _db->waitForOnlineState();
+}
+
+const IFlushTarget *
+extractRealFlushTarget(const IFlushTarget *target)
+{
+    const JobTrackedFlushTarget *tracked =
+            dynamic_cast<const JobTrackedFlushTarget*>(target);
+    if (tracked != nullptr) {
+        const ThreadedFlushTarget *threaded =
+                dynamic_cast<const ThreadedFlushTarget*>(&tracked->getTarget());
+        if (threaded != nullptr) {
+            return threaded->getFlushTarget().get();
+        }
+    }
+    return nullptr;
+}
+
+TEST_F("requireThatIndexFlushTargetIsUsed", Fixture) {
+    std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets();
+    ASSERT_TRUE(!targets.empty());
+    const IndexFlushTarget *index = 0;
+    for (size_t i = 0; i < targets.size(); ++i) {
+        const IFlushTarget *target = extractRealFlushTarget(targets[i].get());
+        if (target != NULL) {
+            index = dynamic_cast<const IndexFlushTarget *>(target);
+        }
+        if (index) {
+            break;
+        }
+    }
+    ASSERT_TRUE(index);
+}
+
+template <typename Target>
+size_t getNumTargets(const std::vector<IFlushTarget::SP> & targets)
+{
+    size_t retval = 0;
+    for (size_t i = 0; i < targets.size(); ++i) {
+        const IFlushTarget *target = extractRealFlushTarget(targets[i].get());
+        if (dynamic_cast<const Target*>(target) == NULL) {
+            continue;
+        }
+        retval++;
+    }
+    return retval;
+}
+
+TEST_F("requireThatFlushTargetsAreNamedBySubDocumentDB", Fixture) {
+    std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets();
+    ASSERT_TRUE(!targets.empty());
+    for (const IFlushTarget::SP & target : f._db->getFlushTargets()) {
+        vespalib::string name = target->getName();
+        EXPECT_TRUE((name.find("0.ready.") == 0) ||
+                    (name.find("1.removed.") == 0) ||
+                    (name.find("2.notready.") == 0));
+    }
+}
+
+TEST_F("requireThatAttributeFlushTargetsAreUsed", Fixture) {
+    std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets();
+    ASSERT_TRUE(!targets.empty());
+    size_t numAttrs = getNumTargets<FlushableAttribute>(targets);
+    // attr1 defined in attributes.cfg
+    EXPECT_EQUAL(1u, numAttrs);
+}
+
+TEST_F("requireThatDocumentMetaStoreFlushTargetIsUsed", Fixture) {
+    std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets();
+    ASSERT_TRUE(!targets.empty());
+    size_t numMetaStores =
+        getNumTargets<DocumentMetaStoreFlushTarget>(targets);
+    // document meta store
+    EXPECT_EQUAL(3u, numMetaStores);
+}
+
+TEST_F("requireThatSummaryFlushTargetsIsUsed", Fixture) {
+    std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets();
+    ASSERT_TRUE(!targets.empty());
+    size_t num = getNumTargets<SummaryFlushTarget>(targets);
+    EXPECT_EQUAL(3u, num);
+}
+
+TEST_F("requireThatCorrectStatusIsReported", Fixture) {
+    StatusReport::UP report(f._db->reportStatus());
+    EXPECT_EQUAL("documentdb:typea", report->getComponent());
+    EXPECT_EQUAL(StatusReport::UPOK, report->getState());
+    EXPECT_EQUAL("", report->getMessage());
+}
+
+TEST_F("requireThatStateIsReported", Fixture)
+{
+    Slime slime;
+    SlimeInserter inserter(slime);
+    DocumentDBExplorer(f._db).get_state(inserter, false);
+
+    EXPECT_EQUAL(
+            "{\n"
+            "    \"documentType\": \"typea\",\n"
+            "    \"status\": {\n"
+            "        \"state\": \"ONLINE\",\n"
+            "        \"configState\": \"OK\"\n"
+            "    },\n"
+            "    \"documents\": {\n"
+            "        \"active\": 0,\n"
+            "        \"indexed\": 0,\n"
+            "        \"stored\": 0,\n"
+            "        \"removed\": 0\n"
+            "    }\n"
+            "}\n",
+            slime.toString());
+}
+
+TEST_F("require that session manager can be explored", Fixture)
+{
+    EXPECT_TRUE(DocumentDBExplorer(f._db).get_child("session").get() != nullptr);    
+}
+
+}  // namespace
+
+TEST_MAIN() {
+    DummyFileHeaderContext::setCreator("documentdb_test");
+    FastOS_File::MakeDirectory("typea");
+    TEST_RUN_ALL();
+    FastOS_FileInterface::EmptyAndRemoveDirectory("typea");
+}
diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.sh b/searchcore/src/tests/proton/documentdb/documentdb_test.sh
new file mode 100644
index 00000000000..272ecacbd8b
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdb_test.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+$VALGRIND ./searchcore_documentdb_test_app
+rm -rf typea tmp
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/.gitignore b/searchcore/src/tests/proton/documentdb/documentdbconfig/.gitignore
new file mode 100644
index 00000000000..18a34a296b9
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/.gitignore
@@ -0,0 +1 @@
+searchcore_documentdbconfig_test_app
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt
new file mode 100644
index 00000000000..b9105fc9c91
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_documentdbconfig_test_app
+    SOURCES
+    documentdbconfig_test.cpp
+    DEPENDS
+    searchcore_server
+)
+vespa_add_test(NAME searchcore_documentdbconfig_test_app COMMAND searchcore_documentdbconfig_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/DESC b/searchcore/src/tests/proton/documentdb/documentdbconfig/DESC
new file mode 100644
index 00000000000..e2893dff557
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/DESC
@@ -0,0 +1 @@
+DocumentDBConfig test. Take a look at documentdbconfig_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/FILES b/searchcore/src/tests/proton/documentdb/documentdbconfig/FILES
new file mode 100644
index 00000000000..5298158fdbd
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/FILES
@@ -0,0 +1 @@
+documentdbconfig_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp
new file mode 100644
index 00000000000..5f9de9ae545
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("documentdbconfig_test");
+
+#include <vespa/searchcore/proton/server/documentdbconfig.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+using namespace vespa::config::search;
+using std::shared_ptr;
+using std::make_shared;
+
+typedef shared_ptr<DocumentDBConfig> DDBCSP;
+
+namespace
+{
+
+DDBCSP
+getConfig(int64_t generation, const Schema::SP &schema,
+          shared_ptr<DocumentTypeRepo> repo,
+          const RankProfilesConfig &rankProfiles)
+{
+    return make_shared<DocumentDBConfig>(
+            generation,
+            make_shared<RankProfilesConfig>(rankProfiles),
+            make_shared<IndexschemaConfig>(),
+            make_shared<AttributesConfig>(),
+            make_shared<SummaryConfig>(),
+            make_shared<SummarymapConfig>(),
+            make_shared<summary::JuniperrcConfig>(),
+            make_shared<DocumenttypesConfig>(),
+            repo,
+            make_shared<TuneFileDocumentDB>(),
+            schema,
+            make_shared<DocumentDBMaintenanceConfig>(),
+            "client", "test");
+}
+
+}
+
+TEST("Test that makeReplayConfig drops unneeded configs")
+{
+    RankProfilesConfigBuilder rp;
+    using DDBC = DocumentDBConfig;
+    shared_ptr<DocumentTypeRepo> repo(make_shared<DocumentTypeRepo>());
+    Schema::SP schema(make_shared<Schema>());
+    DDBCSP cfg0 = getConfig(4, schema, repo, rp);
+    rp.rankprofile.resize(1);
+    RankProfilesConfigBuilder::Rankprofile &rpr = rp.rankprofile.back();
+    rpr.name = "dummy";
+    DDBCSP cfg1 = getConfig(4, schema, repo, rp);
+    EXPECT_FALSE(*cfg0 == *cfg1);
+    DDBCSP cfg2 = DocumentDBConfig::makeReplayConfig(cfg1);
+    EXPECT_TRUE(*cfg0 == *cfg2);
+    EXPECT_TRUE(cfg0->getOriginalConfig().get() == nullptr);
+    EXPECT_TRUE(cfg1->getOriginalConfig().get() == nullptr);
+    EXPECT_TRUE(cfg2->getOriginalConfig().get() == cfg1.get());
+    EXPECT_TRUE(DDBC::preferOriginalConfig(cfg0).get() == cfg0.get());
+    EXPECT_TRUE(DDBC::preferOriginalConfig(cfg1).get() == cfg1.get());
+    EXPECT_TRUE(DDBC::preferOriginalConfig(cfg2).get() == cfg1.get());
+    DDBCSP cfg3;
+    EXPECT_TRUE(DDBC::preferOriginalConfig(cfg3).get() == nullptr);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/.gitignore b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/.gitignore
new file mode 100644
index 00000000000..482e85e5db0
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/.gitignore
@@ -0,0 +1 @@
+searchcore_documentdbconfigscout_test_app
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt
new file mode 100644
index 00000000000..e1dea56782d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_documentdbconfigscout_test_app
+    SOURCES
+    documentdbconfigscout_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_attribute
+)
+vespa_add_test(NAME searchcore_documentdbconfigscout_test_app COMMAND searchcore_documentdbconfigscout_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/DESC b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/DESC
new file mode 100644
index 00000000000..6585b4bc2b2
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/DESC
@@ -0,0 +1 @@
+DocumentDBConfigScout test. Take a look at documentdbconfigscout_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/FILES b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/FILES
new file mode 100644
index 00000000000..38b76884ae0
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/FILES
@@ -0,0 +1 @@
+documentdbconfigscout_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp
new file mode 100644
index 00000000000..e935fff1431
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp
@@ -0,0 +1,264 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("documentdbconfigscout_test");
+
+#include <vespa/searchcore/proton/server/documentdbconfig.h>
+#include <vespa/searchcore/proton/server/documentdbconfigscout.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+using namespace vespa::config::search;
+using std::shared_ptr;
+using std::make_shared;
+
+typedef shared_ptr<DocumentDBConfig> DDBCSP;
+
+namespace
+{
+
+DDBCSP
+getConfig(int64_t generation, const Schema::SP &schema,
+          shared_ptr<DocumentTypeRepo> repo,
+          const AttributesConfig &attributes)
+{
+    return make_shared<DocumentDBConfig>(
+            generation,
+            make_shared<RankProfilesConfig>(),
+            make_shared<IndexschemaConfig>(),
+            make_shared<AttributesConfig>(attributes),
+            make_shared<SummaryConfig>(),
+            make_shared<SummarymapConfig>(),
+            make_shared<summary::JuniperrcConfig>(),
+            make_shared<DocumenttypesConfig>(),
+            repo,
+            make_shared<TuneFileDocumentDB>(),
+            schema,
+            make_shared<DocumentDBMaintenanceConfig>(),
+            "client", "test");
+}
+
+
+bool
+assertDefaultAttribute(const AttributesConfig::Attribute &attribute,
+                       const vespalib::string &name)
+{
+    if (!EXPECT_EQUAL(name, attribute.name)) {
+        return false;
+    }
+    if (!EXPECT_FALSE(attribute.fastsearch)) {
+        return false;
+    }
+    if (!EXPECT_FALSE(attribute.huge)) {
+        return false;
+    }
+    if (!EXPECT_FALSE(attribute.enablebitvectors)) {
+        return false;
+    }
+    if (!EXPECT_FALSE(attribute.enableonlybitvector)) {
+        return false;
+    }
+    return true;
+}
+
+
+bool
+assertFastSearchAttribute(const AttributesConfig::Attribute &attribute,
+                          const vespalib::string &name)
+{
+    if (!EXPECT_EQUAL(name, attribute.name)) {
+        return false;
+    }
+    if (!EXPECT_TRUE(attribute.fastsearch)) {
+        return false;
+    }
+    if (!EXPECT_FALSE(attribute.huge)) {
+        return false;
+    }
+    if (!EXPECT_FALSE(attribute.enablebitvectors)) {
+        return false;
+    }
+    if (!EXPECT_FALSE(attribute.enableonlybitvector)) {
+        return false;
+    }
+    return true;
+}
+
+
+bool
+assertFastSearchAndMoreAttribute(const AttributesConfig::Attribute &attribute,
+                                 const vespalib::string &name)
+{
+    if (!EXPECT_EQUAL(name, attribute.name)) {
+        return false;
+    }
+    if (!EXPECT_TRUE(attribute.fastsearch)) {
+        return false;
+    }
+    if (!EXPECT_TRUE(attribute.huge)) {
+        return false;
+    }
+    if (!EXPECT_TRUE(attribute.enablebitvectors)) {
+        return false;
+    }
+    if (!EXPECT_TRUE(attribute.enableonlybitvector)) {
+        return false;
+    }
+    return true;
+}
+
+
+bool
+assertAttributes(const AttributesConfig::AttributeVector &attributes)
+{
+    if (!EXPECT_EQUAL(4u, attributes.size())) {
+        return false;
+    }
+    if (!assertDefaultAttribute(attributes[0], "a1")) {
+        return false;
+    }
+    if (!assertDefaultAttribute(attributes[1], "a2")) {
+        return false;
+    }
+    if (!assertDefaultAttribute(attributes[2], "a3")) {
+        return false;
+    }
+    if (!assertDefaultAttribute(attributes[3], "a4")) {
+        return false;
+    }
+    return true;
+}
+
+
+bool
+assertLiveAttributes(const AttributesConfig::AttributeVector &attributes)
+{
+    if (!EXPECT_EQUAL(5u, attributes.size())) {
+        return false;
+    }
+    if (!assertFastSearchAttribute(attributes[0], "a0")) {
+        return false;
+    }
+    if (!assertFastSearchAndMoreAttribute(attributes[1], "a1")) {
+        return false;
+    }
+    if (!assertFastSearchAttribute(attributes[2], "a2")) {
+        return false;
+    }
+    if (!assertFastSearchAttribute(attributes[3], "a3")) {
+        return false;
+    }
+    if (!assertFastSearchAttribute(attributes[4], "a4")) {
+        return false;
+    }
+    return true;
+}
+
+
+bool
+assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes)
+{
+    if (!EXPECT_EQUAL(4u, attributes.size())) {
+        return false;
+    }
+    if (!assertFastSearchAndMoreAttribute(attributes[0], "a1")) {
+        return false;
+    }
+    if (!assertDefaultAttribute(attributes[1], "a2")) {
+        return false;
+    }
+    if (!assertDefaultAttribute(attributes[2], "a3")) {
+        return false;
+    }
+    if (!assertDefaultAttribute(attributes[3], "a4")) {
+        return false;
+    }
+    return true;
+}
+
+
+AttributesConfig::Attribute
+setupDefaultAttribute(const vespalib::string name)
+{
+    AttributesConfig::Attribute attribute;
+    attribute.name = name;
+    return attribute;
+}
+
+
+AttributesConfig::Attribute
+setupFastSearchAttribute(const vespalib::string name)
+{
+    AttributesConfig::Attribute attribute;
+    attribute.name = name;
+    attribute.fastsearch = true;
+    return attribute;
+}
+
+
+AttributesConfig::Attribute
+setupFastSearchAndMoreAttribute(const vespalib::string name)
+{
+    AttributesConfig::Attribute attribute;
+    attribute.name = name;
+    attribute.fastsearch = true;
+    attribute.huge = true;
+    attribute.enablebitvectors = true;
+    attribute.enableonlybitvector = true;
+    return attribute;
+}
+
+
+void
+setupDefaultAttributes(AttributesConfigBuilder::AttributeVector &attributes)
+{
+    attributes.push_back(setupDefaultAttribute("a1"));
+    attributes.push_back(setupDefaultAttribute("a2"));
+    attributes.push_back(setupDefaultAttribute("a3"));
+    attributes.push_back(setupDefaultAttribute("a4"));
+}
+
+
+void
+setupLiveAttributes(AttributesConfigBuilder::AttributeVector &attributes)
+{
+    attributes.push_back(setupFastSearchAttribute("a0"));
+    attributes.push_back(setupFastSearchAndMoreAttribute("a1"));
+    attributes.push_back(setupFastSearchAttribute("a2"));
+    attributes.back().datatype = AttributesConfig::Attribute::INT8;
+    attributes.push_back(setupFastSearchAttribute("a3"));
+    attributes.back().collectiontype = AttributesConfig::Attribute::ARRAY;
+    attributes.push_back(setupFastSearchAttribute("a4"));
+    attributes.back().createifnonexistent = true;
+}
+
+}
+
+TEST("Test that DocumentDBConfigScout::scout looks ahead")
+{
+    AttributesConfigBuilder attributes;
+    setupDefaultAttributes(attributes.attribute);
+    
+    AttributesConfigBuilder liveAttributes;
+    setupLiveAttributes(liveAttributes.attribute);
+
+    shared_ptr<DocumentTypeRepo> repo(make_shared<DocumentTypeRepo>());
+    Schema::SP schema(make_shared<Schema>());
+    DDBCSP cfg = getConfig(4, schema, repo, attributes);
+    DDBCSP liveCfg = getConfig(4, schema, repo, liveAttributes);
+    EXPECT_FALSE(*cfg == *liveCfg);
+    DDBCSP scoutedCfg = DocumentDBConfigScout::scout(cfg, *liveCfg);
+    EXPECT_FALSE(*cfg == *scoutedCfg);
+    EXPECT_FALSE(*liveCfg == *scoutedCfg);
+    
+    EXPECT_TRUE(assertAttributes(cfg->getAttributesConfig().attribute));
+    EXPECT_TRUE(assertLiveAttributes(liveCfg->getAttributesConfig().attribute));
+    EXPECT_TRUE(assertScoutedAttributes(scoutedCfg->getAttributesConfig().
+                                        attribute));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/.gitignore b/searchcore/src/tests/proton/documentdb/feedhandler/.gitignore
new file mode 100644
index 00000000000..b464e44d740
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/.gitignore
@@ -0,0 +1 @@
+searchcore_feedhandler_test_app
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt
new file mode 100644
index 00000000000..756c11f35b4
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt
@@ -0,0 +1,18 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_feedhandler_test_app
+    SOURCES
+    feedhandler_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_bucketdb
+    searchcore_persistenceengine
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_feedhandler_test_app COMMAND sh feedhandler_test.sh)
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/DESC b/searchcore/src/tests/proton/documentdb/feedhandler/DESC
new file mode 100644
index 00000000000..c29d8c9a6e9
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/DESC
@@ -0,0 +1 @@
+feedhandler test. Take a look at feedhandler_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/FILES b/searchcore/src/tests/proton/documentdb/feedhandler/FILES
new file mode 100644
index 00000000000..483ca5f0fcd
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/FILES
@@ -0,0 +1 @@
+feedhandler_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
new file mode 100644
index 00000000000..8a7f646383f
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
@@ -0,0 +1,748 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("feedhandler_test");
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/documentapi/messagebus/messages/documentreply.h>
+#include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
+#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
+#include <vespa/persistence/spi/result.h>
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchcore/proton/feedoperation/wipehistoryoperation.h>
+#include <vespa/searchcore/proton/server/configstore.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+#include <vespa/searchcore/proton/server/feedhandler.h>
+#include <vespa/searchcore/proton/server/ddbstate.h>
+#include <vespa/searchcore/proton/test/dummy_feed_view.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
+#include <vespa/searchlib/transactionlog/translogserver.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
+
+using document::BucketId;
+using document::Document;
+using document::DocumentId;
+using document::DocumentTypeRepo;
+using document::DocumentUpdate;
+using document::GlobalId;
+using documentapi::DocumentProtocol;
+using documentapi::DocumentReply;
+using documentapi::RemoveDocumentReply;
+using documentapi::UpdateDocumentReply;
+using mbus::Reply;
+using search::index::DocBuilder;
+using search::index::DummyFileHeaderContext;
+using search::index::Schema;
+using search::SerialNum;
+using search::transactionlog::TransLogServer;
+using storage::spi::PartitionId;
+using storage::spi::RemoveResult;
+using storage::spi::Result;
+using storage::spi::Timestamp;
+using storage::spi::UpdateResult;
+using vespalib::BlockingThreadStackExecutor;
+using vespalib::ThreadStackExecutor;
+using vespalib::ThreadStackExecutorBase;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+using search::makeLambdaTask;
+using namespace proton;
+
+typedef std::unique_ptr<vespalib::CountDownLatch> CountDownLatchUP;
+
+namespace {
+
+struct Rendezvous {
+    vespalib::Gate enter;
+    vespalib::Gate leave;
+    vespalib::Gate gone;
+    typedef std::unique_ptr<Rendezvous> UP;
+    Rendezvous() : enter(), leave(), gone() {}
+    bool run(uint32_t timeout = 80000) {
+        enter.countDown();
+        bool retval = leave.await(timeout);
+        gone.countDown();
+        return retval;
+    }
+    bool waitForEnter(uint32_t timeout = 80000) {
+        return enter.await(timeout);
+    }
+    bool leaveAndWait(uint32_t timeout = 80000) {
+        leave.countDown();
+        return gone.await(timeout);
+    }
+    bool await(uint32_t timeout = 80000) {
+        if (waitForEnter(timeout)) {
+            return leaveAndWait(timeout);
+        }
+        return false;
+    }
+};
+
+
+struct MyOwner : public FeedHandler::IOwner
+{
+    bool rejected_config;
+    bool _allowPrune;
+    int wipe_history_count;
+    
+    MyOwner()
+        :
+        rejected_config(false),
+        _allowPrune(false),
+        wipe_history_count(0)
+    {
+    }
+    virtual void performWipeHistory() { ++wipe_history_count; }
+    virtual void onTransactionLogReplayDone() {
+        LOG(info, "MyOwner::onTransactionLogReplayDone()");
+    }
+    virtual void enterRedoReprocessState() {}
+    virtual void onPerformPrune(SerialNum) {}
+    virtual bool isFeedBlockedByRejectedConfig() { return rejected_config; }
+
+    virtual bool
+    getAllowPrune(void) const
+    {
+        return _allowPrune;
+    }
+};
+
+
+struct MyResourceWriteFilter : public IResourceWriteFilter
+{
+    bool _acceptWriteOperation;
+    vespalib::string _message;
+    MyResourceWriteFilter()
+        : _acceptWriteOperation(true),
+          _message()
+    {}
+
+    virtual bool acceptWriteOperation() const override { return _acceptWriteOperation; }
+    virtual State getAcceptState() const override {
+        return IResourceWriteFilter::State(acceptWriteOperation(), _message);
+    }
+};
+
+
+struct MyReplayConfig : public IReplayConfig {
+    virtual void replayConfig(SerialNum) {}
+    virtual void replayWipeHistory(SerialNum, fastos::TimeStamp) {}
+};
+
+void ackToken(FeedToken *token) {
+    if (token != NULL) {
+        token->ack();
+    }
+}
+
+struct MyDocumentMetaStore {
+    struct Entry {
+        DbDocumentId _id;
+        DbDocumentId _prevId;
+        Timestamp    _prevTimestamp;
+        Entry() : _id(0, 0), _prevId(0, 0), _prevTimestamp(0) {}
+        Entry(uint32_t lid, uint32_t prevLid, Timestamp prevTimestamp)
+            : _id(0, lid),
+              _prevId(0, prevLid),
+              _prevTimestamp(prevTimestamp)
+        {}
+    };
+    std::map<GlobalId, Entry> _pool;
+    std::map<GlobalId, Entry> _allocated;
+    MyDocumentMetaStore() : _pool(), _allocated() {}
+    MyDocumentMetaStore &insert(const GlobalId &gid, const Entry &e) {
+        _pool[gid] = e;
+        return *this;
+    }
+    MyDocumentMetaStore &allocate(const GlobalId &gid) {
+        auto itr = _pool.find(gid);
+        if (itr != _pool.end()) {
+            _allocated[gid] = itr->second;
+        }
+        return *this;
+    }
+    const Entry *get(const GlobalId &gid) const {
+        auto itr = _allocated.find(gid);
+        if (itr != _allocated.end()) {
+            return &itr->second;
+        }
+        return NULL;
+    }
+};
+
+struct MyFeedView : public test::DummyFeedView {
+    Rendezvous putRdz;
+    bool usePutRdz;
+    CountDownLatchUP putLatch;
+    MyDocumentMetaStore metaStore;
+    int put_count;
+    SerialNum put_serial;
+    int heartbeat_count;
+    int remove_count;
+    int move_count;
+    int prune_removed_count;
+    int update_count;
+    SerialNum update_serial;
+    MyFeedView(const DocumentTypeRepo::SP &dtr) :
+        test::DummyFeedView(dtr),
+        putRdz(),
+        usePutRdz(false),
+        putLatch(),
+        metaStore(),
+        put_count(0),
+        put_serial(0),
+        heartbeat_count(0),
+        remove_count(0),
+        move_count(0),
+        prune_removed_count(0),
+        update_count(0),
+        update_serial(0)
+    {
+    }
+    void resetPutLatch(uint32_t count) { putLatch.reset(new vespalib::CountDownLatch(count)); }
+    virtual void preparePut(PutOperation &op) {
+        prepareDocumentOperation(op, op.getDocument()->getId().getGlobalId());
+    }
+    void prepareDocumentOperation(DocumentOperation &op, const GlobalId &gid) {
+        const MyDocumentMetaStore::Entry *entry = metaStore.get(gid);
+        if (entry != NULL) {
+            op.setDbDocumentId(entry->_id);
+            op.setPrevDbDocumentId(entry->_prevId);
+            op.setPrevTimestamp(entry->_prevTimestamp);
+        }
+    }
+    virtual void handlePut(FeedToken *token, const PutOperation &putOp) {
+        LOG(info, "MyFeedView::handlePut(): docId(%s), putCount(%u), putLatchCount(%u)",
+            putOp.getDocument()->getId().toString().c_str(), put_count,
+            (putLatch.get() != NULL ? putLatch->getCount() : 0u));
+        if (usePutRdz) {
+            putRdz.run();
+        }
+        ++put_count;
+        put_serial = putOp.getSerialNum();
+        metaStore.allocate(putOp.getDocument()->getId().getGlobalId());
+        if (putLatch.get() != NULL) {
+            putLatch->countDown();
+        }
+        ackToken(token);
+    }
+    virtual void prepareUpdate(UpdateOperation &op) {
+        prepareDocumentOperation(op, op.getUpdate()->getId().getGlobalId());
+    }
+    virtual void handleUpdate(FeedToken *token, const UpdateOperation &op) {
+        ++update_count;
+        update_serial = op.getSerialNum();
+        ackToken(token);
+    }
+    virtual void handleRemove(FeedToken *token, const RemoveOperation &)
+    { ++remove_count; ackToken(token); }
+    virtual void handleMove(const MoveOperation &) { ++move_count; }
+    virtual void heartBeat(SerialNum) { ++heartbeat_count; }
+    virtual void handlePruneRemovedDocuments(
+            const PruneRemovedDocumentsOperation &) { ++prune_removed_count; }
+    virtual const ISimpleDocumentMetaStore *getDocumentMetaStorePtr() const {
+        return NULL;
+    }
+};
+
+
+struct SchemaContext {
+    Schema::SP                schema;
+    std::unique_ptr<DocBuilder> builder;
+    SchemaContext() :
+        schema(new Schema()),
+        builder()
+    {
+        schema->addIndexField(Schema::IndexField("i1", Schema::STRING, Schema::SINGLE));
+        builder.reset(new DocBuilder(*schema));
+    }
+    DocTypeName getDocType() const {
+        return DocTypeName(builder->getDocumentType().getName());
+    }
+    const document::DocumentTypeRepo::SP &getRepo() const { return builder->getDocumentTypeRepo(); }
+};
+
+
+struct DocumentContext {
+    Document::SP  doc;
+    BucketId      bucketId;
+    DocumentContext(const vespalib::string &docId, DocBuilder &builder) :
+        doc(builder.startDocument(docId).endDocument().release()),
+        bucketId(BucketFactory::getBucketId(doc->getId()))
+    {
+    }
+};
+
+
+struct UpdateContext {
+    DocumentUpdate::SP update;
+    BucketId           bucketId;
+    UpdateContext(const vespalib::string &docId, DocBuilder &builder) :
+        update(new DocumentUpdate(builder.getDocumentType(), DocumentId(docId))),
+        bucketId(BucketFactory::getBucketId(update->getId()))
+    {
+    }
+};
+
+
+struct MyTransport : public FeedToken::ITransport {
+    vespalib::Gate gate;
+    ResultUP result;
+    bool documentWasFound;
+    MyTransport() : gate(), result(), documentWasFound(false) {}
+    virtual void send(Reply::UP, ResultUP res, bool documentWasFound_, double) {
+        result = std::move(res);
+        documentWasFound = documentWasFound_;
+        gate.countDown();
+    }
+};
+
+Reply::UP getReply(uint32_t type) {
+    if (type == DocumentProtocol::REPLY_REMOVEDOCUMENT) {
+        return Reply::UP(new RemoveDocumentReply);
+    } else if (type == DocumentProtocol::REPLY_UPDATEDOCUMENT) {
+        return Reply::UP(new UpdateDocumentReply);
+    }
+    return Reply::UP(new DocumentReply(type));
+}
+
+struct FeedTokenContext {
+    MyTransport transport;
+    FeedToken::UP token_ap;
+    FeedToken &token;
+
+    FeedTokenContext(uint32_t type = 0) :
+        transport(),
+        token_ap(new FeedToken(transport, getReply(type))),
+        token(*token_ap) {
+        token.getReply().getTrace().setLevel(9);
+    }
+    bool await(uint32_t timeout = 80000)
+    { return transport.gate.await(timeout); }
+    const Result *getResult() {
+        if (transport.result.get()) {
+            return transport.result.get();
+        }
+        return &token.getResult();
+    }
+};
+
+
+struct PutContext {
+    FeedTokenContext tokenCtx;
+    DocumentContext  docCtx;
+    typedef std::shared_ptr<PutContext> SP;
+    PutContext(const vespalib::string &docId, DocBuilder &builder) :
+        tokenCtx(DocumentProtocol::REPLY_PUTDOCUMENT),
+        docCtx(docId, builder)
+    {
+    }
+};
+
+
+struct PutHandler {
+    FeedHandler &handler;
+    DocBuilder &builder;
+    Timestamp timestamp;
+    std::vector<PutContext::SP> puts;
+    PutHandler(FeedHandler &fh, DocBuilder &db) :
+        handler(fh),
+        builder(db),
+        timestamp(0),
+        puts()
+    {
+    }
+    void put(const vespalib::string &docId) {
+        PutContext::SP pc(new PutContext(docId, builder));
+        FeedOperation::UP op(new PutOperation(pc->docCtx.bucketId,
+                                              timestamp, pc->docCtx.doc));
+        handler.handleOperation(pc->tokenCtx.token, std::move(op));
+        timestamp = Timestamp(timestamp + 1);
+        puts.push_back(pc);
+    }
+    bool await(uint32_t timeout = 80000) {
+        for (size_t i = 0; i < puts.size(); ++i) {
+            if (!puts[i]->tokenCtx.await(timeout)) {
+                return false;
+            }
+        }
+        return true;
+    }
+};
+
+
+struct MyFeedMetrics : public metrics::MetricSet
+{
+    PerDocTypeFeedMetrics       _feed;
+
+    MyFeedMetrics(void)
+        : metrics::MetricSet("myfeedmetrics", "", "My feed metrics", NULL),
+          _feed(this)
+    {
+    }
+};
+
+
+struct MyTlsWriter : TlsWriter {
+    int store_count;
+    int erase_count;
+    bool erase_return;
+
+    MyTlsWriter() : store_count(0), erase_count(0), erase_return(true) {}
+    virtual void storeOperation(const FeedOperation &) { ++store_count; }
+    virtual bool erase(SerialNum) { ++erase_count; return erase_return; }
+
+    virtual SerialNum
+    sync(SerialNum syncTo)
+    {
+        return syncTo;
+    } 
+};
+
+
+struct FeedHandlerFixture
+{
+    DummyFileHeaderContext       _fileHeaderContext;
+    TransLogServer               tls;
+    vespalib::string             tlsSpec;
+    ExecutorThreadingService     writeService;
+    SchemaContext                schema;
+    MyOwner                      owner;
+    MyResourceWriteFilter        writeFilter;
+    DDBState                     _state;
+    MyReplayConfig               replayConfig;
+    MyFeedView                   feedView;
+    MyFeedMetrics                feedMetrics;
+    MyTlsWriter                  tls_writer;
+    BucketDBOwner                _bucketDB;
+    bucketdb::BucketDBHandler    _bucketDBHandler;
+    FeedHandler                  handler;
+    FeedHandlerFixture()
+        : _fileHeaderContext(),
+          tls("mytls", 9016, "mytlsdir", _fileHeaderContext, 0x10000),
+          tlsSpec("tcp/localhost:9016"),
+          writeService(),
+          schema(),
+          owner(),
+          _state(),
+          replayConfig(),
+          feedView(schema.getRepo()),
+          _bucketDB(),
+          _bucketDBHandler(_bucketDB),
+          handler(writeService, tlsSpec, schema.getDocType(),
+                  feedMetrics._feed, _state, owner, writeFilter, replayConfig, NULL, &tls_writer)
+    {
+        _state.enterLoadState();
+        _state.enterReplayTransactionLogState();
+        handler.setActiveFeedView(&feedView);
+        handler.setBucketDBHandler(&_bucketDBHandler);
+        handler.init(1);
+    }
+
+    ~FeedHandlerFixture()
+    {
+        writeService.sync();
+    }
+    template <class FunctionType>
+    inline void runAsMaster(FunctionType &&function) {
+        writeService.master().execute(makeLambdaTask(std::move(function)));
+        writeService.master().sync();
+    }
+    void syncMaster() {
+        writeService.master().sync();
+    }
+};
+
+
+struct MyConfigStore : ConfigStore {
+    virtual SerialNum getBestSerialNum() const { return 1; }
+    virtual SerialNum getOldestSerialNum() const { return 1; }
+    virtual void saveConfig(const DocumentDBConfig &,
+                            const search::index::Schema &, SerialNum) {}
+    virtual void loadConfig(const DocumentDBConfig &, SerialNum,
+                            DocumentDBConfig::SP &,
+                            search::index::Schema::SP &) {}
+    virtual void removeInvalid() {}
+    void prune(SerialNum) {}
+    virtual bool hasValidSerial(SerialNum) const { return true; }
+    virtual SerialNum getPrevValidSerial(SerialNum) const { return 1; }
+    virtual void saveWipeHistoryConfig(SerialNum,
+                                       fastos::TimeStamp) {}
+    virtual void serializeConfig(SerialNum, vespalib::nbostream &) {}
+    virtual void deserializeConfig(SerialNum, vespalib::nbostream &) {}
+    virtual void setProtonConfig(const ProtonConfigSP &) override { }
+};
+
+
+struct ReplayTransactionLogContext {
+    IIndexWriter::SP iwriter;
+    MyConfigStore config_store;
+    DocumentDBConfig::SP cfgSnap;
+};
+
+
+TEST_F("require that heartBeat calls FeedView's heartBeat",
+       FeedHandlerFixture)
+{
+    f.runAsMaster([&]() { f.handler.heartBeat(); });
+    EXPECT_EQUAL(1, f.feedView.heartbeat_count);
+}
+
+TEST_F("require that rejected config disables operations and heartbeat",
+       FeedHandlerFixture)
+{
+    f.owner.rejected_config = true;
+    f.handler.changeToNormalFeedState();
+    f.owner._allowPrune = true;
+
+    DocumentContext doc_context("doc:test:foo", *f.schema.builder);
+    FeedOperation::UP op(new PutOperation(doc_context.bucketId,
+                                          Timestamp(10), doc_context.doc));
+    FeedTokenContext token1;
+    f.handler.performOperation(std::move(token1.token_ap), std::move(op));
+    EXPECT_EQUAL(0, f.feedView.put_count);
+    EXPECT_EQUAL(Result::PERMANENT_ERROR, token1.getResult()->getErrorCode());
+
+    FeedTokenContext token2(DocumentProtocol::REPLY_REMOVEDOCUMENT);
+    op.reset(new RemoveOperation(doc_context.bucketId, Timestamp(10),
+                                 doc_context.doc->getId()));
+    f.handler.performOperation(std::move(token2.token_ap), std::move(op));
+    EXPECT_EQUAL(0, f.feedView.remove_count);
+    EXPECT_TRUE(dynamic_cast<const RemoveResult *>(token2.getResult()));
+    EXPECT_EQUAL(Result::PERMANENT_ERROR, token2.getResult()->getErrorCode());
+
+    FeedTokenContext token3(DocumentProtocol::REPLY_UPDATEDOCUMENT);
+    op.reset(new UpdateOperation(doc_context.bucketId, Timestamp(10),
+                                 document::DocumentUpdate::SP()));
+    f.handler.performOperation(std::move(token3.token_ap), std::move(op));
+    EXPECT_EQUAL(0, f.feedView.update_count);
+    EXPECT_TRUE(dynamic_cast<const UpdateResult *>(token3.getResult()));
+    EXPECT_EQUAL(Result::PERMANENT_ERROR, token3.getResult()->getErrorCode());
+
+    f.runAsMaster([&]() { f.handler.heartBeat(); });
+    EXPECT_EQUAL(0, f.feedView.heartbeat_count);
+
+    EXPECT_EQUAL(0, f.tls_writer.store_count);
+}
+
+TEST_F("require that outdated remove is ignored", FeedHandlerFixture)
+{
+    DocumentContext doc_context("doc:test:foo", *f.schema.builder);
+    FeedOperation::UP op(new RemoveOperation(doc_context.bucketId,
+                                             Timestamp(10),
+                                             doc_context.doc->getId()));
+    static_cast<DocumentOperation &>(*op).setPrevDbDocumentId(DbDocumentId(4));
+    static_cast<DocumentOperation &>(*op).setPrevTimestamp(Timestamp(10000));
+    FeedTokenContext token_context(DocumentProtocol::REPLY_REMOVEDOCUMENT);
+    f.handler.performOperation(std::move(token_context.token_ap), std::move(op));
+    EXPECT_EQUAL(0, f.feedView.remove_count);
+    EXPECT_EQUAL(0, f.tls_writer.store_count);
+}
+
+TEST_F("require that outdated put is ignored", FeedHandlerFixture)
+{
+    DocumentContext doc_context("doc:test:foo", *f.schema.builder);
+    FeedOperation::UP op(new PutOperation(doc_context.bucketId,
+                                          Timestamp(10), doc_context.doc));
+    static_cast<DocumentOperation &>(*op).setPrevTimestamp(Timestamp(10000));
+    FeedTokenContext token_context;
+    f.handler.performOperation(std::move(token_context.token_ap), std::move(op));
+    EXPECT_EQUAL(0, f.feedView.put_count);
+    EXPECT_EQUAL(0, f.tls_writer.store_count);
+}
+
+void
+addLidToRemove(RemoveDocumentsOperation &op)
+{
+    LidVectorContext::LP lids(new LidVectorContext(42));
+    lids->addLid(4);
+    op.setLidsToRemove(0, lids);
+}
+
+
+TEST_F("require that WipeHistory calls owner", FeedHandlerFixture)
+{
+    MyTransport transport;
+    FeedTokenContext token_context;
+    f.handler.performOperation(std::move(token_context.token_ap),
+                               FeedOperation::UP(new WipeHistoryOperation));
+    EXPECT_EQUAL(1, f.owner.wipe_history_count);
+    EXPECT_EQUAL(0, f.tls_writer.store_count);  // Not stored in tls.
+}
+
+TEST_F("require that handleMove calls FeedView", FeedHandlerFixture)
+{
+    DocumentContext doc_context("doc:test:foo", *f.schema.builder);
+    MoveOperation op(doc_context.bucketId, Timestamp(2), doc_context.doc,
+                     DbDocumentId(0, 2), 1);
+    op.setDbDocumentId(DbDocumentId(1, 2));
+    f.runAsMaster([&]() { f.handler.handleMove(op); });
+    EXPECT_EQUAL(1, f.feedView.move_count);
+    EXPECT_EQUAL(1, f.tls_writer.store_count);
+}
+
+TEST_F("require that performPruneRemovedDocuments calls FeedView",
+       FeedHandlerFixture)
+{
+    PruneRemovedDocumentsOperation op;
+    f.handler.performPruneRemovedDocuments(op);
+    EXPECT_EQUAL(0, f.feedView.prune_removed_count);
+    EXPECT_EQUAL(0, f.tls_writer.store_count);
+
+    addLidToRemove(op);
+    f.handler.performPruneRemovedDocuments(op);
+    EXPECT_EQUAL(1, f.feedView.prune_removed_count);
+    EXPECT_EQUAL(1, f.tls_writer.store_count);
+}
+
+TEST_F("require that failed prune throws", FeedHandlerFixture)
+{
+    f.tls_writer.erase_return = false;
+    EXPECT_EXCEPTION(f.handler.tlsPrune(10), vespalib::IllegalStateException,
+                     "Failed to prune TLS to token 10.");
+}
+
+TEST_F("require that flush done calls prune", FeedHandlerFixture)
+{
+    f.handler.changeToNormalFeedState();
+    f.owner._allowPrune = true;
+    f.handler.flushDone(10);
+    f.syncMaster();
+    EXPECT_EQUAL(1, f.tls_writer.erase_count);
+    EXPECT_EQUAL(10u, f.handler.getPrunedSerialNum());
+}
+
+TEST_F("require that flush in init state delays pruning", FeedHandlerFixture)
+{
+    f.handler.flushDone(10);
+    f.syncMaster();
+    EXPECT_EQUAL(0, f.tls_writer.erase_count);
+    EXPECT_EQUAL(10u, f.handler.getPrunedSerialNum());
+}
+
+TEST_F("require that flush cannot unprune", FeedHandlerFixture)
+{
+    f.handler.flushDone(10);
+    f.syncMaster();
+    EXPECT_EQUAL(10u, f.handler.getPrunedSerialNum());
+
+    f.handler.flushDone(5);  // Try to unprune.
+    f.syncMaster();
+    EXPECT_EQUAL(10u, f.handler.getPrunedSerialNum());
+}
+
+TEST_F("require that remove of unknown document with known data type "
+       "stores remove", FeedHandlerFixture)
+{
+    DocumentContext doc_context("id:test:searchdocument::foo",
+                                *f.schema.builder);
+    FeedOperation::UP op(new RemoveOperation(doc_context.bucketId,
+                                             Timestamp(10),
+                                             doc_context.doc->getId()));
+    FeedTokenContext token_context(DocumentProtocol::REPLY_REMOVEDOCUMENT);
+    f.handler.performOperation(std::move(token_context.token_ap), std::move(op));
+    EXPECT_EQUAL(1, f.feedView.remove_count);
+    EXPECT_EQUAL(1, f.tls_writer.store_count);
+}
+
+TEST_F("require that partial update for non-existing document is tagged as such",
+       FeedHandlerFixture)
+{
+    UpdateContext upCtx("id:test:searchdocument::foo", *f.schema.builder);
+    FeedOperation::UP op(new UpdateOperation(upCtx.bucketId,
+                                             Timestamp(10),
+                                             upCtx.update));
+    FeedTokenContext token_context(DocumentProtocol::REPLY_UPDATEDOCUMENT);
+    f.handler.performOperation(std::move(token_context.token_ap), std::move(op));
+    const UpdateResult *result = static_cast<const UpdateResult *>(token_context.getResult());
+
+    EXPECT_FALSE(token_context.transport.documentWasFound);
+    EXPECT_EQUAL(0u, result->getExistingTimestamp());
+    EXPECT_EQUAL(0, f.feedView.put_count);
+    EXPECT_EQUAL(0, f.feedView.update_count);
+    EXPECT_EQUAL(0, f.tls_writer.store_count);
+}
+
+TEST_F("require that partial update for non-existing document is created if specified",
+       FeedHandlerFixture)
+{
+    f.handler.setSerialNum(15);
+    UpdateContext upCtx("id:test:searchdocument::foo", *f.schema.builder);
+    upCtx.update->setCreateIfNonExistent(true);
+    f.feedView.metaStore.insert(upCtx.update->getId().getGlobalId(),
+                                MyDocumentMetaStore::Entry(5, 5, Timestamp(10)));
+    FeedOperation::UP op(new UpdateOperation(upCtx.bucketId,
+                                             Timestamp(10),
+                                             upCtx.update));
+    FeedTokenContext token_context(DocumentProtocol::REPLY_UPDATEDOCUMENT);
+    f.handler.performOperation(std::move(token_context.token_ap), std::move(op));
+    const UpdateResult *result = static_cast<const UpdateResult *>(token_context.getResult());
+
+    EXPECT_TRUE(token_context.transport.documentWasFound);
+    EXPECT_EQUAL(10u, result->getExistingTimestamp());
+    EXPECT_EQUAL(1, f.feedView.put_count);
+    EXPECT_EQUAL(16u, f.feedView.put_serial);
+    EXPECT_EQUAL(0, f.feedView.update_count);
+    EXPECT_EQUAL(0u, f.feedView.update_serial);
+    EXPECT_EQUAL(1u, f.feedView.metaStore._allocated.size());
+    EXPECT_EQUAL(1, f.tls_writer.store_count);
+}
+
+TEST_F("require that put is rejected if resource limit is reached", FeedHandlerFixture)
+{
+    f.writeFilter._acceptWriteOperation = false;
+    f.writeFilter._message = "Attribute resource limit reached";
+
+    DocumentContext docCtx("id:test:searchdocument::foo", *f.schema.builder);
+    FeedOperation::UP op = std::make_unique<PutOperation>(docCtx.bucketId, Timestamp(10), docCtx.doc);
+    FeedTokenContext token;
+    f.handler.performOperation(std::move(token.token_ap), std::move(op));
+    EXPECT_EQUAL(0, f.feedView.put_count);
+    EXPECT_EQUAL(Result::RESOURCE_EXHAUSTED, token.getResult()->getErrorCode());
+    EXPECT_EQUAL("Put operation rejected for document 'id:test:searchdocument::foo' of type 'searchdocument': 'Attribute resource limit reached'",
+                 token.getResult()->getErrorMessage());
+}
+
+TEST_F("require that update is rejected if resource limit is reached", FeedHandlerFixture)
+{
+    f.writeFilter._acceptWriteOperation = false;
+    f.writeFilter._message = "Attribute resource limit reached";
+
+    UpdateContext updCtx("id:test:searchdocument::foo", *f.schema.builder);
+    FeedOperation::UP op = std::make_unique<UpdateOperation>(updCtx.bucketId, Timestamp(10), updCtx.update);
+    FeedTokenContext token(DocumentProtocol::REPLY_UPDATEDOCUMENT);
+    f.handler.performOperation(std::move(token.token_ap), std::move(op));
+    EXPECT_EQUAL(0, f.feedView.update_count);
+    EXPECT_TRUE(dynamic_cast<const UpdateResult *>(token.getResult()));
+    EXPECT_EQUAL(Result::RESOURCE_EXHAUSTED, token.getResult()->getErrorCode());
+    EXPECT_EQUAL("Update operation rejected for document 'id:test:searchdocument::foo' of type 'searchdocument': 'Attribute resource limit reached'",
+                 token.getResult()->getErrorMessage());
+}
+
+TEST_F("require that remove is NOT rejected if resource limit is reached", FeedHandlerFixture)
+{
+    f.writeFilter._acceptWriteOperation = false;
+    f.writeFilter._message = "Attribute resource limit reached";
+
+    DocumentContext docCtx("id:test:searchdocument::foo", *f.schema.builder);
+    FeedOperation::UP op = std::make_unique<RemoveOperation>(docCtx.bucketId, Timestamp(10), docCtx.doc->getId());
+    FeedTokenContext token(DocumentProtocol::REPLY_REMOVEDOCUMENT);
+    f.handler.performOperation(std::move(token.token_ap), std::move(op));
+    EXPECT_EQUAL(1, f.feedView.remove_count);
+    EXPECT_EQUAL(Result::NONE, token.getResult()->getErrorCode());
+    EXPECT_EQUAL("", token.getResult()->getErrorMessage());
+}
+
+}  // namespace
+
+TEST_MAIN()
+{
+    DummyFileHeaderContext::setCreator("feedhandler_test");
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.sh b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.sh
new file mode 100644
index 00000000000..bc49b207155
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+$VALGRIND ./searchcore_feedhandler_test_app
+rm -rf mytlsdir
+rm -rf myfilecfg
diff --git a/searchcore/src/tests/proton/documentdb/feedview/.gitignore b/searchcore/src/tests/proton/documentdb/feedview/.gitignore
new file mode 100644
index 00000000000..596e11ac15a
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedview/.gitignore
@@ -0,0 +1 @@
+searchcore_feedview_test_app
diff --git a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt
new file mode 100644
index 00000000000..0ea395a339a
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_feedview_test_app
+    SOURCES
+    feedview_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_index
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_feedview_test_app COMMAND searchcore_feedview_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/feedview/DESC b/searchcore/src/tests/proton/documentdb/feedview/DESC
new file mode 100644
index 00000000000..4c35f3d2a61
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedview/DESC
@@ -0,0 +1 @@
+feedview test. Take a look at feedview_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/feedview/FILES b/searchcore/src/tests/proton/documentdb/feedview/FILES
new file mode 100644
index 00000000000..3f8ae0c6889
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedview/FILES
@@ -0,0 +1 @@
+feedview_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
new file mode 100644
index 00000000000..94167b8216a
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -0,0 +1,1211 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("feedview_test");
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/documentapi/messagebus/messages/documentreply.h>
+#include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
+#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_writer.h>
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/common/commit_time_tracker.h>
+#include <vespa/searchcore/proton/index/i_index_writer.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+#include <vespa/searchcore/proton/server/ifrozenbuckethandler.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+#include <vespa/searchcore/proton/server/searchable_feed_view.h>
+#include <vespa/searchcore/proton/server/isummaryadapter.h>
+#include <vespa/searchcore/proton/server/matchview.h>
+#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/test/document_meta_store_context_observer.h>
+#include <vespa/searchcore/proton/test/dummy_document_store.h>
+#include <vespa/searchcore/proton/test/dummy_summary_manager.h>
+#include <vespa/searchcore/proton/test/mock_index_writer.h>
+#include <vespa/searchcore/proton/test/mock_index_manager.h>
+#include <vespa/searchcore/proton/test/thread_utils.h>
+#include <vespa/searchcore/proton/test/threading_service_observer.h>
+#include <vespa/searchlib/docstore/cachestats.h>
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
+#include <mutex>
+
+using documentapi::RemoveDocumentReply;
+using namespace proton;
+using document::BucketId;
+using document::DataType;
+using document::Document;
+using document::DocumentId;
+using document::DocumentUpdate;
+using documentapi::DocumentProtocol;
+using proton::matching::SessionManager;
+using search::index::DocBuilder;
+using search::index::Schema;
+using search::AttributeVector;
+using search::CacheStats;
+using search::DocumentMetaData;
+using search::SearchableStats;
+using searchcorespi::IndexSearchable;
+using storage::spi::BucketChecksum;
+using storage::spi::BucketInfo;
+using storage::spi::PartitionId;
+using storage::spi::Timestamp;
+using storage::spi::UpdateResult;
+using vespalib::tensor::TensorType;
+using fastos::TimeStamp;
+
+typedef SearchableFeedView::SerialNum SerialNum;
+typedef search::DocumentIdT DocumentIdT;
+typedef DocumentProtocol::MessageType MessageType;
+
+struct MyLidVector : public std::vector<DocumentIdT>
+{
+    MyLidVector &add(DocumentIdT lid) { push_back(lid); return *this; }
+};
+
+
+const uint32_t subdb_id = 0;
+const vespalib::string indexAdapterTypeName = "index";
+const vespalib::string attributeAdapterTypeName = "attribute";
+
+struct MyTracer
+{
+    vespalib::asciistream _os;
+    using Mutex = std::mutex;
+    using Guard = std::lock_guard<Mutex>;
+    Mutex _mutex;
+
+    MyTracer()
+        : _os(),
+          _mutex()
+    {
+    }
+
+    void addComma() { if (!_os.empty()) { _os << ","; } }
+
+    void traceAck(const ResultUP &result) {
+        Guard guard(_mutex);
+        addComma();
+        _os << "ack(";
+        if (result) {
+            _os << result->toString();
+        } else {
+            _os << "null";
+        }
+        _os << ")";
+    }
+
+    void tracePut(const vespalib::string &adapterType,
+                  SerialNum serialNum, uint32_t lid, bool immediateCommit) {
+        Guard guard(_mutex);
+        addComma();
+        _os << "put(adapter=" << adapterType <<
+            ",serialNum=" << serialNum << ",lid=" << lid << ",commit=" << immediateCommit << ")";
+    }
+
+    void traceRemove(const vespalib::string &adapterType,
+                     SerialNum serialNum, uint32_t lid, bool immediateCommit) {
+        Guard guard(_mutex);
+        addComma();
+        _os << "remove(adapter=" << adapterType <<
+            ",serialNum=" << serialNum << ",lid=" << lid << ",commit=" << immediateCommit << ")";
+    }
+
+    void traceCommit(const vespalib::string &adapterType, SerialNum serialNum) {
+        Guard guard(_mutex);
+        addComma();
+        _os << "commit(adapter=" << adapterType <<
+            ",serialNum=" << serialNum << ")";
+    }
+};
+
+struct ParamsContext
+{
+    DocTypeName                          _docTypeName;
+    FeedMetrics                          _feedMetrics;
+    PerDocTypeFeedMetrics                _metrics;
+    SearchableFeedView::PersistentParams _params;
+
+    ParamsContext(const vespalib::string &docType,
+                  const vespalib::string &baseDir)
+        : _docTypeName(docType),
+          _feedMetrics(),
+          _metrics(&_feedMetrics),
+          _params(0,
+                  0,
+                  _docTypeName,
+                  _metrics,
+                  subdb_id,
+                  SubDbType::READY)
+    {
+        (void) baseDir;
+    }
+    const SearchableFeedView::PersistentParams &getParams() const { return _params; }
+};
+
+struct MyIndexWriter : public test::MockIndexWriter
+{
+    MyLidVector _removes;
+    int _heartBeatCount;
+    uint32_t _commitCount;
+    MyTracer &_tracer;
+    MyIndexWriter(MyTracer &tracer)
+        : test::MockIndexWriter(IIndexManager::SP(new test::MockIndexManager())),
+          _removes(),
+          _heartBeatCount(0),
+          _commitCount(0),
+          _tracer(tracer)
+    {}
+    virtual void put(SerialNum serialNum, const document::Document &doc,
+                     const DocumentIdT lid) override {
+        (void) doc;
+        _tracer.tracePut(indexAdapterTypeName, serialNum, lid, false);
+    }
+    virtual void remove(SerialNum serialNum, const search::DocumentIdT lid) override {
+        LOG(info, "MyIndexAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)",
+            serialNum, lid);
+        _removes.push_back(lid);
+        _tracer.traceRemove(indexAdapterTypeName, serialNum, lid, false);
+    }
+    virtual void commit(SerialNum serialNum, OnWriteDoneType) override {
+        ++_commitCount;
+        _tracer.traceCommit(indexAdapterTypeName, serialNum);
+    }
+    virtual void heartBeat(SerialNum) override { ++_heartBeatCount; }
+};
+
+struct MyDocumentStore : public test::DummyDocumentStore
+{
+    typedef std::map<DocumentIdT, document::Document::SP> DocMap;
+    DocMap           _docs;
+    uint64_t         _lastSyncToken;
+    MyDocumentStore()
+        : test::DummyDocumentStore("."),
+          _docs(),
+          _lastSyncToken(0)
+    {}
+    virtual Document::UP read(DocumentIdT lid, const document::DocumentTypeRepo &) const {
+        DocMap::const_iterator itr = _docs.find(lid);
+        if (itr != _docs.end()) {
+            Document::UP retval(itr->second->clone());
+            return retval;
+        }
+        return Document::UP();
+    }
+    virtual void write(uint64_t syncToken, const document::Document& doc, DocumentIdT lid) {
+        _lastSyncToken = syncToken;
+        _docs[lid] = Document::SP(doc.clone());
+    }
+    virtual void remove(uint64_t syncToken, DocumentIdT lid) {
+        _lastSyncToken = syncToken;
+        _docs.erase(lid);
+    }
+    virtual uint64_t initFlush(uint64_t syncToken) {
+        return syncToken;
+    }
+    virtual uint64_t lastSyncToken() const { return _lastSyncToken; }
+};
+
+struct MySummaryManager : public test::DummySummaryManager
+{
+    MyDocumentStore _store;
+    MySummaryManager() : _store() {}
+    virtual search::IDocumentStore &getBackingStore() { return _store; }
+};
+
+struct MySummaryAdapter : public ISummaryAdapter
+{
+    ISummaryManager::SP _sumMgr;
+    MyDocumentStore    &_store;
+    MyLidVector           _removes;
+
+    MySummaryAdapter()
+        : _sumMgr(new MySummaryManager()),
+          _store(static_cast<MyDocumentStore &>(_sumMgr->getBackingStore())),
+          _removes()
+    {
+    }
+    virtual void put(SerialNum serialNum, const document::Document &doc,
+                     const DocumentIdT lid) {
+        (void) serialNum;
+        _store.write(serialNum, doc, lid);
+    }
+    virtual void remove(SerialNum serialNum, const DocumentIdT lid) {
+        LOG(info,
+            "MySummaryAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)",
+            serialNum, lid);
+        _store.remove(serialNum, lid);
+        _removes.push_back(lid);
+    }
+    virtual void update(SerialNum serialNum, const document::DocumentUpdate &upd,
+           const DocumentIdT lid, const document::DocumentTypeRepo &repo) {
+        (void) serialNum; (void) upd; (void) lid; (void) repo;
+    }
+    virtual void heartBeat(SerialNum) {}
+    virtual const search::IDocumentStore &getDocumentStore() const {
+        return _store;
+    }
+    virtual std::unique_ptr<document::Document> get(const search::DocumentIdT lid,
+                                                    const document::DocumentTypeRepo &repo) {
+        return _store.read(lid, repo);
+    }
+};
+
+struct MyAttributeWriter : public IAttributeWriter
+{
+    MyLidVector _removes;
+    SerialNum _putSerial;
+    DocumentId _putDocId;
+    DocumentIdT _putLid;
+    SerialNum _updateSerial;
+    DocumentId _updateDocId;
+    DocumentIdT _updateLid;
+    SerialNum _removeSerial;
+    DocumentIdT _removeLid;
+    int _heartBeatCount;
+    uint32_t _commitCount;
+    uint32_t _wantedLidLimit;
+    using AttrMap = std::map<vespalib::string,
+                             std::shared_ptr<AttributeVector>>;
+    AttrMap _attrMap;
+    std::set<vespalib::string> _attrs;
+    proton::IAttributeManager::SP _mgr;
+    MyTracer &_tracer;
+    MyAttributeWriter(MyTracer &tracer)
+        : _removes(),
+          _putSerial(0),
+          _putDocId(),
+          _putLid(0),
+          _updateSerial(0),
+          _updateDocId(),
+          _updateLid(0),
+          _removeSerial(0),
+          _removeLid(0),
+          _heartBeatCount(0),
+          _commitCount(0),
+          _wantedLidLimit(0),
+          _attrMap(),
+          _attrs(),
+          _mgr(),
+          _tracer(tracer)
+    {
+        search::attribute::Config cfg(search::attribute::BasicType::INT32);
+        _attrMap["a1"] = search::AttributeFactory::createAttribute("test", cfg);
+        search::attribute::Config
+            cfg2(search::attribute::BasicType::PREDICATE);
+        _attrMap["a2"] = search::AttributeFactory::createAttribute("test2",
+                                                                   cfg2);
+        search::attribute::Config cfg3(search::attribute::BasicType::TENSOR);
+        cfg3.setTensorType(TensorType::fromSpec("tensor(x[10])"));
+        _attrMap["a3"] = search::AttributeFactory::createAttribute("test3",
+                                                                   cfg3);
+    }
+    virtual std::vector<AttributeVector *>
+    getWritableAttributes() const override {
+        return std::vector<AttributeVector *>();
+    }
+    virtual AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override {
+        if (_attrs.count(attrName) == 0) {
+            return nullptr;
+        }
+        AttrMap::const_iterator itr = _attrMap.find(attrName);
+        return ((itr == _attrMap.end()) ? nullptr : itr->second.get());
+    }
+    virtual void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid,
+                     bool immediateCommit, OnWriteDoneType) override {
+        _putSerial = serialNum;
+        _putDocId = doc.getId();
+        _putLid = lid;
+        _tracer.tracePut(attributeAdapterTypeName, serialNum, lid, immediateCommit);
+        if (immediateCommit) {
+            ++_commitCount;
+        }
+    }
+    virtual void remove(SerialNum serialNum, DocumentIdT lid,
+                        bool immediateCommit, OnWriteDoneType) override {
+        _removeSerial = serialNum;
+        _removeLid = lid;
+        _tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit);
+        if (immediateCommit) {
+            ++_commitCount;
+        }
+    }
+    virtual void remove(const LidVector & lidsToRemove, SerialNum serialNum,
+                        bool immediateCommit, OnWriteDoneType) override {
+        for (uint32_t lid : lidsToRemove) {
+            LOG(info, "MyAttributeAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)", serialNum, lid);
+           _removes.push_back(lid);
+           _tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit);
+        }
+    }
+    virtual void update(SerialNum serialNum, const document::DocumentUpdate &upd,
+                        DocumentIdT lid, bool, OnWriteDoneType) override {
+        _updateSerial = serialNum;
+        _updateDocId = upd.getId();
+        _updateLid = lid;
+    }
+    virtual void heartBeat(SerialNum) override { ++_heartBeatCount; }
+    virtual void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override {
+        (void) serialNum;
+        _wantedLidLimit = wantedLidLimit;
+    }
+    virtual const proton::IAttributeManager::SP &getAttributeManager() const override {
+        return _mgr;
+    }
+    void commit(SerialNum serialNum, OnWriteDoneType) override {
+        (void) serialNum; ++_commitCount;
+        _tracer.traceCommit(attributeAdapterTypeName, serialNum);
+    }
+
+    virtual void onReplayDone(uint32_t docIdLimit) override
+    {
+        (void) docIdLimit;
+    }
+};
+
+struct MyTransport : public FeedToken::ITransport
+{
+    ResultUP lastResult;
+    vespalib::Gate _gate;
+    MyTracer &_tracer;
+    MyTransport(MyTracer &tracer) : lastResult(), _gate(), _tracer(tracer) {}
+    virtual void send(mbus::Reply::UP reply,
+                      ResultUP result,
+                      bool documentWasFound,
+                      double latency_ms) {
+        (void) reply; (void) documentWasFound, (void) latency_ms;
+        lastResult = std::move(result);
+        _tracer.traceAck(lastResult);
+        _gate.countDown();
+    }
+    void await() { _gate.await(); }
+};
+
+
+struct MyResultHandler : public IGenericResultHandler
+{
+    vespalib::Gate _gate;
+    MyResultHandler() : _gate() {}
+    virtual void handle(const storage::spi::Result &) {
+        _gate.countDown();
+    }
+    void await() { _gate.await(); }
+};
+
+struct SchemaContext
+{
+    Schema::SP                _schema;
+    std::unique_ptr<DocBuilder> _builder;
+    SchemaContext() :
+        _schema(new Schema()),
+        _builder()
+    {
+        _schema->addIndexField(Schema::IndexField("i1", Schema::STRING, Schema::SINGLE));
+        _schema->addAttributeField(Schema::AttributeField("a1", Schema::STRING, Schema::SINGLE));
+        _schema->addAttributeField(Schema::AttributeField("a2", Schema::BOOLEANTREE, Schema::SINGLE));
+        _schema->addAttributeField(Schema::AttributeField("a3", Schema::TENSOR, Schema::SINGLE));
+        _schema->addSummaryField(Schema::SummaryField("s1", Schema::STRING, Schema::SINGLE));
+        _builder.reset(new DocBuilder(*_schema));
+    }
+    const document::DocumentTypeRepo::SP &getRepo() const { return _builder->getDocumentTypeRepo(); }
+};
+
+struct DocumentContext
+{
+    Document::SP       doc;
+    DocumentUpdate::SP upd;
+    BucketId           bid;
+    Timestamp          ts;
+    typedef std::vector<DocumentContext> List;
+    DocumentContext(const vespalib::string &docId, uint64_t timestamp, DocBuilder &builder) :
+        doc(builder.startDocument(docId)
+                   .startSummaryField("s1").addStr(docId).endField()
+                   .endDocument().release()),
+        upd(new DocumentUpdate(builder.getDocumentType(), doc->getId())),
+        bid(BucketFactory::getNumBucketBits(),
+            doc->getId().getGlobalId().convertToBucketId().getRawId()),
+        ts(timestamp)
+    {
+    }
+    void addFieldUpdate(DocBuilder &builder,
+                        const vespalib::string &fieldName) {
+        const document::Field &field =
+            builder.getDocumentType().getField(fieldName);
+        upd->addUpdate(document::FieldUpdate(field));
+    }
+};
+
+namespace {
+
+mbus::Reply::UP
+createReply(MessageType mtype)
+{
+    if (mtype == DocumentProtocol::REPLY_UPDATEDOCUMENT) {
+        return mbus::Reply::UP(new documentapi::UpdateDocumentReply);
+    } else if (mtype == DocumentProtocol::REPLY_REMOVEDOCUMENT) {
+        return mbus::Reply::UP(new documentapi::RemoveDocumentReply);
+    } else {
+        return mbus::Reply::UP(new documentapi::DocumentReply(mtype));
+    }
+}
+
+}  // namespace
+
+struct FeedTokenContext
+{
+    MyTransport mt;
+    FeedToken   ft;
+    typedef std::shared_ptr<FeedTokenContext> SP;
+    typedef std::vector<SP> List;
+    FeedTokenContext(MyTracer &tracer, MessageType mtype) :
+        mt(tracer),
+        ft(mt, createReply(mtype))
+    {
+    }
+};
+
+struct FixtureBase
+{
+    MyTracer             _tracer;
+    IIndexWriter::SP     iw;
+    ISummaryAdapter::SP  sa;
+    IAttributeWriter::SP aw;
+    MyIndexWriter        &miw;
+    MySummaryAdapter     &msa;
+    MyAttributeWriter    &maw;
+    SchemaContext        sc;
+    DocIdLimit           _docIdLimit;
+    DocumentMetaStoreContext::SP _dmscReal;
+    test::DocumentMetaStoreContextObserver::SP _dmsc;
+    ParamsContext         pc;
+    ExecutorThreadingService _writeServiceReal;
+    test::ThreadingServiceObserver _writeService;
+    documentmetastore::LidReuseDelayer _lidReuseDelayer;
+    CommitTimeTracker     _commitTimeTracker;
+    SerialNum             serial;
+    FixtureBase(TimeStamp visibilityDelay) :
+        _tracer(),
+        iw(new MyIndexWriter(_tracer)),
+        sa(new MySummaryAdapter),
+        aw(new MyAttributeWriter(_tracer)),
+        miw(static_cast<MyIndexWriter&>(*iw)),
+        msa(static_cast<MySummaryAdapter&>(*sa)),
+        maw(static_cast<MyAttributeWriter&>(*aw)),
+        sc(),
+        _docIdLimit(0u),
+        _dmscReal(new DocumentMetaStoreContext(std::make_shared<BucketDBOwner>())),
+        _dmsc(new test::DocumentMetaStoreContextObserver(*_dmscReal)),
+        pc(sc._builder->getDocumentType().getName(), "fileconfig_test"),
+        _writeServiceReal(),
+        _writeService(_writeServiceReal),
+        _lidReuseDelayer(_writeService, _dmsc->get()),
+        _commitTimeTracker(visibilityDelay),
+        serial(0)
+    {
+        _dmsc->constructFreeList();
+        _lidReuseDelayer.setImmediateCommit(visibilityDelay == 0);
+    }
+
+    virtual ~FixtureBase() {
+        _writeServiceReal.sync();
+    }
+
+    void syncMaster() {
+        _writeService.master().sync();
+    }
+
+    void syncIndex() {
+        _writeService.sync();
+    }
+
+    void sync() {
+        _writeServiceReal.sync();
+    }
+
+    const test::DocumentMetaStoreObserver &metaStoreObserver() {
+        return _dmsc->getObserver();
+    }
+
+    const test::ThreadingServiceObserver &writeServiceObserver() {
+        return _writeService;
+    }
+
+    template <typename FunctionType>
+    void runInMaster(FunctionType func) {
+        test::runInMaster(_writeService, func);
+    }
+
+    virtual IFeedView &getFeedView() = 0;
+
+    const IDocumentMetaStore &getMetaStore() const {
+        return _dmsc->get();
+    }
+
+    BucketDBOwner::Guard getBucketDB() const {
+        return getMetaStore().getBucketDB().takeGuard();
+    }
+
+    DocumentMetaData getMetaData(const DocumentContext &doc_) const {
+        return getMetaStore().getMetaData(doc_.doc->getId().getGlobalId());
+    }
+
+    DocBuilder &getBuilder() { return *sc._builder; }
+
+    DocumentContext doc(const vespalib::string &docId, uint64_t timestamp) {
+        return DocumentContext(docId, timestamp, getBuilder());
+    }
+
+    DocumentContext doc1(uint64_t timestamp = 10) {
+        return doc("doc:test:1", timestamp);
+    }
+
+    void performPut(FeedToken *token, PutOperation &op) {
+        getFeedView().preparePut(op);
+        op.setSerialNum(++serial);
+        getFeedView().handlePut(token, op);
+    }
+
+    void putAndWait(const DocumentContext::List &docs) {
+        for (size_t i = 0; i < docs.size(); ++i) {
+            putAndWait(docs[i]);
+        }
+    }
+
+    void putAndWait(const DocumentContext &docCtx) {
+        FeedTokenContext token(_tracer, DocumentProtocol::REPLY_PUTDOCUMENT);
+        PutOperation op(docCtx.bid, docCtx.ts, docCtx.doc);
+        runInMaster([&] () { performPut(&token.ft, op); });
+    }
+
+    void performUpdate(FeedToken *token, UpdateOperation &op) {
+        getFeedView().prepareUpdate(op);
+        op.setSerialNum(++serial);
+        getFeedView().handleUpdate(token, op);
+    }
+
+    void updateAndWait(const DocumentContext &docCtx) {
+        FeedTokenContext token(_tracer, DocumentProtocol::REPLY_UPDATEDOCUMENT);
+        UpdateOperation op(docCtx.bid, docCtx.ts, docCtx.upd);
+        runInMaster([&] () { performUpdate(&token.ft, op); });
+    }
+
+    void performRemove(FeedToken *token, RemoveOperation &op) {
+        getFeedView().prepareRemove(op);
+        if (op.getValidNewOrPrevDbdId()) {
+            op.setSerialNum(++serial);
+            getFeedView().handleRemove(token, op);
+        } else {
+            if (token != NULL) {
+                token->ack(op.getType(), pc._metrics);
+            }
+        }
+    }
+
+    void removeAndWait(const DocumentContext &docCtx) {
+        FeedTokenContext token(_tracer, DocumentProtocol::REPLY_REMOVEDOCUMENT);
+        RemoveOperation op(docCtx.bid, docCtx.ts, docCtx.doc->getId());
+        runInMaster([&] () { performRemove(&token.ft, op); });
+    }
+
+    void removeAndWait(const DocumentContext::List &docs) {
+        for (size_t i = 0; i < docs.size(); ++i) {
+            removeAndWait(docs[i]);
+        }
+    }
+    void performDeleteBucket(DeleteBucketOperation &op) {
+        getFeedView().prepareDeleteBucket(op);
+        op.setSerialNum(++serial);
+        getFeedView().handleDeleteBucket(op);
+    }
+
+    void performForceCommit() { getFeedView().forceCommit(serial); }
+    void forceCommitAndWait() { runInMaster([&]() { performForceCommit(); }); }
+
+    bool assertTrace(const vespalib::string &exp) {
+        return EXPECT_EQUAL(exp, _tracer._os.str());
+    }
+
+    DocumentContext::List
+    makeDummyDocs(uint32_t first, uint32_t count, uint64_t tsfirst) {
+        DocumentContext::List docs;
+        for (uint32_t i = 0; i < count; ++i) {
+            uint32_t id = first + i;
+            uint64_t ts = tsfirst + i;
+            vespalib::asciistream os;
+            os << "doc:test:" << id;
+            docs.push_back(doc(os.str(), ts));
+        }
+        return docs;
+    }
+
+    void performCompactLidSpace(uint32_t wantedLidLimit) {
+        auto &fv = getFeedView();
+        CompactLidSpaceOperation op(0, wantedLidLimit);
+        op.setSerialNum(++serial);
+        fv.handleCompactLidSpace(op);
+    }
+    void compactLidSpaceAndWait(uint32_t wantedLidLimit) {
+        runInMaster([&] () { performCompactLidSpace(wantedLidLimit); });
+    }
+};
+
+struct SearchableFeedViewFixture : public FixtureBase
+{
+    SearchableFeedView fv;
+    SearchableFeedViewFixture(TimeStamp visibilityDelay = 0) :
+        FixtureBase(visibilityDelay),
+        fv(StoreOnlyFeedView::Context(sa,
+                sc._schema,
+                _dmsc,
+                sc.getRepo(),
+                _writeService,
+                _lidReuseDelayer,
+                _commitTimeTracker),
+           pc.getParams(),
+           FastAccessFeedView::Context(aw, _docIdLimit),
+           SearchableFeedView::Context(iw))
+    {
+        runInMaster([&]() { _lidReuseDelayer.setHasIndexedFields(true); });
+    }
+    virtual IFeedView &getFeedView() { return fv; }
+};
+
+struct FastAccessFeedViewFixture : public FixtureBase
+{
+    FastAccessFeedView fv;
+    FastAccessFeedViewFixture(TimeStamp visibilityDelay = 0) :
+        FixtureBase(visibilityDelay),
+        fv(StoreOnlyFeedView::Context(sa,
+                sc._schema,
+                _dmsc,
+                sc.getRepo(),
+                _writeService,
+                _lidReuseDelayer,
+                _commitTimeTracker),
+           pc.getParams(),
+           FastAccessFeedView::Context(aw, _docIdLimit))
+    {
+    }
+    virtual IFeedView &getFeedView() { return fv; }
+};
+
+void
+assertBucketInfo(const BucketId &ebid,
+                 const Timestamp &ets,
+                 uint32_t lid,
+                 const IDocumentMetaStore &metaStore)
+{
+    document::GlobalId gid;
+    EXPECT_TRUE(metaStore.getGid(lid, gid));
+    search::DocumentMetaData meta = metaStore.getMetaData(gid);
+    EXPECT_TRUE(meta.valid());
+    BucketId abid;
+    EXPECT_EQUAL(ebid, meta.bucketId);
+    Timestamp ats;
+    EXPECT_EQUAL(ets, meta.timestamp);
+}
+
+void
+assertLidVector(const MyLidVector &exp, const MyLidVector &act)
+{
+    EXPECT_EQUAL(exp.size(), act.size());
+    for (size_t i = 0; i < exp.size(); ++i) {
+        EXPECT_TRUE(std::find(act.begin(), act.end(), exp[i]) != act.end());
+    }
+}
+
+void
+assertAttributeUpdate(SerialNum serialNum,
+                      const document::DocumentId &docId,
+                      DocumentIdT lid,
+                      MyAttributeWriter adapter)
+{
+    EXPECT_EQUAL(serialNum, adapter._updateSerial);
+    EXPECT_EQUAL(docId, adapter._updateDocId);
+    EXPECT_EQUAL(lid, adapter._updateLid);
+}
+
+
+TEST_F("require that put() updates document meta store with bucket info",
+       SearchableFeedViewFixture)
+{
+    DocumentContext dc = f.doc1();
+    f.putAndWait(dc);
+
+    assertBucketInfo(dc.bid, dc.ts, 1, f.getMetaStore());
+    // TODO: rewrite to use getBucketInfo() when available
+    BucketInfo bucketInfo = f.getBucketDB()->get(dc.bid);
+    EXPECT_EQUAL(1u, bucketInfo.getDocumentCount());
+    EXPECT_NOT_EQUAL(bucketInfo.getChecksum(), BucketChecksum(0));
+}
+
+TEST_F("require that put() calls attribute adapter", SearchableFeedViewFixture)
+{
+    DocumentContext dc = f.doc1();
+    EXPECT_EQUAL(0u, f._docIdLimit.get());
+    f.putAndWait(dc);
+
+    EXPECT_EQUAL(1u, f.maw._putSerial);
+    EXPECT_EQUAL(DocumentId("doc:test:1"), f.maw._putDocId);
+    EXPECT_EQUAL(1u, f.maw._putLid);
+    EXPECT_EQUAL(2u, f._docIdLimit.get());
+}
+
+TEST_F("require that update() updates document meta store with bucket info",
+       SearchableFeedViewFixture)
+{
+    DocumentContext dc1 = f.doc1(10);
+    DocumentContext dc2 = f.doc1(20);
+    f.putAndWait(dc1);
+    BucketChecksum bcs = f.getBucketDB()->get(dc1.bid).getChecksum();
+    f.updateAndWait(dc2);
+
+    assertBucketInfo(dc1.bid, Timestamp(20), 1, f.getMetaStore());
+    // TODO: rewrite to use getBucketInfo() when available
+    BucketInfo bucketInfo = f.getBucketDB()->get(dc1.bid);
+    EXPECT_EQUAL(1u, bucketInfo.getDocumentCount());
+    EXPECT_NOT_EQUAL(bucketInfo.getChecksum(), bcs);
+    EXPECT_NOT_EQUAL(bucketInfo.getChecksum(), BucketChecksum(0));
+}
+
+TEST_F("require that update() calls attribute adapter", SearchableFeedViewFixture)
+{
+    DocumentContext dc1 = f.doc1(10);
+    DocumentContext dc2 = f.doc1(20);
+    f.putAndWait(dc1);
+    f.updateAndWait(dc2);
+
+    assertAttributeUpdate(2u, DocumentId("doc:test:1"), 1u, f.maw);
+}
+
+TEST_F("require that remove() updates document meta store with bucket info",
+       SearchableFeedViewFixture)
+{
+    DocumentContext dc1 = f.doc("userdoc:test:1:1", 10);
+    DocumentContext dc2 = f.doc("userdoc:test:1:2", 11);
+    f.putAndWait(dc1);
+    BucketChecksum bcs1 = f.getBucketDB()->get(dc1.bid).getChecksum();
+    f.putAndWait(dc2);
+    BucketChecksum bcs2 = f.getBucketDB()->get(dc2.bid).getChecksum();
+    f.removeAndWait(DocumentContext("userdoc:test:1:2", 20, f.getBuilder()));
+
+    assertBucketInfo(dc1.bid, Timestamp(10), 1, f.getMetaStore());
+    EXPECT_FALSE(f.getMetaStore().validLid(2)); // don't remember remove
+    // TODO: rewrite to use getBucketInfo() when available
+    BucketInfo bucketInfo = f.getBucketDB()->get(dc1.bid);
+    EXPECT_EQUAL(1u, bucketInfo.getDocumentCount());
+    EXPECT_NOT_EQUAL(bucketInfo.getChecksum(), bcs2);
+    EXPECT_EQUAL(bucketInfo.getChecksum(), bcs1);
+}
+
+TEST_F("require that remove() calls attribute adapter", SearchableFeedViewFixture)
+{
+    DocumentContext dc1 = f.doc1(10);
+    DocumentContext dc2 = f.doc1(20);
+    f.putAndWait(dc1);
+    f.removeAndWait(dc2);
+
+    EXPECT_EQUAL(2u, f.maw._removeSerial);
+    EXPECT_EQUAL(1u, f.maw._removeLid);
+}
+
+bool
+assertThreadObserver(uint32_t masterExecuteCnt,
+                     uint32_t indexExecuteCnt,
+                     const test::ThreadingServiceObserver &observer)
+{
+    if (!EXPECT_EQUAL(masterExecuteCnt, observer.masterObserver().getExecuteCnt())) return false;
+    if (!EXPECT_EQUAL(indexExecuteCnt, observer.indexObserver().getExecuteCnt())) return false;
+    return true;
+}
+
+TEST_F("require that remove() calls removeComplete() via delayed thread service",
+        SearchableFeedViewFixture)
+{
+    EXPECT_TRUE(assertThreadObserver(1, 0, f.writeServiceObserver()));
+    f.putAndWait(f.doc1(10));
+    // put index fields handled in index thread
+    EXPECT_TRUE(assertThreadObserver(2, 1, f.writeServiceObserver()));
+    f.removeAndWait(f.doc1(20));
+    // remove index fields handled in index thread
+    // delayed remove complete handled in same index thread, then master thread
+    EXPECT_TRUE(assertThreadObserver(4, 2, f.writeServiceObserver()));
+    EXPECT_EQUAL(1u, f.metaStoreObserver()._removeCompleteCnt);
+    EXPECT_EQUAL(1u, f.metaStoreObserver()._removeCompleteLid);
+}
+
+TEST_F("require that handleDeleteBucket() removes documents", SearchableFeedViewFixture)
+{
+    DocumentContext::List docs;
+    docs.push_back(f.doc("userdoc:test:1:1", 10));
+    docs.push_back(f.doc("userdoc:test:1:2", 11));
+    docs.push_back(f.doc("userdoc:test:1:3", 12));
+    docs.push_back(f.doc("userdoc:test:2:1", 13));
+    docs.push_back(f.doc("userdoc:test:2:2", 14));
+
+    f.putAndWait(docs);
+
+    DocumentIdT lid;
+    EXPECT_TRUE(f.getMetaStore().getLid(docs[0].doc->getId().getGlobalId(), lid));
+    EXPECT_EQUAL(1u, lid);
+    EXPECT_TRUE(f.getMetaStore().getLid(docs[1].doc->getId().getGlobalId(), lid));
+    EXPECT_EQUAL(2u, lid);
+    EXPECT_TRUE(f.getMetaStore().getLid(docs[2].doc->getId().getGlobalId(), lid));
+    EXPECT_EQUAL(3u, lid);
+
+    // delete bucket for user 1
+    DeleteBucketOperation op(docs[0].bid);
+    f.runInMaster([&] () { f.performDeleteBucket(op); });
+
+    EXPECT_EQUAL(0u, f.getBucketDB()->get(docs[0].bid).getDocumentCount());
+    EXPECT_EQUAL(2u, f.getBucketDB()->get(docs[3].bid).getDocumentCount());
+    EXPECT_FALSE(f.getMetaStore().getLid(docs[0].doc->getId().getGlobalId(), lid));
+    EXPECT_FALSE(f.getMetaStore().getLid(docs[1].doc->getId().getGlobalId(), lid));
+    EXPECT_FALSE(f.getMetaStore().getLid(docs[2].doc->getId().getGlobalId(), lid));
+    MyLidVector exp = MyLidVector().add(1).add(2).add(3);
+    assertLidVector(exp, f.miw._removes);
+    assertLidVector(exp, f.msa._removes);
+    assertLidVector(exp, f.maw._removes);
+}
+
+void
+assertPostConditionAfterRemoves(const DocumentContext::List &docs,
+                                SearchableFeedViewFixture &f)
+{
+    EXPECT_EQUAL(3u, f.getMetaStore().getNumUsedLids());
+    EXPECT_FALSE(f.getMetaData(docs[0]).valid());
+    EXPECT_TRUE(f.getMetaData(docs[1]).valid());
+    EXPECT_FALSE(f.getMetaData(docs[1]).removed);
+    EXPECT_TRUE(f.getMetaData(docs[2]).valid());
+    EXPECT_FALSE(f.getMetaData(docs[2]).removed);
+    EXPECT_FALSE(f.getMetaData(docs[3]).valid());
+    EXPECT_TRUE(f.getMetaData(docs[4]).valid());
+    EXPECT_FALSE(f.getMetaData(docs[4]).removed);
+
+    assertLidVector(MyLidVector().add(1).add(4), f.miw._removes);
+    assertLidVector(MyLidVector().add(1).add(4), f.msa._removes);
+    MyDocumentStore::DocMap &sdocs = f.msa._store._docs;
+    EXPECT_EQUAL(3u, sdocs.size());
+    EXPECT_TRUE(sdocs.find(1) == sdocs.end());
+    EXPECT_TRUE(sdocs.find(4) == sdocs.end());
+}
+
+TEST_F("require that removes are not remembered", SearchableFeedViewFixture)
+{
+    DocumentContext::List docs;
+    docs.push_back(f.doc("userdoc:test:1:1", 10));
+    docs.push_back(f.doc("userdoc:test:1:2", 11));
+    docs.push_back(f.doc("userdoc:test:1:3", 12));
+    docs.push_back(f.doc("userdoc:test:2:1", 13));
+    docs.push_back(f.doc("userdoc:test:2:2", 14));
+
+    f.putAndWait(docs);
+    f.removeAndWait(docs[0]);
+    f.removeAndWait(docs[3]);
+    assertPostConditionAfterRemoves(docs, f);
+
+    // try to remove again : should have little effect
+    f.removeAndWait(docs[0]);
+    f.removeAndWait(docs[3]);
+    assertPostConditionAfterRemoves(docs, f);
+
+    // re-add docs
+    f.putAndWait(docs[3]);
+    f.putAndWait(docs[0]);
+    EXPECT_EQUAL(5u, f.getMetaStore().getNumUsedLids());
+    EXPECT_TRUE(f.getMetaData(docs[0]).valid());
+    EXPECT_TRUE(f.getMetaData(docs[1]).valid());
+    EXPECT_TRUE(f.getMetaData(docs[2]).valid());
+    EXPECT_TRUE(f.getMetaData(docs[3]).valid());
+    EXPECT_TRUE(f.getMetaData(docs[4]).valid());
+    EXPECT_FALSE(f.getMetaData(docs[0]).removed);
+    EXPECT_FALSE(f.getMetaData(docs[1]).removed);
+    EXPECT_FALSE(f.getMetaData(docs[2]).removed);
+    EXPECT_FALSE(f.getMetaData(docs[3]).removed);
+    EXPECT_FALSE(f.getMetaData(docs[4]).removed);
+    EXPECT_EQUAL(5u, f.msa._store._docs.size());
+    const Document::SP &doc1 = f.msa._store._docs[1];
+    EXPECT_EQUAL(docs[3].doc->getId(), doc1->getId());
+    EXPECT_EQUAL(docs[3].doc->getId().toString(),
+                 doc1->getValue("s1")->toString());
+    const Document::SP &doc4 = f.msa._store._docs[4];
+    EXPECT_EQUAL(docs[0].doc->getId(), doc4->getId());
+    EXPECT_EQUAL(docs[0].doc->getId().toString(),
+                 doc4->getValue("s1")->toString());
+    EXPECT_EQUAL(5u, f.msa._store._docs.size());
+
+    f.removeAndWait(docs[0]);
+    f.removeAndWait(docs[3]);
+    EXPECT_EQUAL(3u, f.msa._store._docs.size());
+}
+
+TEST_F("require that heartbeat propagates to index- and attributeadapter",
+       SearchableFeedViewFixture)
+{
+    f.runInMaster([&] () { f.fv.heartBeat(2); });
+    EXPECT_EQUAL(1, f.miw._heartBeatCount);
+    EXPECT_EQUAL(1, f.maw._heartBeatCount);
+}
+
+template <typename Fixture>
+void putDocumentAndUpdate(Fixture &f, const vespalib::string &fieldName)
+{
+    DocumentContext dc1 = f.doc1();
+    f.putAndWait(dc1);
+    EXPECT_EQUAL(1u, f.msa._store._lastSyncToken);
+
+    DocumentContext dc2("doc:test:1", 20, f.getBuilder());
+    dc2.addFieldUpdate(f.getBuilder(), fieldName);
+    f.updateAndWait(dc2);
+}
+
+template <typename Fixture>
+void requireThatUpdateOnlyUpdatesAttributeAndNotDocumentStore(Fixture &f)
+{
+    putDocumentAndUpdate(f, "a1");
+
+    EXPECT_EQUAL(1u, f.msa._store._lastSyncToken); // document store not updated
+    assertAttributeUpdate(2u, DocumentId("doc:test:1"), 1, f.maw);
+}
+
+TEST_F("require that update() to fast-access attribute only updates attribute and not document store",
+       FastAccessFeedViewFixture)
+{
+    f.maw._attrs.insert("a1"); // mark a1 as fast-access attribute field
+    requireThatUpdateOnlyUpdatesAttributeAndNotDocumentStore(f);
+}
+
+TEST_F("require that update() to attribute only updates attribute and not document store",
+       SearchableFeedViewFixture)
+{
+    f.maw._attrs.insert("a1"); // mark a1 as attribute field
+    requireThatUpdateOnlyUpdatesAttributeAndNotDocumentStore(f);
+}
+
+TEST_F("require that update to non fast-access attribute also updates document store",
+        FastAccessFeedViewFixture)
+{
+    putDocumentAndUpdate(f, "a1");
+
+    EXPECT_EQUAL(2u, f.msa._store._lastSyncToken); // document store updated
+    assertAttributeUpdate(2u, DocumentId("doc:test:1"), 1, f.maw);
+}
+
+template <typename Fixture>
+void requireThatUpdateUpdatesAttributeAndDocumentStore(Fixture &f,
+                                                       const vespalib::string &
+                                                       fieldName)
+{
+    putDocumentAndUpdate(f, fieldName);
+
+    EXPECT_EQUAL(2u, f.msa._store._lastSyncToken); // document store updated
+    assertAttributeUpdate(2u, DocumentId("doc:test:1"), 1, f.maw);
+}
+
+TEST_F("require that update() to fast-access predicate attribute updates attribute and document store",
+       FastAccessFeedViewFixture)
+{
+    f.maw._attrs.insert("a2"); // mark a2 as fast-access attribute field
+    requireThatUpdateUpdatesAttributeAndDocumentStore(f, "a2");
+}
+
+TEST_F("require that update() to predicate attribute updates attribute and document store",
+       SearchableFeedViewFixture)
+{
+    f.maw._attrs.insert("a2"); // mark a2 as attribute field
+    requireThatUpdateUpdatesAttributeAndDocumentStore(f, "a2");
+}
+
+TEST_F("require that update() to fast-access tensor attribute updates attribute and document store",
+       FastAccessFeedViewFixture)
+{
+    f.maw._attrs.insert("a3"); // mark a3 as fast-access attribute field
+    requireThatUpdateUpdatesAttributeAndDocumentStore(f, "a3");
+}
+
+TEST_F("require that update() to tensor attribute updates attribute and document store",
+       SearchableFeedViewFixture)
+{
+    f.maw._attrs.insert("a3"); // mark a3 as attribute field
+    requireThatUpdateUpdatesAttributeAndDocumentStore(f, "a3");
+}
+
+TEST_F("require that compactLidSpace() propagates to document meta store and "
+       "blocks lid space shrinkage until generation is no longer used",
+       SearchableFeedViewFixture)
+{
+    EXPECT_TRUE(assertThreadObserver(1, 0, f.writeServiceObserver()));
+    CompactLidSpaceOperation op(0, 99);
+    op.setSerialNum(1);
+    f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); });
+    // performIndexForceCommit in index thread, then completion callback
+    // in master thread.
+    EXPECT_TRUE(assertThreadObserver(3, 1, f.writeServiceObserver()));
+    EXPECT_EQUAL(99u, f.metaStoreObserver()._compactLidSpaceLidLimit);
+    EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
+    EXPECT_EQUAL(99u, f._docIdLimit.get());
+}
+
+TEST_F("require that compactLidSpace() doesn't propagate to "
+       "document meta store and "
+       "blocks lid space shrinkage until generation is no longer used",
+       SearchableFeedViewFixture)
+{
+    EXPECT_TRUE(assertThreadObserver(1, 0, f.writeServiceObserver()));
+    CompactLidSpaceOperation op(0, 99);
+    op.setSerialNum(0);
+    f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); });
+    // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread
+    EXPECT_TRUE(assertThreadObserver(2, 0, f.writeServiceObserver()));
+    EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit);
+    EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
+}
+
+TEST_F("require that compactLidSpace() propagates to attributeadapter",
+       FastAccessFeedViewFixture)
+{
+    f.runInMaster([&] () { f.fv.handleCompactLidSpace(CompactLidSpaceOperation(0, 99)); });
+    EXPECT_EQUAL(99u, f.maw._wantedLidLimit);
+}
+
+
+TEST_F("require that commit is called if visibility delay is 0",
+       SearchableFeedViewFixture)
+{
+    DocumentContext dc = f.doc1();
+    f.putAndWait(dc);
+    EXPECT_EQUAL(1u, f.miw._commitCount);
+    EXPECT_EQUAL(1u, f.maw._commitCount);
+    f.removeAndWait(dc);
+    EXPECT_EQUAL(2u, f.miw._commitCount);
+    EXPECT_EQUAL(2u, f.maw._commitCount);
+    f.assertTrace("put(adapter=attribute,serialNum=1,lid=1,commit=1),"
+                  "put(adapter=index,serialNum=1,lid=1,commit=0),"
+                  "commit(adapter=index,serialNum=1),"
+                  "ack(Result(0, )),"
+                  "remove(adapter=attribute,serialNum=2,lid=1,commit=1),"
+                  "remove(adapter=index,serialNum=2,lid=1,commit=0),"
+                  "commit(adapter=index,serialNum=2),"
+                  "ack(Result(0, ))");
+}
+
+const TimeStamp LONG_DELAY(TimeStamp::Seconds(60.0));
+const TimeStamp SHORT_DELAY(TimeStamp::Seconds(0.5));
+
+TEST_F("require that commit is not called when inside a commit interval",
+       SearchableFeedViewFixture(LONG_DELAY))
+{
+    DocumentContext dc = f.doc1();
+    f.putAndWait(dc);
+    EXPECT_EQUAL(0u, f.miw._commitCount);
+    EXPECT_EQUAL(0u, f.maw._commitCount);
+    EXPECT_EQUAL(0u, f._docIdLimit.get());
+    f.removeAndWait(dc);
+    EXPECT_EQUAL(0u, f.miw._commitCount);
+    EXPECT_EQUAL(0u, f.maw._commitCount);
+    EXPECT_EQUAL(0u, f._docIdLimit.get());
+    f.assertTrace("ack(Result(0, )),"
+                  "put(adapter=attribute,serialNum=1,lid=1,commit=0),"
+                  "put(adapter=index,serialNum=1,lid=1,commit=0),"
+                  "ack(Result(0, )),"
+                  "remove(adapter=attribute,serialNum=2,lid=1,commit=0),"
+                  "remove(adapter=index,serialNum=2,lid=1,commit=0)");
+}
+
+TEST_F("require that commit is called when crossing a commit interval",
+       SearchableFeedViewFixture(SHORT_DELAY))
+{
+    FastOS_Thread::Sleep(SHORT_DELAY.ms() + 10);
+    DocumentContext dc = f.doc1();
+    f.putAndWait(dc);
+    EXPECT_EQUAL(1u, f.miw._commitCount);
+    EXPECT_EQUAL(1u, f.maw._commitCount);
+    EXPECT_EQUAL(2u, f._docIdLimit.get());
+    FastOS_Thread::Sleep(SHORT_DELAY.ms() + 10);
+    f.removeAndWait(dc);
+    EXPECT_EQUAL(2u, f.miw._commitCount);
+    EXPECT_EQUAL(2u, f.maw._commitCount);
+    f.assertTrace("ack(Result(0, )),"
+                  "put(adapter=attribute,serialNum=1,lid=1,commit=1),"
+                  "put(adapter=index,serialNum=1,lid=1,commit=0),"
+                  "commit(adapter=index,serialNum=1),"
+                  "ack(Result(0, )),"
+                  "remove(adapter=attribute,serialNum=2,lid=1,commit=1),"
+                  "remove(adapter=index,serialNum=2,lid=1,commit=0),"
+                  "commit(adapter=index,serialNum=2)");
+}
+
+
+TEST_F("require that commit is not implicitly called after "
+       "handover to maintenance job",
+       SearchableFeedViewFixture(SHORT_DELAY))
+{
+    f._commitTimeTracker.setReplayDone();
+    FastOS_Thread::Sleep(SHORT_DELAY.ms() + 10);
+    DocumentContext dc = f.doc1();
+    f.putAndWait(dc);
+    EXPECT_EQUAL(0u, f.miw._commitCount);
+    EXPECT_EQUAL(0u, f.maw._commitCount);
+    EXPECT_EQUAL(0u, f._docIdLimit.get());
+    FastOS_Thread::Sleep(SHORT_DELAY.ms() + 10);
+    f.removeAndWait(dc);
+    EXPECT_EQUAL(0u, f.miw._commitCount);
+    EXPECT_EQUAL(0u, f.maw._commitCount);
+    EXPECT_EQUAL(0u, f._docIdLimit.get());
+    f.assertTrace("ack(Result(0, )),"
+                  "put(adapter=attribute,serialNum=1,lid=1,commit=0),"
+                  "put(adapter=index,serialNum=1,lid=1,commit=0),"
+                  "ack(Result(0, )),"
+                  "remove(adapter=attribute,serialNum=2,lid=1,commit=0),"
+                  "remove(adapter=index,serialNum=2,lid=1,commit=0)");
+}
+
+TEST_F("require that forceCommit updates docid limit",
+       SearchableFeedViewFixture(LONG_DELAY))
+{
+    f._commitTimeTracker.setReplayDone();
+    DocumentContext dc = f.doc1();
+    f.putAndWait(dc);
+    EXPECT_EQUAL(0u, f.miw._commitCount);
+    EXPECT_EQUAL(0u, f.maw._commitCount);
+    EXPECT_EQUAL(0u, f._docIdLimit.get());
+    f.forceCommitAndWait();
+    EXPECT_EQUAL(1u, f.miw._commitCount);
+    EXPECT_EQUAL(1u, f.maw._commitCount);
+    EXPECT_EQUAL(2u, f._docIdLimit.get());
+    f.assertTrace("ack(Result(0, )),"
+                  "put(adapter=attribute,serialNum=1,lid=1,commit=0),"
+                  "put(adapter=index,serialNum=1,lid=1,commit=0),"
+                  "commit(adapter=attribute,serialNum=1),"
+                  "commit(adapter=index,serialNum=1)");
+}
+
+TEST_F("require that forceCommit updates docid limit during shrink",
+       SearchableFeedViewFixture(LONG_DELAY))
+{
+    f._commitTimeTracker.setReplayDone();
+    f.putAndWait(f.makeDummyDocs(0, 3, 1000));
+    EXPECT_EQUAL(0u, f._docIdLimit.get());
+    f.forceCommitAndWait();
+    EXPECT_EQUAL(4u, f._docIdLimit.get());
+    f.removeAndWait(f.makeDummyDocs(1, 2, 2000));
+    EXPECT_EQUAL(4u, f._docIdLimit.get());
+    f.forceCommitAndWait();
+    EXPECT_EQUAL(4u, f._docIdLimit.get());
+    f.compactLidSpaceAndWait(2);
+    EXPECT_EQUAL(2u, f._docIdLimit.get());
+    f.forceCommitAndWait();
+    EXPECT_EQUAL(2u, f._docIdLimit.get());
+    f.putAndWait(f.makeDummyDocs(1, 1, 3000));
+    EXPECT_EQUAL(2u, f._docIdLimit.get());
+    f.forceCommitAndWait();
+    EXPECT_EQUAL(3u, f._docIdLimit.get());
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
+
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/.gitignore b/searchcore/src/tests/proton/documentdb/fileconfigmanager/.gitignore
new file mode 100644
index 00000000000..7aac27360eb
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/.gitignore
@@ -0,0 +1,5 @@
+config-mycfg.cpp
+config-mycfg.h
+/out
+/out2
+searchcore_fileconfigmanager_test_app
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/fileconfigmanager/CMakeLists.txt
new file mode 100644
index 00000000000..458607e66c5
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_fileconfigmanager_test_app
+    SOURCES
+    fileconfigmanager_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_pcommon
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_fileconfigmanager_test_app COMMAND sh fileconfigmanager_test.sh)
+vespa_generate_config(searchcore_fileconfigmanager_test_app mycfg.def)
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/DESC b/searchcore/src/tests/proton/documentdb/fileconfigmanager/DESC
new file mode 100644
index 00000000000..ab2f0ed8b46
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/DESC
@@ -0,0 +1 @@
+fileconfigmanager test. Take a look at fileconfigmanager.cpp for details.
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/FILES b/searchcore/src/tests/proton/documentdb/fileconfigmanager/FILES
new file mode 100644
index 00000000000..842440c7182
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/FILES
@@ -0,0 +1 @@
+fileconfigmanager_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/attributes.cfg b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/attributes.cfg
new file mode 100644
index 00000000000..fa887fb404e
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/attributes.cfg
@@ -0,0 +1,2 @@
+attribute[1]
+attribute[0].name "afield"
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/documenttypes.cfg b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/documenttypes.cfg
new file mode 100644
index 00000000000..9cfeb00111a
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/documenttypes.cfg
@@ -0,0 +1,15 @@
+documenttype[1]
+documenttype[0].bodystruct -1270491200
+documenttype[0].headerstruct 306916075
+documenttype[0].id -877171244
+documenttype[0].name "test"
+documenttype[0].version 0
+documenttype[0].datatype[2]
+documenttype[0].datatype[0].id -1270491200
+documenttype[0].datatype[0].type STRUCT
+documenttype[0].datatype[0].sstruct.name "test.body"
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[1].id 306916075
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].sstruct.name "test.header"
+documenttype[0].datatype[1].sstruct.version 0
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/indexschema.cfg b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/indexschema.cfg
new file mode 100644
index 00000000000..fdd519542db
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/indexschema.cfg
@@ -0,0 +1,3 @@
+indexfield[1]
+indexfield[0].name "ifield"
+
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/juniperrc.cfg b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/juniperrc.cfg
new file mode 100644
index 00000000000..8f89b73e22d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/juniperrc.cfg
@@ -0,0 +1,2 @@
+override[1]
+override[0].fieldname "jfield"
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/mycfg.cfg b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/mycfg.cfg
new file mode 100644
index 00000000000..09e75cc45f8
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/mycfg.cfg
@@ -0,0 +1 @@
+myField "foo"
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/rank-profiles.cfg b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/rank-profiles.cfg
new file mode 100644
index 00000000000..a8ed6c47477
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/rank-profiles.cfg
@@ -0,0 +1,2 @@
+rankprofile[1]
+rankprofile[0].name "default"
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/summary.cfg b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/summary.cfg
new file mode 100644
index 00000000000..02e3b0cdafe
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/summary.cfg
@@ -0,0 +1,7 @@
+defaultsummaryid 1
+classes[1]
+classes[0].id 1
+classes[0].name "sclass"
+classes[0].fields[1]
+classes[0].fields[0].name "sfield"
+classes[0].fields[0].type "longstring"
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/summarymap.cfg b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/summarymap.cfg
new file mode 100644
index 00000000000..17ce68e3319
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/cfg/summarymap.cfg
@@ -0,0 +1,3 @@
+override[1]
+override[0].field "ofield"
+override[0].command "empty"
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
new file mode 100644
index 00000000000..0960ff6b2fe
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
@@ -0,0 +1,322 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("fileconfigmanager_test");
+
+#include "config-mycfg.h"
+#include <vespa/searchcore/proton/server/fileconfigmanager.h>
+#include <vespa/searchcore/proton/server/documentdbconfigmanager.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/closure.h>
+#include <vespa/searchcore/proton/common/schemautil.h>
+
+using namespace config;
+using namespace document;
+using namespace proton;
+using namespace search::index;
+using namespace search;
+using namespace vespa::config::search::core;
+using namespace vespa::config::search;
+
+typedef DocumentDBConfigHelper DBCM;
+typedef DocumentDBConfig::DocumenttypesConfigSP DocumenttypesConfigSP;
+using vespalib::nbostream;
+
+vespalib::string myId("myconfigid");
+
+namespace
+{
+
+DocumentDBConfig::SP
+getConfig(int64_t generation, const Schema::SP &schema)
+{
+    typedef DocumentDBConfig::RankProfilesConfigSP RankProfilesConfigSP; 
+    typedef DocumentDBConfig::IndexschemaConfigSP IndexschemaConfigSP;
+    typedef DocumentDBConfig::AttributesConfigSP AttributesConfigSP;
+    typedef DocumentDBConfig::SummaryConfigSP SummaryConfigSP; 
+    typedef DocumentDBConfig::SummarymapConfigSP SummarymapConfigSP;
+    typedef DocumentDBConfig::JuniperrcConfigSP JuniperrcConfigSP;
+    typedef DocumentDBConfig::DocumenttypesConfigSP DocumenttypesConfigSP;
+
+    RankProfilesConfigSP rp(new vespa::config::search::RankProfilesConfig);
+    IndexschemaConfigSP is(new vespa::config::search::IndexschemaConfig);
+    AttributesConfigSP a(new vespa::config::search::AttributesConfig);
+    SummaryConfigSP s(new vespa::config::search::SummaryConfig);
+    SummarymapConfigSP sm(new vespa::config::search::SummarymapConfig);
+    JuniperrcConfigSP j(new vespa::config::search::summary::JuniperrcConfig);
+    DocumenttypesConfigSP dt(new document::DocumenttypesConfig);
+    document::DocumentTypeRepo::SP dtr(new document::DocumentTypeRepo);
+    search::TuneFileDocumentDB::SP tf(new search::TuneFileDocumentDB);
+    DocumentDBMaintenanceConfig::SP ddbm(new DocumentDBMaintenanceConfig);
+    
+    return DocumentDBConfig::SP(
+            new DocumentDBConfig(
+                    generation,
+                    rp,
+                    is,
+                    a,
+                    s,
+                    sm,
+                    j,
+                    dt,
+                    dtr,
+                    tf,
+                    schema,
+                    ddbm,
+                    "client", "test"));
+}
+
+Schema::SP
+getSchema(int step)
+{
+    Schema::SP schema(new Schema);
+    schema->addIndexField(Schema::IndexField("foo1", Schema::STRING));
+    if (step < 2) {
+        schema->addIndexField(Schema::IndexField("foo2", Schema::STRING));
+    }
+    if (step < 1) {
+        schema->addIndexField(Schema::IndexField("foo3", Schema::STRING));
+    }
+    return schema;
+}
+
+        }
+
+DocumentDBConfig::SP
+makeBaseConfigSnapshot()
+{
+    config::DirSpec spec("cfg");
+    ConfigKeySet extraKeySet;
+    extraKeySet.add<MycfgConfig>("");
+    DBCM dbcm(spec, "test", extraKeySet);
+    DocumenttypesConfigSP dtcfg(config::ConfigGetter<DocumenttypesConfig>::getConfig("", spec).release());
+    BootstrapConfig::SP b(new BootstrapConfig(1,
+                                              dtcfg,
+                                              DocumentTypeRepo::SP(new DocumentTypeRepo(*dtcfg)),
+                                              BootstrapConfig::ProtonConfigSP(new ProtonConfig()),
+                                              TuneFileDocumentDB::SP(new TuneFileDocumentDB())));
+    dbcm.forwardConfig(b);
+    dbcm.nextGeneration(0);
+    DocumentDBConfig::SP snap = dbcm.getConfig();
+    snap->setConfigId(myId);
+    ASSERT_TRUE(snap.get() != NULL);
+    return snap;
+}
+
+Schema
+makeHistorySchema()
+{
+    Schema hs;
+    hs.addIndexField(Schema::IndexField("history", Schema::STRING));
+    return hs;
+}
+
+void
+saveBaseConfigSnapshot(const DocumentDBConfig &snap, const Schema &history, SerialNum num)
+{
+    FileConfigManager cm("out", myId, snap.getDocTypeName());
+    cm.saveConfig(snap, history, num);
+}
+
+
+DocumentDBConfig::SP
+makeEmptyConfigSnapshot(void)
+{
+    return DocumentDBConfig::SP(new DocumentDBConfig(
+                                      0,
+                                      DocumentDBConfig::RankProfilesConfigSP(),
+                                      DocumentDBConfig::IndexschemaConfigSP(),
+                                      DocumentDBConfig::AttributesConfigSP(),
+                                      DocumentDBConfig::SummaryConfigSP(),
+                                      DocumentDBConfig::SummarymapConfigSP(),
+                                      DocumentDBConfig::JuniperrcConfigSP(),
+                                      DocumenttypesConfigSP(),
+                                      DocumentTypeRepo::SP(),
+                                      TuneFileDocumentDB::SP(
+                                              new TuneFileDocumentDB()),
+                                      Schema::SP(),
+                                      DocumentDBMaintenanceConfig::SP(),
+                                      "client", "test"));
+}
+
+void incInt(int *i, const DocumentType&) { ++*i; }
+
+void
+assertEqualExtraConfigs(const DocumentDBConfig &expSnap, const DocumentDBConfig &actSnap)
+{
+    const ConfigSnapshot &exp = expSnap.getExtraConfigs();
+    const ConfigSnapshot &act = actSnap.getExtraConfigs();
+    EXPECT_EQUAL(1u, exp.size());
+    EXPECT_EQUAL(1u, act.size());
+    std::unique_ptr<MycfgConfig> expCfg = exp.getConfig<MycfgConfig>("");
+    std::unique_ptr<MycfgConfig> actCfg = act.getConfig<MycfgConfig>("");
+    EXPECT_EQUAL("foo", expCfg->myField);
+    EXPECT_EQUAL("foo", actCfg->myField);
+}
+
+void
+assertEqualSnapshot(const DocumentDBConfig &exp, const DocumentDBConfig &act)
+{
+    EXPECT_TRUE(exp.getRankProfilesConfig() == act.getRankProfilesConfig());
+    EXPECT_TRUE(exp.getIndexschemaConfig() == act.getIndexschemaConfig());
+    EXPECT_TRUE(exp.getAttributesConfig() == act.getAttributesConfig());
+    EXPECT_TRUE(exp.getSummaryConfig() == act.getSummaryConfig());
+    EXPECT_TRUE(exp.getSummarymapConfig() == act.getSummarymapConfig());
+    EXPECT_TRUE(exp.getJuniperrcConfig() == act.getJuniperrcConfig());
+    int expTypeCount = 0;
+    int actTypeCount = 0;
+    exp.getDocumentTypeRepoSP()->forEachDocumentType(
+            *vespalib::makeClosure(incInt, &expTypeCount));
+    act.getDocumentTypeRepoSP()->forEachDocumentType(
+            *vespalib::makeClosure(incInt, &actTypeCount));
+    EXPECT_EQUAL(expTypeCount, actTypeCount);
+    EXPECT_TRUE(*exp.getSchemaSP() == *act.getSchemaSP());
+    EXPECT_EQUAL(expTypeCount, actTypeCount);
+    EXPECT_EQUAL(exp.getConfigId(), act.getConfigId());
+    assertEqualExtraConfigs(exp, act);
+}
+
+TEST_FF("requireThatConfigCanBeSavedAndLoaded", DocumentDBConfig::SP(makeBaseConfigSnapshot()),
+                                                Schema(makeHistorySchema()))
+{
+    saveBaseConfigSnapshot(*f1, f2, 20);
+    DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot());
+    Schema::SP ehs;
+    {
+        FileConfigManager cm("out", myId, "dummy");
+        cm.loadConfig(*esnap, 20, esnap, ehs);
+    }
+    assertEqualSnapshot(*f1, *esnap);
+    EXPECT_TRUE(f2 == *ehs);
+}
+
+TEST_FF("requireThatConfigCanBeSerializedAndDeserialized", DocumentDBConfig::SP(makeBaseConfigSnapshot()),
+                                                           Schema(makeHistorySchema()))
+{
+    saveBaseConfigSnapshot(*f1, f2, 30);
+    nbostream stream;
+    {
+        FileConfigManager cm("out", myId, "dummy");
+        cm.serializeConfig(30, stream);
+    }
+    {
+        FileConfigManager cm("out", myId, "dummy");
+        cm.deserializeConfig(40, stream);
+    }
+    DocumentDBConfig::SP fsnap(makeEmptyConfigSnapshot());
+    Schema::SP fhs;
+    {
+        FileConfigManager cm("out", myId, "dummy");
+        cm.loadConfig(*fsnap, 40, fsnap, fhs);
+    }
+    assertEqualSnapshot(*f1, *fsnap);
+    EXPECT_TRUE(f2 == *fhs);
+    EXPECT_EQUAL("dummy", fsnap->getDocTypeName());
+}
+
+TEST_FF("requireThatWipeHistoryCanBeSaved", DocumentDBConfig::SP(makeBaseConfigSnapshot()),
+                                            Schema(makeHistorySchema()))
+{
+    saveBaseConfigSnapshot(*f1, f2, 50);
+    {
+        FileConfigManager cm("out", myId, "dummy");
+        cm.saveWipeHistoryConfig(60, 0);
+    }
+    DocumentDBConfig::SP gsnap(makeEmptyConfigSnapshot());
+    Schema::SP ghs;
+    {
+        FileConfigManager cm("out", myId, "dummy");
+        cm.loadConfig(*gsnap, 60, gsnap, ghs);
+    }
+    assertEqualSnapshot(*f1, *gsnap);
+    EXPECT_TRUE(f2 != *ghs);
+    EXPECT_TRUE(!f2.empty());
+    EXPECT_TRUE(ghs->empty());
+}
+
+
+TEST("require that wipe history clears only portions of history")
+{
+    FileConfigManager cm("out2", myId, "dummy");
+    Schema::SP schema(getSchema(0));
+    Schema::SP history(new Schema);
+    DocumentDBConfig::SP config(getConfig(5, schema));
+    cm.saveConfig(*config, *history, 5);
+    Schema::SP oldSchema(schema);
+    schema = getSchema(1);
+    config = getConfig(6, schema);
+    history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history,
+                                            100);
+    cm.saveConfig(*config, *history, 10);
+    oldSchema = schema;
+    schema = getSchema(2);
+    config = getConfig(7, schema);
+    history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history,
+                                            200);
+    cm.saveConfig(*config, *history, 15);
+    cm.saveWipeHistoryConfig(20, 50);
+    cm.saveWipeHistoryConfig(25, 100);
+    cm.saveWipeHistoryConfig(30, 150);
+    cm.saveWipeHistoryConfig(35, 200);
+    cm.saveWipeHistoryConfig(40, 250);
+    DocumentDBConfig::SP oldconfig(config);
+    cm.loadConfig(*oldconfig, 20, config, history);
+    EXPECT_EQUAL(2u, history->getNumIndexFields());
+    oldconfig = config;
+    cm.loadConfig(*oldconfig, 25, config, history);
+    EXPECT_EQUAL(2u, history->getNumIndexFields());
+    oldconfig = config;
+    cm.loadConfig(*oldconfig, 30, config, history);
+    EXPECT_EQUAL(1u, history->getNumIndexFields());
+    oldconfig = config;
+    cm.loadConfig(*oldconfig, 35, config, history);
+    EXPECT_EQUAL(1u, history->getNumIndexFields());
+    oldconfig = config;
+    cm.loadConfig(*oldconfig, 40, config, history);
+    EXPECT_EQUAL(0u, history->getNumIndexFields());
+}
+
+TEST_FF("requireThatConfigCanBeLoadedWithoutExtraConfigsDataFile", DocumentDBConfig::SP(makeBaseConfigSnapshot()),
+                                                                   Schema(makeHistorySchema()))
+{
+    saveBaseConfigSnapshot(*f1, f2, 70);
+    EXPECT_TRUE(vespalib::unlink("out/config-70/extraconfigs.dat"));
+    DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot());
+    Schema::SP ehs;
+    {
+        FileConfigManager cm("out", myId, "dummy");
+        cm.loadConfig(*esnap, 70, esnap, ehs);
+    }
+    EXPECT_EQUAL(0u, esnap->getExtraConfigs().size());
+}
+
+
+TEST_FF("requireThatVisibilityDelayIsPropagated",
+        DocumentDBConfig::SP(makeBaseConfigSnapshot()),
+        Schema(makeHistorySchema()))
+{
+    saveBaseConfigSnapshot(*f1, f2, 80);
+    DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot());
+    Schema::SP ehs;
+    {
+        ProtonConfigBuilder protonConfigBuilder;
+        ProtonConfigBuilder::Documentdb ddb;
+        ddb.inputdoctypename = "dummy";
+        ddb.visibilitydelay = 61.0;
+        protonConfigBuilder.documentdb.push_back(ddb);
+        protonConfigBuilder.maxvisibilitydelay = 100.0;
+        FileConfigManager cm("out", myId, "dummy");
+        using ProtonConfigSP = BootstrapConfig::ProtonConfigSP;
+        cm.setProtonConfig(
+                ProtonConfigSP(new ProtonConfig(protonConfigBuilder)));
+        cm.loadConfig(*esnap, 70, esnap, ehs);
+    }
+    EXPECT_EQUAL(0u, esnap->getExtraConfigs().size());
+    EXPECT_EQUAL(61.0, esnap->getMaintenanceConfigSP()->getVisibilityDelay().sec());
+}
+
+
+
+TEST_MAIN() { TEST_RUN_ALL(); }
+
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.sh b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.sh
new file mode 100644
index 00000000000..4d1279a8413
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+rm -rf out
+rm -rf out2
+$VALGRIND ./searchcore_fileconfigmanager_test_app
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/mycfg.def b/searchcore/src/tests/proton/documentdb/fileconfigmanager/mycfg.def
new file mode 100644
index 00000000000..d31c1b61f07
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/mycfg.def
@@ -0,0 +1,4 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=config
+
+myField string default=""
diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/.gitignore b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/.gitignore
new file mode 100644
index 00000000000..1e657f33c1a
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/.gitignore
@@ -0,0 +1 @@
+searchcore_job_tracked_maintenance_job_test_app
diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt
new file mode 100644
index 00000000000..3b81994e7da
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_job_tracked_maintenance_job_test_app
+    SOURCES
+    job_tracked_maintenance_job_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_proton_metrics
+)
+vespa_add_test(NAME searchcore_job_tracked_maintenance_job_test_app COMMAND searchcore_job_tracked_maintenance_job_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/DESC b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/DESC
new file mode 100644
index 00000000000..4ba7520eab8
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/DESC
@@ -0,0 +1,2 @@
+job tracked maintenance job test. Take a look at job_tracked_maintenance_job_test.cpp for details.
+
diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/FILES b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/FILES
new file mode 100644
index 00000000000..a871a1fa8aa
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/FILES
@@ -0,0 +1 @@
+job_tracked_maintenance_job_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp
new file mode 100644
index 00000000000..e483bc35b96
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp
@@ -0,0 +1,134 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("job_tracked_maintenance_test");
+
+#include <vespa/searchcore/proton/server/job_tracked_maintenance_job.h>
+#include <vespa/searchcore/proton/test/simple_job_tracker.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/sync.h>
+
+using namespace proton;
+using namespace vespalib;
+using test::SimpleJobTracker;
+typedef std::unique_ptr<Gate> GateUP;
+typedef std::vector<GateUP> GateVector;
+
+GateVector
+getGateVector(size_t size)
+{
+    GateVector retval;
+    for (size_t i = 0; i < size; ++i) {
+        retval.push_back(std::move(GateUP(new Gate())));
+    }
+    return retval;
+}
+
+struct MyMaintenanceJob : public IMaintenanceJob
+{
+    GateVector _runGates;
+    size_t     _runIdx;
+    MyMaintenanceJob(size_t numRuns)
+        : IMaintenanceJob("myjob", 10, 20),
+          _runGates(getGateVector(numRuns)),
+          _runIdx(0)
+    {}
+    void block() {
+        setBlocked(true);
+    }
+    virtual bool run() {
+        _runGates[_runIdx++]->await(5000);
+        return _runIdx == _runGates.size();
+    }
+};
+
+struct Fixture
+{
+    SimpleJobTracker::SP _tracker;
+    IMaintenanceJob::UP _job;
+    MyMaintenanceJob *_myJob;
+    IMaintenanceJob::UP _trackedJob;
+    bool _runRetval;
+    GateVector _runGates;
+    size_t _runIdx;
+    ThreadStackExecutor _exec;
+    Fixture(size_t numRuns = 1)
+        : _tracker(new SimpleJobTracker(1)),
+          _job(new MyMaintenanceJob(numRuns)),
+          _myJob(static_cast<MyMaintenanceJob *>(_job.get())),
+          _trackedJob(new JobTrackedMaintenanceJob(_tracker, std::move(_job))),
+          _runRetval(false),
+          _runGates(getGateVector(numRuns)),
+          _runIdx(0),
+          _exec(1, 64000)
+    {
+    }
+    void runJob() {
+        _runRetval = _trackedJob->run();
+        _runGates[_runIdx++]->countDown();
+    }
+    void assertTracker(size_t startedGateCount, size_t endedGateCount) {
+        EXPECT_EQUAL(startedGateCount, _tracker->_started.getCount());
+        EXPECT_EQUAL(endedGateCount, _tracker->_ended.getCount());
+    }
+    void runJobAndWait(size_t runIdx, size_t startedGateCount, size_t endedGateCount) {
+        _exec.execute(makeTask(makeClosure(this, &Fixture::runJob)));
+        _tracker->_started.await(5000);
+        assertTracker(startedGateCount, endedGateCount);
+        _myJob->_runGates[runIdx]->countDown();
+        _runGates[runIdx]->await(5000);
+    }
+};
+
+TEST_F("require that maintenance job name, delay and interval are preserved", Fixture)
+{
+    EXPECT_EQUAL("myjob", f._trackedJob->getName());
+    EXPECT_EQUAL(10, f._trackedJob->getDelay());
+    EXPECT_EQUAL(20, f._trackedJob->getInterval());
+}
+
+TEST_F("require that maintenance job that needs 1 run is tracked", Fixture)
+{
+    f.assertTracker(1, 1);
+    f.runJobAndWait(0, 0, 1);
+    f.assertTracker(0, 0);
+    EXPECT_TRUE(f._runRetval);
+}
+
+TEST_F("require that maintenance job that needs several runs is tracked", Fixture(2))
+{
+    f.assertTracker(1, 1);
+    f.runJobAndWait(0, 0, 1);
+    f.assertTracker(0, 1);
+    EXPECT_FALSE(f._runRetval);
+
+    f.runJobAndWait(1, 0, 1);
+    f.assertTracker(0, 0);
+    EXPECT_TRUE(f._runRetval);
+}
+
+TEST_F("require that maintenance job that is destroyed is tracked", Fixture(2))
+{
+    f.assertTracker(1, 1);
+    f.runJobAndWait(0, 0, 1);
+    f.assertTracker(0, 1);
+    EXPECT_FALSE(f._runRetval);
+
+    f._trackedJob.reset();
+    f.assertTracker(0, 0);
+}
+
+TEST_F("require that block calls are sent to underlying job", Fixture)
+{
+    EXPECT_FALSE(f._trackedJob->isBlocked());
+    f._myJob->block();
+    EXPECT_TRUE(f._myJob->isBlocked());
+    EXPECT_TRUE(f._trackedJob->isBlocked());
+    f._trackedJob->unBlock();
+    EXPECT_FALSE(f._myJob->isBlocked());
+    EXPECT_FALSE(f._trackedJob->isBlocked());
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/.gitignore b/searchcore/src/tests/proton/documentdb/lid_space_compaction/.gitignore
new file mode 100644
index 00000000000..c031fe6605d
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/.gitignore
@@ -0,0 +1 @@
+searchcore_lid_space_compaction_test_app
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt
new file mode 100644
index 00000000000..938e0dc7baf
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_lid_space_compaction_test_app
+    SOURCES
+    lid_space_compaction_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_initializer
+    searchcore_feedoperation
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_lid_space_compaction_test_app COMMAND searchcore_lid_space_compaction_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/DESC b/searchcore/src/tests/proton/documentdb/lid_space_compaction/DESC
new file mode 100644
index 00000000000..b361d232d13
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/DESC
@@ -0,0 +1,2 @@
+Test for lid space compaction. Take a look at lid_space_compaction_test.cpp for details.
+
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/FILES b/searchcore/src/tests/proton/documentdb/lid_space_compaction/FILES
new file mode 100644
index 00000000000..48fa9ef87c6
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/FILES
@@ -0,0 +1 @@
+lid_space_compaction_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
new file mode 100644
index 00000000000..aa0ab2ebfb1
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
@@ -0,0 +1,450 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("lid_space_compaction_test");
+
+#include <vespa/searchcore/proton/server/i_lid_space_compaction_handler.h>
+#include <vespa/searchcore/proton/server/lid_space_compaction_handler.h>
+#include <vespa/searchcore/proton/server/lid_space_compaction_job.h>
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/server/ifrozenbuckethandler.h>
+
+using namespace document;
+using namespace proton;
+using namespace search;
+using namespace search::index;
+using namespace vespalib;
+using storage::spi::Timestamp;
+
+const uint32_t SUBDB_ID = 2;
+const double JOB_DELAY = 1.0;
+const uint32_t ALLOWED_LID_BLOAT = 1;
+const double ALLOWED_LID_BLOAT_FACTOR = 0.3;
+const uint32_t MAX_DOCS_TO_SCAN = 100;
+const vespalib::string DOC_ID = "id:test:searchdocument::0";
+const BucketId BUCKET_ID_1(1);
+const BucketId BUCKET_ID_2(2);
+const Timestamp TIMESTAMP_1(1);
+const GlobalId GID_1;
+
+typedef std::vector<uint32_t> LidVector;
+typedef std::pair<uint32_t, uint32_t> LidPair;
+typedef std::vector<LidPair> LidPairVector;
+
+struct MyScanIterator : public IDocumentScanIterator
+{
+    LidVector _lids;
+    LidVector::const_iterator _itr;
+    bool _validItr;
+    MyScanIterator(const LidVector &lids) : _lids(lids), _itr(_lids.begin()), _validItr(true) {}
+    virtual bool valid() const {
+        return _validItr;
+    }
+    virtual search::DocumentMetaData next(uint32_t compactLidLimit,
+                                          uint32_t maxDocsToScan,
+                                          bool retry) {
+        if (!retry && _itr != _lids.begin()) {
+            ++_itr;
+        }
+        for (uint32_t i = 0; i < maxDocsToScan && _itr != _lids.end() && (*_itr) <= compactLidLimit;
+                ++i, ++_itr) {}
+        if (_itr != _lids.end()) {
+            uint32_t lid = *_itr;
+            if (lid > compactLidLimit) {
+                return search::DocumentMetaData(lid, TIMESTAMP_1, BUCKET_ID_1, GID_1);
+            }
+        } else {
+            _validItr = false;
+        }
+        return search::DocumentMetaData();
+    }
+};
+
+struct MyHandler : public ILidSpaceCompactionHandler
+{
+    std::vector<LidUsageStats> _stats;
+    std::vector<LidVector> _lids;
+    mutable uint32_t _moveFromLid;
+    mutable uint32_t _moveToLid;
+    uint32_t _handleMoveCnt;
+    uint32_t _wantedSubDbId;
+    uint32_t _wantedLidLimit;
+    mutable uint32_t _iteratorCnt;
+
+    MyHandler()
+        : _stats(),
+          _moveFromLid(0),
+          _moveToLid(0),
+          _handleMoveCnt(0),
+          _wantedSubDbId(0),
+          _wantedLidLimit(0),
+          _iteratorCnt(0)
+    {}
+    virtual vespalib::string getName() const {
+        return "myhandler";
+    }
+    virtual uint32_t getSubDbId() const { return 2; }
+    virtual LidUsageStats getLidStatus() const {
+        ASSERT_TRUE(_handleMoveCnt < _stats.size());
+        return _stats[_handleMoveCnt];
+    }
+    virtual IDocumentScanIterator::UP getIterator() const {
+        ASSERT_TRUE(_iteratorCnt < _lids.size());
+        return IDocumentScanIterator::UP(new MyScanIterator(_lids[_iteratorCnt++]));
+    }
+    virtual MoveOperation::UP createMoveOperation(const search::DocumentMetaData &document,
+                                                  uint32_t moveToLid) const {
+        ASSERT_TRUE(document.lid > moveToLid);
+        _moveFromLid = document.lid;
+        _moveToLid = moveToLid;
+        return MoveOperation::UP(new MoveOperation());
+    }
+    virtual void handleMove(const MoveOperation &) {
+        ++_handleMoveCnt;
+    }
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) {
+        _wantedSubDbId = op.getSubDbId();
+        _wantedLidLimit = op.getLidLimit();
+    }
+};
+
+struct MyStorer : public IOperationStorer
+{
+    uint32_t _moveCnt;
+    uint32_t _compactCnt;
+    MyStorer()
+        : _moveCnt(0),
+          _compactCnt(0)
+    {}
+    virtual void storeOperation(FeedOperation &op) {
+        if (op.getType() == FeedOperation::MOVE) {
+            ++ _moveCnt;
+        } else if (op.getType() == FeedOperation::COMPACT_LID_SPACE) {
+            ++_compactCnt;
+        }
+    }
+};
+
+struct MyFrozenBucketHandler : public IFrozenBucketHandler
+{
+    BucketId _bucket;
+    MyFrozenBucketHandler() : _bucket() {}
+    virtual ExclusiveBucketGuard::UP acquireExclusiveBucket(BucketId bucket) override {
+        return (_bucket == bucket)
+               ? ExclusiveBucketGuard::UP()
+               : std::make_unique<ExclusiveBucketGuard>(bucket);
+    }
+    virtual void addListener(IBucketFreezeListener *) override { }
+    virtual void removeListener(IBucketFreezeListener *) override { }
+};
+
+struct MyFeedView : public test::DummyFeedView
+{
+    MyFeedView(const DocumentTypeRepo::SP &repo)
+        : test::DummyFeedView(repo)
+    {
+    }
+};
+
+struct MyDocumentStore : public test::DummyDocumentStore
+{
+    Document::SP _readDoc;
+    mutable uint32_t _readLid;
+    MyDocumentStore() : _readDoc(), _readLid(0) {}
+    virtual document::Document::UP
+    read(search::DocumentIdT lid, const document::DocumentTypeRepo &) const {
+        _readLid = lid;
+        return Document::UP(_readDoc->clone());
+    }
+};
+
+struct MySummaryManager : public test::DummySummaryManager
+{
+    MyDocumentStore _store;
+    MySummaryManager() : _store() {}
+    virtual search::IDocumentStore &getBackingStore() { return _store; }
+};
+
+struct MySubDb : public test::DummyDocumentSubDb
+{
+    DocumentTypeRepo::SP _repo;
+    MySubDb(const DocumentTypeRepo::SP &repo,
+            std::shared_ptr<BucketDBOwner> bucketDB)
+        : test::DummyDocumentSubDb(bucketDB, SUBDB_ID),
+          _repo(repo)
+    {
+        _summaryManager.reset(new MySummaryManager());
+    }
+    virtual IFeedView::SP getFeedView() const {
+        return IFeedView::SP(new MyFeedView(_repo));
+    }
+};
+
+struct JobFixture
+{
+    MyHandler _handler;
+    MyStorer _storer;
+    MyFrozenBucketHandler _frozenHandler;
+    LidSpaceCompactionJob _job;
+    JobFixture(uint32_t allowedLidBloat = ALLOWED_LID_BLOAT,
+               double allowedLidBloatFactor = ALLOWED_LID_BLOAT_FACTOR,
+               uint32_t maxDocsToScan = MAX_DOCS_TO_SCAN)
+        : _handler(),
+          _job(DocumentDBLidSpaceCompactionConfig(JOB_DELAY,
+                  allowedLidBloat, allowedLidBloatFactor, maxDocsToScan),
+                  _handler, _storer, _frozenHandler)
+    {
+    }
+    JobFixture &addStats(uint32_t docIdLimit,
+                         const LidVector &usedLids,
+                         const LidPairVector &usedFreePairs) {
+        return addMultiStats(docIdLimit, {usedLids}, usedFreePairs);
+    }
+    JobFixture &addMultiStats(uint32_t docIdLimit,
+                              const std::vector<LidVector> &usedLidsVector,
+                              const LidPairVector &usedFreePairs) {
+        uint32_t numDocs = usedLidsVector[0].size();
+        for (auto pair : usedFreePairs) {
+            uint32_t highestUsedLid = pair.first;
+            uint32_t lowestFreeLid = pair.second;
+            _handler._stats.push_back(LidUsageStats
+                    (docIdLimit, numDocs, lowestFreeLid, highestUsedLid));
+        }
+        _handler._lids = usedLidsVector;
+        return *this;
+    }
+    JobFixture &addStats(uint32_t docIdLimit,
+                         uint32_t numDocs,
+                         uint32_t lowestFreeLid,
+                         uint32_t highestUsedLid) {
+        _handler._stats.push_back(LidUsageStats
+                (docIdLimit, numDocs, lowestFreeLid, highestUsedLid));
+        return *this;
+    }
+    bool run() {
+        return _job.run();
+    }
+    JobFixture &endScan() {
+        EXPECT_FALSE(run());
+        return *this;
+    }
+    JobFixture &compact() {
+        EXPECT_TRUE(run());
+        return *this;
+    }
+};
+
+struct HandlerFixture
+{
+    DocBuilder _docBuilder;
+    std::shared_ptr<BucketDBOwner> _bucketDB;
+    MySubDb _subDb;
+    MySummaryManager &_summaryMgr;
+    MyDocumentStore &_docStore;
+    LidSpaceCompactionHandler _handler;
+    HandlerFixture()
+        : _docBuilder(Schema()),
+          _bucketDB(std::make_shared<BucketDBOwner>()),
+          _subDb(_docBuilder.getDocumentTypeRepo(), _bucketDB),
+          _summaryMgr(static_cast<MySummaryManager &>(*_subDb.getSummaryManager())),
+          _docStore(_summaryMgr._store),
+          _handler(_subDb, "test")
+    {
+        _docStore._readDoc = _docBuilder.startDocument(DOC_ID).endDocument();
+    }
+};
+
+bool
+assertJobContext(uint32_t moveToLid,
+                 uint32_t moveFromLid,
+                 uint32_t handleMoveCnt,
+                 uint32_t wantedLidLimit,
+                 uint32_t compactStoreCnt,
+                 const JobFixture &f)
+{
+    if (!EXPECT_EQUAL(moveToLid, f._handler._moveToLid)) return false;
+    if (!EXPECT_EQUAL(moveFromLid, f._handler._moveFromLid)) return false;
+    if (!EXPECT_EQUAL(handleMoveCnt, f._handler._handleMoveCnt)) return false;
+    if (!EXPECT_EQUAL(handleMoveCnt, f._storer._moveCnt)) return false;
+    if (!EXPECT_EQUAL(wantedLidLimit, f._handler._wantedLidLimit)) return false;
+    if (!EXPECT_EQUAL(compactStoreCnt, f._storer._compactCnt)) return false;
+    return true;
+}
+
+TEST_F("require that handler name is used as part of job name", JobFixture)
+{
+    EXPECT_EQUAL("lid_space_compaction.myhandler", f._job.getName());
+}
+
+TEST_F("require that no move operation is created if lid bloat factor is below limit", JobFixture)
+{
+    // 20% bloat < 30% allowed bloat
+    f.addStats(10, {1,3,4,5,6,7,9}, {{9,2}});
+    EXPECT_TRUE(f.run());
+    EXPECT_TRUE(assertJobContext(0, 0, 0, 0, 0, f));
+}
+
+TEST("require that no move operation is created if lid bloat is below limit")
+{
+    JobFixture f(3, 0.1);
+    // 20% bloat >= 10% allowed bloat BUT lid bloat (2) < allowed lid bloat (3)
+    f.addStats(10, {1,3,4,5,6,7,9}, {{9,2}});
+    EXPECT_TRUE(f.run());
+    EXPECT_TRUE(assertJobContext(0, 0, 0, 0, 0, f));
+}
+
+TEST_F("require that no move operation is created and compaction is initiated", JobFixture)
+{
+    // no documents to move: lowestFreeLid(7) > highestUsedLid(6)
+    f.addStats(10, {1,2,3,4,5,6}, {{6,7}});
+
+    // must scan to find that no documents should be moved
+    f.endScan().compact();
+    EXPECT_TRUE(assertJobContext(0, 0, 0, 7, 1, f));
+}
+
+TEST_F("require that 1 move operation is created and compaction is initiated", JobFixture)
+{
+    f.addStats(10, {1,3,4,5,6,9},
+            {{9,2},   // 30% bloat: move 9 -> 2
+             {6,7}}); // no documents to move
+
+    EXPECT_FALSE(f.run()); // scan
+    EXPECT_TRUE(assertJobContext(2, 9, 1, 0, 0, f));
+    f.endScan().compact();
+    EXPECT_TRUE(assertJobContext(2, 9, 1, 7, 1, f));
+}
+
+TEST_F("require that job returns false when multiple move operations or compaction are needed",
+        JobFixture)
+{
+    f.addStats(10, {1,5,6,9,8,7},
+            {{9,2},   // 30% bloat: move 9 -> 2
+             {8,3},   // move 8 -> 3
+             {7,4},   // move 7 -> 4
+             {6,7}}); // no documents to move
+
+    EXPECT_FALSE(f.run());
+    EXPECT_TRUE(assertJobContext(2, 9, 1, 0, 0, f));
+    EXPECT_FALSE(f.run());
+    EXPECT_TRUE(assertJobContext(3, 8, 2, 0, 0, f));
+    EXPECT_FALSE(f.run());
+    EXPECT_TRUE(assertJobContext(4, 7, 3, 0, 0, f));
+    f.endScan().compact();
+    EXPECT_TRUE(assertJobContext(4, 7, 3, 7, 1, f));
+}
+
+TEST_F("require that job is blocked if trying to move document for frozen bucket", JobFixture)
+{
+    f._frozenHandler._bucket = BUCKET_ID_1;
+    EXPECT_FALSE(f._job.isBlocked());
+    f.addStats(10, {1,3,4,5,6,9}, {{9,2}}); // 30% bloat: try to move 9 -> 2
+    f.addStats(0, 0, 0, 0);
+
+    EXPECT_TRUE(f.run()); // bucket frozen
+    EXPECT_TRUE(assertJobContext(0, 0, 0, 0, 0, f));
+    EXPECT_TRUE(f._job.isBlocked());
+
+    f._frozenHandler._bucket = BUCKET_ID_2;
+    f._job.unBlock();
+
+    EXPECT_FALSE(f.run()); // unblocked
+    EXPECT_TRUE(assertJobContext(2, 9, 1, 0, 0, f));
+    EXPECT_FALSE(f._job.isBlocked());
+}
+
+TEST_F("require that job handles invalid document meta data when max docs are scanned",
+        JobFixture(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, 3))
+{
+    f.addStats(10, {1,3,4,5,6,9},
+            {{9,2},   // 30% bloat: move 9 -> 2
+             {6,7}}); // no documents to move
+
+    EXPECT_FALSE(f.run()); // does not find 9 in first scan
+    EXPECT_TRUE(assertJobContext(0, 0, 0, 0, 0, f));
+    EXPECT_FALSE(f.run()); // move 9 -> 2
+    EXPECT_TRUE(assertJobContext(2, 9, 1, 0, 0, f));
+    f.endScan().compact();
+    EXPECT_TRUE(assertJobContext(2, 9, 1, 7, 1, f));
+}
+
+TEST_F("require that job can restart documents scan if lid bloat is still to large",
+        JobFixture(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, 3))
+{
+    f.addMultiStats(10, {{1,3,4,5,6,9},{1,2,4,5,6,8}},
+            {{9,2},   // 30% bloat: move 9 -> 2
+             {8,3},   // move 8 -> 3 (this should trigger rescan as the set of used docs have changed)
+             {6,7}}); // no documents to move
+
+    EXPECT_FALSE(f.run()); // does not find 9 in first scan
+    EXPECT_EQUAL(1u, f._handler._iteratorCnt);
+    // We simulate that the set of used docs have changed between these 2 runs
+    EXPECT_FALSE(f.run()); // move 9 -> 2
+    f.endScan();
+    EXPECT_TRUE(assertJobContext(2, 9, 1, 0, 0, f));
+    EXPECT_EQUAL(2u, f._handler._iteratorCnt);
+    EXPECT_FALSE(f.run()); // does not find 8 in first scan
+    EXPECT_FALSE(f.run()); // move 8 -> 3
+    EXPECT_TRUE(assertJobContext(3, 8, 2, 0, 0, f));
+    f.endScan().compact();
+    EXPECT_TRUE(assertJobContext(3, 8, 2, 7, 1, f));
+}
+
+TEST_F("require that handler uses doctype and subdb name", HandlerFixture)
+{
+    EXPECT_EQUAL("test.dummysubdb", f._handler.getName());
+}
+
+TEST_F("require that createMoveOperation() works as expected", HandlerFixture)
+{
+    const uint32_t moveToLid = 5;
+    const uint32_t moveFromLid = 10;
+    const BucketId bucketId(100);
+    const Timestamp timestamp(200);
+    DocumentMetaData document(moveFromLid, timestamp, bucketId, GlobalId());
+    MoveOperation::UP op = f._handler.createMoveOperation(document, moveToLid);
+    EXPECT_EQUAL(10u, f._docStore._readLid);
+    EXPECT_EQUAL(DbDocumentId(SUBDB_ID, moveFromLid).toString(),
+            op->getPrevDbDocumentId().toString()); // source
+    EXPECT_EQUAL(DbDocumentId(SUBDB_ID, moveToLid).toString(),
+            op->getDbDocumentId().toString()); // target
+    EXPECT_EQUAL(DocumentId(DOC_ID), op->getDocument()->getId());
+    EXPECT_EQUAL(bucketId, op->getBucketId());
+    EXPECT_EQUAL(timestamp, op->getTimestamp());
+}
+
+
+TEST_F("require that held lid is not considered free, blocks job", JobFixture)
+{
+    // Lid 1 on hold or pendingHold, i.e. neither free nor used.
+    f.addMultiStats(3, {{2}}, {{2, 3}});
+    EXPECT_TRUE(f.run());
+    EXPECT_TRUE(assertJobContext(0, 0, 0, 0, 0, f));
+}
+
+TEST_F("require that held lid is not considered free, only compact", JobFixture)
+{
+    // Lid 1 on hold or pendingHold, i.e. neither free nor used.
+    f.addMultiStats(10, {{2}}, {{2, 3}});
+    EXPECT_FALSE(f.run());
+    EXPECT_TRUE(assertJobContext(0, 0, 0, 0, 0, f));
+    f.compact();
+    EXPECT_TRUE(assertJobContext(0, 0, 0, 3, 1, f));
+}
+
+TEST_F("require that held lids are not considered free, one move", JobFixture)
+{
+    // Lids 1,2,3 on hold or pendingHold, i.e. neither free nor used.
+    f.addMultiStats(10, {{5}}, {{5, 4}, {4, 5}});
+    EXPECT_FALSE(f.run());
+    EXPECT_TRUE(assertJobContext(4, 5, 1, 0, 0, f));
+    f.endScan().compact();
+    EXPECT_TRUE(assertJobContext(4, 5, 1, 5, 1, f));
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/.gitignore b/searchcore/src/tests/proton/documentdb/maintenancecontroller/.gitignore
new file mode 100644
index 00000000000..7ce70f9cbcd
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/.gitignore
@@ -0,0 +1,2 @@
+searchcore_frozenbucketsmap_test_app
+searchcore_maintenancecontroller_test_app
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt
new file mode 100644
index 00000000000..4f26a79f0eb
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt
@@ -0,0 +1,38 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_maintenancecontroller_test_app
+    SOURCES
+    maintenancecontroller_test.cpp
+    DEPENDS
+    searchcore_test
+    searchcore_server
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_persistenceengine
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_maintenancecontroller_test_app COMMAND searchcore_maintenancecontroller_test_app)
+vespa_add_executable(searchcore_frozenbucketsmap_test_app
+    SOURCES
+    frozenbucketsmap_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_feedoperation
+    searchcore_matching
+    searchcore_attribute
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_persistenceengine
+    searchcore_grouping
+    searchcore_proton_metrics
+    searchcore_util
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_frozenbucketsmap_test_app COMMAND searchcore_frozenbucketsmap_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/DESC b/searchcore/src/tests/proton/documentdb/maintenancecontroller/DESC
new file mode 100644
index 00000000000..ad4e910a6f1
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/DESC
@@ -0,0 +1,2 @@
+maintenance controller test. Take a look at maintenancecontroller_test.cpp
+for details.
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/FILES b/searchcore/src/tests/proton/documentdb/maintenancecontroller/FILES
new file mode 100644
index 00000000000..4bd439640f1
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/FILES
@@ -0,0 +1,2 @@
+maintenancecontroller_test.cpp
+frozenbucketsmap_test.cpp
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/frozenbucketsmap_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/frozenbucketsmap_test.cpp
new file mode 100644
index 00000000000..5dc72d02b15
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/frozenbucketsmap_test.cpp
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("frozenbucketsmap_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/server/frozenbuckets.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+
+using namespace proton;
+using document::BucketId;
+
+class RWTask : public vespalib::Executor::Task {
+public:
+    RWTask(FrozenBucketsMap & m, BucketId b, size_t count) : _b(b), _m(m), _count(count) {}
+protected:
+    const BucketId     _b;
+    FrozenBucketsMap & _m;
+    const size_t       _count;
+};
+
+class Reader : public RWTask {
+public:
+    Reader(FrozenBucketsMap & m, BucketId b, size_t count) :
+        RWTask(m, b, count),
+        numContended(0)
+    {}
+    ~Reader() {
+        LOG(info, "NumContended = %ld", numContended);
+    }
+    void run() override {
+        for (size_t i(0); i < _count; i++) {
+            _m.freezeBucket(_b);
+            if (_m.thawBucket(_b)) {
+                numContended++;
+            }
+        }
+    }
+    size_t numContended;
+};
+
+class Writer : public RWTask {
+public:
+    Writer(FrozenBucketsMap & m, BucketId b, size_t count) :
+            RWTask(m, b, count),
+            numFailed(0),
+            numSucces(0)
+    {}
+    ~Writer() {
+        EXPECT_EQUAL(_count, numSucces + numFailed);
+        LOG(info, "NumSuccess = %ld, NumFailed = %ld", numSucces, numFailed);
+    }
+    void run() override {
+        for (size_t i(0); i < _count; i++) {
+            IFrozenBucketHandler::ExclusiveBucketGuard::UP guard = _m.acquireExclusiveBucket(_b);
+            if (guard) {
+                numSucces++;
+            } else {
+                numFailed++;
+            }
+        }
+    }
+    size_t numFailed;
+    size_t numSucces;
+};
+
+TEST("Race reader and writer on FrozenBucketsMap") {
+    FrozenBucketsMap m;
+    BucketId a(8, 6);
+    constexpr size_t NUM_READERS = 3;
+    constexpr size_t NUM_WRITERS = 1;
+    constexpr size_t READER_COUNT = 1000000;
+    constexpr size_t WRITER_COUNT = 1000000;
+    vespalib::ThreadStackExecutor executor(NUM_READERS+NUM_WRITERS, 0x10000);
+    for (size_t i(0); i < NUM_READERS; i++) {
+        EXPECT_FALSE(bool(executor.execute(std::make_unique<Reader>(m, a, READER_COUNT))));
+    }
+    for (size_t i(0); i < NUM_WRITERS; i++) {
+        EXPECT_FALSE(bool(executor.execute(std::make_unique<Writer>(m, a, WRITER_COUNT))));
+    }
+    executor.sync();
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
new file mode 100644
index 00000000000..0513d8f45d9
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -0,0 +1,1472 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("maintenancecontroller_test");
+#include <vespa/searchcore/proton/test/test.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/server/idocumentmovehandler.h>
+#include <vespa/searchcore/proton/server/executor_thread_service.h>
+#include <vespa/searchcore/proton/server/i_operation_storer.h>
+#include <vespa/searchcore/proton/server/ipruneremoveddocumentshandler.h>
+#include <vespa/searchcore/proton/server/iheartbeathandler.h>
+#include <vespa/searchcore/proton/server/maintenance_controller_explorer.h>
+#include <vespa/searchcore/proton/server/maintenance_jobs_injector.h>
+#include <vespa/searchcore/proton/server/maintenancecontroller.h>
+#include <vespa/searchcore/proton/server/ibucketmodifiedhandler.h>
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/attribute/attributecontext.h>
+#include <vespa/searchlib/common/idocumentmetastore.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchcore/proton/test/clusterstatehandler.h>
+#include <vespa/searchcore/proton/test/buckethandler.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+
+using namespace proton;
+using namespace vespalib::slime;
+using document::BucketId;
+using document::Document;
+using document::DocumentId;
+using fastos::ClockSystem;
+using fastos::TimeStamp;
+using search::AttributeGuard;
+using search::DocumentIdT;
+using search::DocumentMetaData;
+using search::SerialNum;
+using storage::spi::BucketInfo;
+using storage::spi::Timestamp;
+using vespalib::makeTask;
+using vespalib::makeClosure;
+using vespalib::Slime;
+using proton::matching::ISessionCachePruner;
+
+typedef BucketId::List BucketIdVector;
+typedef std::set<BucketId> BucketIdSet;
+
+constexpr int TIMEOUT_MS = 60000;
+constexpr double TIMEOUT_SEC = 60.0;
+
+namespace
+{
+
+void
+sampleThreadId(FastOS_ThreadId *threadId)
+{
+    *threadId = FastOS_Thread::GetCurrentThreadId();
+}
+
+}  // namespace
+
+
+class MyDocumentSubDB
+{
+    typedef std::map<DocumentIdT, Document::SP> DocMap;
+    DocMap _docs;
+    uint32_t _subDBId;
+    DocumentMetaStore::SP _metaStoreSP;
+    DocumentMetaStore & _metaStore;
+    const document::DocumentTypeRepo::SP &_repo;
+    const DocTypeName &_docTypeName;
+
+public:
+    MyDocumentSubDB(uint32_t subDBId,
+                    SubDbType subDbType,
+                    const document::DocumentTypeRepo::SP &repo,
+                    std::shared_ptr<BucketDBOwner> bucketDB,
+                    const DocTypeName &docTypeName)
+        : _docs(),
+          _subDBId(subDBId),
+          _metaStoreSP(std::make_shared<DocumentMetaStore>(bucketDB,
+                       DocumentMetaStore::getFixedName(),
+                       search::GrowStrategy(),
+                       DocumentMetaStore::IGidCompare::SP(new DocumentMetaStore::DefaultGidCompare),
+                       subDbType)),
+          _metaStore(*_metaStoreSP),
+          _repo(repo),
+          _docTypeName(docTypeName)
+    {
+        _metaStore.constructFreeList();
+    }
+
+    uint32_t
+    getSubDBId(void) const
+    {
+        return _subDBId;
+    }
+
+    Document::UP
+    getDocument(DocumentIdT lid) const
+    {
+        DocMap::const_iterator it(_docs.find(lid));
+        if (it != _docs.end()) {
+            return Document::UP(it->second->clone());
+        } else {
+            return Document::UP();
+        }
+    }
+
+    MaintenanceDocumentSubDB
+    getSubDB(void);
+
+    void
+    handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &op);
+
+    void
+    handlePut(PutOperation &op);
+
+    void
+    handleRemove(RemoveOperation &op);
+
+    void
+    prepareMove(MoveOperation &op);
+
+    void
+    handleMove(const MoveOperation &op);
+
+    uint32_t
+    getNumUsedLids(void) const;
+
+    uint32_t
+    getDocumentCount(void) const
+    {
+        return _docs.size();
+    }
+
+    void setBucketState(const BucketId &bucket, bool active) {
+        _metaStore.setBucketState(bucket, active);
+    }
+
+    const IDocumentMetaStore &
+    getMetaStore() const
+    {
+        return _metaStore;
+    }
+};
+
+
+struct MyDocumentRetriever : public DocumentRetrieverBaseForTest
+{
+    MyDocumentSubDB &_subDB;
+
+    MyDocumentRetriever(MyDocumentSubDB &subDB)
+        : _subDB(subDB)
+    {
+    }
+
+    virtual const document::DocumentTypeRepo &
+    getDocumentTypeRepo(void) const
+    {
+        abort();
+    }
+
+    virtual void
+    getBucketMetaData(const storage::spi::Bucket &,
+                      DocumentMetaData::Vector &) const
+    {
+        abort();
+    }
+    virtual DocumentMetaData
+    getDocumentMetaData(const DocumentId &) const
+    {
+        return DocumentMetaData();
+    }
+
+    virtual Document::UP
+    getDocument(DocumentIdT lid) const
+    {
+        return _subDB.getDocument(lid);
+    }
+
+    virtual CachedSelect::SP
+    parseSelect(const vespalib::string &) const
+    {
+        return CachedSelect::SP();
+    }
+};
+
+
+struct MyBucketModifiedHandler : public IBucketModifiedHandler
+{
+    BucketIdVector _modified;
+    virtual void notifyBucketModified(const BucketId &bucket) {
+        BucketIdVector::const_iterator itr = std::find(_modified.begin(), _modified.end(), bucket);
+        _modified.push_back(bucket);
+    }
+    void reset() { _modified.clear(); }
+};
+
+
+struct MySessionCachePruner : public ISessionCachePruner
+{
+    bool isInvoked;
+    MySessionCachePruner() : isInvoked(false) { }
+    void pruneTimedOutSessions(fastos::TimeStamp current) {
+        (void) current;
+        isInvoked = true;
+    }
+};
+
+
+class MyFeedHandler : public IDocumentMoveHandler,
+                      public IPruneRemovedDocumentsHandler,
+                      public IHeartBeatHandler,
+                      public IWipeOldRemovedFieldsHandler,
+                      public IOperationStorer
+{
+    FastOS_ThreadId                _executorThreadId;
+    std::vector<MyDocumentSubDB *> _subDBs;
+    SerialNum                      _serialNum;
+    uint32_t                       _heartBeats;
+    fastos::TimeStamp              _wipeTimeLimit;
+public:
+    MyFeedHandler(FastOS_ThreadId &executorThreadId);
+
+    virtual
+    ~MyFeedHandler();
+
+    bool
+    isExecutorThread(void);
+
+    virtual void
+    handleMove(MoveOperation &op);
+
+    virtual void
+    performPruneRemovedDocuments(PruneRemovedDocumentsOperation &op);
+
+    virtual void
+    heartBeat(void);
+
+    virtual void
+    wipeOldRemovedFields(TimeStamp wipeTimeLimit);
+
+    void
+    setSubDBs(const std::vector<MyDocumentSubDB *> &subDBs);
+
+    SerialNum
+    incSerialNum(void)
+    {
+        return ++_serialNum;
+    }
+
+    // Implements IOperationStorer
+    virtual void
+    storeOperation(FeedOperation &op);
+
+    uint32_t
+    getHeartBeats(void)
+    {
+        return _heartBeats;
+    }
+
+    fastos::TimeStamp
+    getWipeTimeLimit()
+    {
+        return _wipeTimeLimit;
+    }
+};
+
+
+class MyExecutor: public vespalib::ThreadStackExecutor
+{
+public:
+    FastOS_ThreadId		  _threadId;
+
+    MyExecutor(void);
+
+    virtual
+    ~MyExecutor(void);
+
+    bool
+    isIdle(void);
+
+    bool
+    waitIdle(double timeout);
+};
+
+
+class MyFrozenBucket
+{
+    IBucketFreezer &_freezer;
+    BucketId _bucketId;
+public:
+    typedef std::unique_ptr<MyFrozenBucket> UP;
+
+    MyFrozenBucket(IBucketFreezer &freezer,
+                   const BucketId &bucketId)
+        : _freezer(freezer),
+          _bucketId(bucketId)
+    {
+        _freezer.freezeBucket(_bucketId);
+    }
+
+    ~MyFrozenBucket(void)
+    {
+        _freezer.thawBucket(_bucketId);
+    }
+};
+
+struct MySimpleJob : public IMaintenanceJob
+{
+    vespalib::CountDownLatch _latch;
+    size_t                   _runCnt;
+
+    MySimpleJob(double delay,
+                double interval,
+                uint32_t finishCount)
+        : IMaintenanceJob("my_job", delay, interval),
+          _latch(finishCount),
+          _runCnt(0)
+    {
+    }
+    void block() { setBlocked(true); }
+    virtual bool run() {
+        LOG(info, "MySimpleJob::run()");
+        _latch.countDown();
+        ++_runCnt;
+        return true;
+    }
+};
+
+struct MySplitJob : public MySimpleJob
+{
+    MySplitJob(double delay,
+               double interval,
+               uint32_t finishCount)
+        : MySimpleJob(delay, interval, finishCount)
+    {
+    }
+    virtual bool run() {
+        LOG(info, "MySplitJob::run()");
+        _latch.countDown();
+        ++_runCnt;
+        return _latch.getCount() == 0;
+    }
+};
+
+struct MyLongRunningJob : public IMaintenanceJob
+{
+    vespalib::Gate _firstRun;
+
+    MyLongRunningJob(double delay,
+                     double interval)
+        : IMaintenanceJob("long_running_job", delay, interval),
+          _firstRun()
+    {
+    }
+    void block() { setBlocked(true); }
+    virtual bool run() {
+        _firstRun.countDown();
+        usleep(10000);
+        return false;
+    }
+};
+
+
+struct MyAttributeManager : public proton::IAttributeManager
+{
+    virtual AttributeGuard::UP
+    getAttribute(const string &) const override {
+        abort();
+    }
+
+    virtual AttributeGuard::UP
+    getAttributeStableEnum(const string &) const override {
+        abort();
+    }
+
+    virtual void
+    getAttributeList(std::vector<AttributeGuard> &) const override {
+        abort();
+    }
+
+    virtual search::attribute::IAttributeContext::UP
+    createContext() const override {
+        abort();
+    }
+
+    virtual IAttributeManager::SP
+    create(const AttributeCollectionSpec &) const override {
+        abort();
+    }
+
+    virtual std::vector<searchcorespi::IFlushTarget::SP>
+    getFlushTargets() const override {
+        abort();
+    }
+
+    virtual search::SerialNum
+    getFlushedSerialNum(const vespalib::string &) const override {
+        abort();
+    }
+
+    virtual search::SerialNum getOldestFlushedSerialNumber() const override {
+        abort();
+    }
+
+    virtual search::SerialNum
+    getNewestFlushedSerialNumber() const override {
+        abort();
+    }
+
+    virtual void
+    getAttributeListAll(std::vector<search::AttributeGuard> &)
+        const override {
+        abort();
+    }
+
+    virtual void
+    wipeHistory(const search::index::Schema &) override {
+        abort();
+    }
+
+    virtual const IAttributeFactory::SP &
+    getFactory() const override {
+        abort();
+    }
+
+    virtual search::ISequencedTaskExecutor &
+    getAttributeFieldWriter() const override {
+        abort();
+    }
+
+    virtual search::AttributeVector *
+    getWritableAttribute(const vespalib::string &) const override {
+        abort();
+    }
+
+    virtual const std::vector<search::AttributeVector *> &
+    getWritableAttributes() const override {
+        abort();
+    }
+
+    virtual void
+    asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>)
+        const override {
+    }
+
+    virtual ExclusiveAttributeReadAccessor::UP
+    getExclusiveReadAccessor(const vespalib::string &) const override {
+        abort();
+    }
+};
+
+class MaintenanceControllerFixture : public ICommitable
+{
+public:
+    MyExecutor                    _executor;
+    ExecutorThreadService         _threadService;
+    DocTypeName                   _docTypeName;
+    test::UserDocumentsBuilder    _builder;
+    std::shared_ptr<BucketDBOwner> _bucketDB;
+    test::BucketStateCalculator::SP _calc;
+    test::ClusterStateHandler     _clusterStateHandler;
+    test::BucketHandler           _bucketHandler;
+    MyBucketModifiedHandler       _bmc;
+    MyDocumentSubDB               _ready;
+    MyDocumentSubDB               _removed;
+    MyDocumentSubDB               _notReady;
+    MySessionCachePruner          _gsp;
+    MyFeedHandler                 _fh;
+    DocumentDBMaintenanceConfig::SP _mcCfg;
+    bool                          _injectDefaultJobs;
+    DocumentDBJobTrackers         _jobTrackers;
+    std::shared_ptr<proton::IAttributeManager> _readyAttributeManager;
+    std::shared_ptr<proton::IAttributeManager> _notReadyAttributeManager;
+    AttributeUsageFilter          _attributeUsageFilter;
+    MaintenanceController         _mc;
+
+    MaintenanceControllerFixture(void);
+
+    virtual
+    ~MaintenanceControllerFixture(void);
+
+    void
+    syncSubDBs(void);
+
+    void commit() override {
+    }
+
+    void commitAndWait() override {
+    }
+
+    void
+    performSyncSubDBs(void);
+
+    void
+    notifyClusterStateChanged(void);
+
+    void
+    performNotifyClusterStateChanged(void);
+
+    void
+    startMaintenance(void);
+
+    void injectMaintenanceJobs();
+
+    void
+    performStartMaintenance(void);
+
+    void
+    stopMaintenance(void);
+
+    void
+    forwardMaintenanceConfig(void);
+
+    void
+    performForwardMaintenanceConfig(void);
+
+    void
+    insertDocs(const test::UserDocuments &docs,
+               MyDocumentSubDB &subDb);
+
+    void
+    removeDocs(const test::UserDocuments &docs,
+               Timestamp timestamp);
+
+    void
+    setPruneConfig(const DocumentDBPruneRemovedDocumentsConfig &pruneConfig)
+    {
+        DocumentDBMaintenanceConfig::SP
+            newCfg(new DocumentDBMaintenanceConfig(
+                           pruneConfig,
+                           _mcCfg->getHeartBeatConfig(),
+                           _mcCfg->getWipeOldRemovedFieldsConfig(),
+                           _mcCfg->getSessionCachePruneInterval(),
+                           _mcCfg->getVisibilityDelay(),
+                           _mcCfg->getLidSpaceCompactionConfig(),
+                           _mcCfg->getAttributeUsageFilterConfig(),
+                           _mcCfg->getAttributeUsageSampleInterval()));
+        _mcCfg = newCfg;
+        forwardMaintenanceConfig();
+    }
+
+    void
+    setHeartBeatConfig(const DocumentDBHeartBeatConfig &heartBeatConfig)
+    {
+        DocumentDBMaintenanceConfig::SP
+            newCfg(new DocumentDBMaintenanceConfig(
+                           _mcCfg->getPruneRemovedDocumentsConfig(),
+                           heartBeatConfig,
+                           _mcCfg->getWipeOldRemovedFieldsConfig(),
+                           _mcCfg->getSessionCachePruneInterval(),
+                           _mcCfg->getVisibilityDelay(),
+                           _mcCfg->getLidSpaceCompactionConfig(),
+                           _mcCfg->getAttributeUsageFilterConfig(),
+                           _mcCfg->getAttributeUsageSampleInterval()));
+        _mcCfg = newCfg;
+        forwardMaintenanceConfig();
+    }
+
+    void
+    setWipeOldRemovedFieldsConfig(const DocumentDBWipeOldRemovedFieldsConfig &wipeConfig)
+    {
+        DocumentDBMaintenanceConfig::SP
+            newCfg(new DocumentDBMaintenanceConfig(
+                           _mcCfg->getPruneRemovedDocumentsConfig(),
+                           _mcCfg->getHeartBeatConfig(),
+                           wipeConfig,
+                           _mcCfg->getSessionCachePruneInterval(),
+                           _mcCfg->getVisibilityDelay(),
+                           _mcCfg->getLidSpaceCompactionConfig(),
+                           _mcCfg->getAttributeUsageFilterConfig(),
+                           _mcCfg->getAttributeUsageSampleInterval()));
+        _mcCfg = newCfg;
+        forwardMaintenanceConfig();
+    }
+
+
+    void
+    setGroupingSessionPruneInterval(double groupingSessionPruneInterval)
+    {
+        DocumentDBMaintenanceConfig::SP
+            newCfg(new DocumentDBMaintenanceConfig(
+                           _mcCfg->getPruneRemovedDocumentsConfig(),
+                           _mcCfg->getHeartBeatConfig(),
+                           _mcCfg->getWipeOldRemovedFieldsConfig(),
+                           groupingSessionPruneInterval,
+                           _mcCfg->getVisibilityDelay(),
+                           _mcCfg->getLidSpaceCompactionConfig(),
+                           _mcCfg->getAttributeUsageFilterConfig(),
+                           _mcCfg->getAttributeUsageSampleInterval()));
+        _mcCfg = newCfg;
+        forwardMaintenanceConfig();
+    }
+
+
+    void
+    performNotifyBucketStateChanged(document::BucketId bucketId,
+                                    BucketInfo::ActiveState newState)
+    {
+        _bucketHandler.notifyBucketStateChanged(bucketId, newState);
+    }
+
+    void
+    notifyBucketStateChanged(const document::BucketId &bucketId,
+                             BucketInfo::ActiveState newState)
+    {
+        _executor.execute(makeTask(makeClosure(this,
+                                               &MaintenanceControllerFixture::
+                                               performNotifyBucketStateChanged,
+                                               bucketId, newState)));
+        _executor.sync();
+    }
+};
+
+
+MaintenanceDocumentSubDB
+MyDocumentSubDB::getSubDB(void)
+{
+    IDocumentRetriever::SP retriever(new MyDocumentRetriever(*this));
+
+    return MaintenanceDocumentSubDB(_metaStoreSP,
+                                    retriever,
+                                    _subDBId);
+}
+
+
+void
+MyDocumentSubDB::handlePruneRemovedDocuments(
+        const PruneRemovedDocumentsOperation &op)
+{
+    assert(_subDBId == 1u);
+    typedef LidVectorContext::LidVector LidVector;
+    const SerialNum serialNum = op.getSerialNum();
+    const LidVectorContext &lidCtx = *op.getLidsToRemove();
+    const LidVector &lidsToRemove(lidCtx.getLidVector());
+    _metaStore.removeBatch(lidsToRemove, lidCtx.getDocIdLimit());
+    _metaStore.removeBatchComplete(lidsToRemove);
+    _metaStore.commit(serialNum);
+    for (LidVector::const_iterator it = lidsToRemove.begin(),
+                                  ite = lidsToRemove.end();
+         it != ite; ++it) {
+        search::DocumentIdT lid(*it);
+        _docs.erase(lid);
+    }
+}
+
+
+void
+MyDocumentSubDB::handlePut(PutOperation &op)
+{
+    const SerialNum serialNum = op.getSerialNum();
+    const Document::SP &doc = op.getDocument();
+    const DocumentId &docId = doc->getId();
+    const document::GlobalId &gid = docId.getGlobalId();
+    bool needCommit = false;
+
+    if (op.getValidDbdId(_subDBId)) {
+        typedef DocumentMetaStore::Result PutRes;
+
+        PutRes putRes(_metaStore.put(gid,
+                                     op.getBucketId(),
+                                     op.getTimestamp(),
+                                     op.getLid()));
+        assert(putRes.ok());
+        assert(op.getLid() == putRes._lid);
+        _docs[op.getLid()] = doc;
+        needCommit = true;
+    }
+    if (op.getValidPrevDbdId(_subDBId) && op.changedDbdId()) {
+        assert(_metaStore.validLid(op.getPrevLid()));
+        const RawDocumentMetaData &meta(_metaStore.getRawMetaData(op.getPrevLid()));
+        assert((_subDBId == 1u) == op.getPrevMarkedAsRemoved());
+        assert(meta.getGid() == gid);
+        (void) meta;
+
+        bool remres = _metaStore.remove(op.getPrevLid());
+        assert(remres);
+        (void) remres;
+        _metaStore.removeComplete(op.getPrevLid());
+
+        _docs.erase(op.getPrevLid());
+        needCommit = true;
+    }
+    if (needCommit) {
+        _metaStore.commit(serialNum, serialNum);
+    }
+}
+
+
+void
+MyDocumentSubDB::handleRemove(RemoveOperation &op)
+{
+    const SerialNum serialNum = op.getSerialNum();
+    const DocumentId &docId = op.getDocumentId();
+    const document::GlobalId &gid = docId.getGlobalId();
+    bool needCommit = false;
+
+    if (op.getValidDbdId(_subDBId)) {
+        typedef DocumentMetaStore::Result PutRes;
+
+        PutRes putRes(_metaStore.put(gid,
+                                     op.getBucketId(),
+                                     op.getTimestamp(),
+                                     op.getLid()));
+        assert(putRes.ok());
+        assert(op.getLid() == putRes._lid);
+        const document::DocumentType *docType =
+            _repo->getDocumentType(_docTypeName.getName());
+        Document::UP doc(new Document(*docType, docId));
+        doc->setRepo(*_repo);
+        _docs[op.getLid()] = std::move(doc);
+        needCommit = true;
+    }
+    if (op.getValidPrevDbdId(_subDBId) && op.changedDbdId()) {
+        assert(_metaStore.validLid(op.getPrevLid()));
+        const RawDocumentMetaData &meta(_metaStore.getRawMetaData(op.getPrevLid()));
+        assert((_subDBId == 1u) == op.getPrevMarkedAsRemoved());
+        assert(meta.getGid() == gid);
+        (void) meta;
+
+        bool remres = _metaStore.remove(op.getPrevLid());
+        assert(remres);
+        (void) remres;
+
+        _metaStore.removeComplete(op.getPrevLid());
+        _docs.erase(op.getPrevLid());
+        needCommit = true;
+    }
+    if (needCommit) {
+        _metaStore.commit(serialNum, serialNum);
+    }
+}
+
+
+void
+MyDocumentSubDB::prepareMove(MoveOperation &op)
+{
+    const DocumentId &docId = op.getDocument()->getId();
+    const document::GlobalId &gid = docId.getGlobalId();
+    DocumentMetaStore::Result inspectResult = _metaStore.inspect(gid);
+    assert(!inspectResult._found);
+    op.setDbDocumentId(DbDocumentId(_subDBId, inspectResult._lid));
+}
+
+
+void
+MyDocumentSubDB::handleMove(const MoveOperation &op)
+{
+    const SerialNum serialNum = op.getSerialNum();
+    const Document::SP &doc = op.getDocument();
+    const DocumentId &docId = doc->getId();
+    const document::GlobalId &gid = docId.getGlobalId();
+    bool needCommit = false;
+
+    if (op.getValidDbdId(_subDBId)) {
+        typedef DocumentMetaStore::Result PutRes;
+
+        PutRes putRes(_metaStore.put(gid,
+                                     op.getBucketId(),
+                                     op.getTimestamp(),
+                                     op.getLid()));
+        assert(putRes.ok());
+        assert(op.getLid() == putRes._lid);
+        _docs[op.getLid()] = doc;
+        needCommit = true;
+    }
+    if (op.getValidPrevDbdId(_subDBId)) {
+        assert(_metaStore.validLid(op.getPrevLid()));
+        const RawDocumentMetaData &meta(_metaStore.getRawMetaData(op.getPrevLid()));
+        assert((_subDBId == 1u) == op.getPrevMarkedAsRemoved());
+        assert(meta.getGid() == gid);
+        (void) meta;
+
+        bool remres = _metaStore.remove(op.getPrevLid());
+        assert(remres);
+        (void) remres;
+
+        _metaStore.removeComplete(op.getPrevLid());
+        _docs.erase(op.getPrevLid());
+        needCommit = true;
+    }
+    if (needCommit) {
+        _metaStore.commit(serialNum, serialNum);
+    }
+}
+
+
+uint32_t
+MyDocumentSubDB::getNumUsedLids(void) const
+{
+    return _metaStore.getNumUsedLids();
+}
+
+
+MyFeedHandler::MyFeedHandler(FastOS_ThreadId &executorThreadId)
+    : IDocumentMoveHandler(),
+      IPruneRemovedDocumentsHandler(),
+      IHeartBeatHandler(),
+      _executorThreadId(executorThreadId),
+      _subDBs(),
+      _serialNum(0u),
+      _heartBeats(0u),
+      _wipeTimeLimit()
+{
+}
+
+
+MyFeedHandler::~MyFeedHandler(void)
+{
+}
+
+
+bool
+MyFeedHandler::isExecutorThread(void)
+{
+    FastOS_ThreadId threadId(FastOS_Thread::GetCurrentThreadId());
+    return FastOS_Thread::CompareThreadIds(_executorThreadId, threadId);
+}
+
+
+void
+MyFeedHandler::handleMove(MoveOperation &op)
+{
+    assert(isExecutorThread());
+    assert(op.getValidPrevDbdId());
+    _subDBs[op.getSubDbId()]->prepareMove(op);
+    assert(op.getValidDbdId());
+    assert(op.getSubDbId() != op.getPrevSubDbId());
+    // Check for wrong magic numbers
+    assert(op.getSubDbId() != 1u);
+    assert(op.getPrevSubDbId() != 1u);
+    assert(op.getSubDbId() < _subDBs.size());
+    assert(op.getPrevSubDbId() < _subDBs.size());
+    storeOperation(op);
+    _subDBs[op.getSubDbId()]->handleMove(op);
+    _subDBs[op.getPrevSubDbId()]->handleMove(op);
+}
+
+
+void
+MyFeedHandler::performPruneRemovedDocuments(PruneRemovedDocumentsOperation &op)
+{
+    assert(isExecutorThread());
+    if (op.getLidsToRemove()->getNumLids() != 0u) {
+        storeOperation(op);
+        // magic number.
+        _subDBs[1u]->handlePruneRemovedDocuments(op);
+    }
+}
+
+
+void
+MyFeedHandler::heartBeat(void)
+{
+    assert(isExecutorThread());
+    ++_heartBeats;
+}
+
+
+void
+MyFeedHandler::wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit)
+{
+    assert(isExecutorThread());
+    _wipeTimeLimit = wipeTimeLimit;
+}
+
+
+void
+MyFeedHandler::setSubDBs(const std::vector<MyDocumentSubDB *> &subDBs)
+{
+    _subDBs = subDBs;
+}
+
+
+void
+MyFeedHandler::storeOperation(FeedOperation &op)
+{
+    op.setSerialNum(incSerialNum());
+}
+
+
+MyExecutor::MyExecutor(void)
+    : vespalib::ThreadStackExecutor(1, 128 * 1024),
+      _threadId()
+{
+    execute(makeTask(makeClosure(&sampleThreadId, &_threadId)));
+    sync();
+}
+
+
+MyExecutor::~MyExecutor(void)
+{
+}
+
+
+bool
+MyExecutor::isIdle(void)
+{
+    (void) getStats();
+    sync();
+    Stats stats(getStats());
+    return stats.acceptedTasks == 0u;
+}
+
+
+bool
+MyExecutor::waitIdle(double timeout)
+{
+    FastOS_Time startTime;
+    startTime.SetNow();
+    while (!isIdle()) {
+        FastOS_Time cTime;
+        cTime.SetNow();
+        if (cTime.Secs() - startTime.Secs() >= timeout)
+            return false;
+    }
+    return true;
+}
+
+
+MaintenanceControllerFixture::MaintenanceControllerFixture(void)
+    : _executor(),
+      _threadService(_executor),
+      _docTypeName("searchdocument"), // must match document builder
+      _builder(),
+      _bucketDB(std::make_shared<BucketDBOwner>()),
+      _calc(new test::BucketStateCalculator()),
+      _clusterStateHandler(),
+      _bucketHandler(),
+      _bmc(),
+      _ready(0u, SubDbType::READY, _builder.getRepo(), _bucketDB, _docTypeName),
+      _removed(1u, SubDbType::REMOVED, _builder.getRepo(), _bucketDB,
+               _docTypeName),
+      _notReady(2u, SubDbType::NOTREADY, _builder.getRepo(), _bucketDB,
+                _docTypeName),
+      _gsp(),
+      _fh(_executor._threadId),
+      _mcCfg(new DocumentDBMaintenanceConfig),
+      _injectDefaultJobs(true),
+      _jobTrackers(),
+      _readyAttributeManager(std::make_shared<MyAttributeManager>()),
+      _notReadyAttributeManager(std::make_shared<MyAttributeManager>()),
+      _attributeUsageFilter(),
+      _mc(_threadService, _docTypeName)
+{
+    std::vector<MyDocumentSubDB *> subDBs;
+    subDBs.push_back(&_ready);
+    subDBs.push_back(&_removed);
+    subDBs.push_back(&_notReady);
+    _fh.setSubDBs(subDBs);
+    syncSubDBs();
+}
+
+
+MaintenanceControllerFixture::~MaintenanceControllerFixture(void)
+{
+    stopMaintenance();
+}
+
+
+void
+MaintenanceControllerFixture::syncSubDBs(void)
+{
+    _executor.execute(makeTask(makeClosure(this,
+                                       &MaintenanceControllerFixture::
+                                       performSyncSubDBs)));
+    _executor.sync();
+}
+
+
+void
+MaintenanceControllerFixture::performSyncSubDBs(void)
+{
+    _mc.syncSubDBs(_ready.getSubDB(),
+                   _removed.getSubDB(),
+                   _notReady.getSubDB());
+}
+
+
+void
+MaintenanceControllerFixture::notifyClusterStateChanged(void)
+{
+    _executor.execute(makeTask(makeClosure(this,
+                                       &MaintenanceControllerFixture::
+                                       performNotifyClusterStateChanged)));
+    _executor.sync();
+}
+
+
+void
+MaintenanceControllerFixture::performNotifyClusterStateChanged(void)
+{
+    _clusterStateHandler.notifyClusterStateChanged(_calc);
+}
+
+
+void
+MaintenanceControllerFixture::startMaintenance(void)
+{
+    _executor.execute(makeTask(makeClosure(this,
+                                       &MaintenanceControllerFixture::
+                                       performStartMaintenance)));
+    _executor.sync();
+}
+
+void
+MaintenanceControllerFixture::injectMaintenanceJobs()
+{
+    if (_injectDefaultJobs) {
+        ILidSpaceCompactionHandler::Vector lscHandlers;
+        MaintenanceJobsInjector::injectJobs(_mc, *_mcCfg, _fh, _gsp, _fh,
+                                            lscHandlers, _fh, _mc, _docTypeName.getName(),
+                                            _fh, _fh, _bmc, _clusterStateHandler, _bucketHandler,
+                                            _calc, _jobTrackers, *this,
+                                            _readyAttributeManager,
+                                            _notReadyAttributeManager,
+                                            _attributeUsageFilter);
+    }
+}
+
+void
+MaintenanceControllerFixture::performStartMaintenance(void)
+{
+    injectMaintenanceJobs();
+    _mc.start(_mcCfg);
+}
+
+
+void
+MaintenanceControllerFixture::stopMaintenance(void)
+{
+    _mc.stop();
+    _executor.sync();
+}
+
+
+void
+MaintenanceControllerFixture::forwardMaintenanceConfig(void)
+{
+    _executor.execute(makeTask(makeClosure(this,
+                                       &MaintenanceControllerFixture::
+                                       performForwardMaintenanceConfig)));
+    _executor.sync();
+}
+
+
+void
+MaintenanceControllerFixture::performForwardMaintenanceConfig(void)
+{
+    _mc.killJobs();
+    injectMaintenanceJobs();
+    _mc.newConfig(_mcCfg);
+}
+
+
+void
+MaintenanceControllerFixture::insertDocs(const test::UserDocuments &docs,
+        MyDocumentSubDB &subDb)
+{
+
+    for (test::UserDocuments::Iterator itr = docs.begin();
+         itr != docs.end();
+         ++itr) {
+        const test::BucketDocuments &bucketDocs = itr->second;
+        for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) {
+            const test::Document &testDoc = bucketDocs.getDocs()[i];
+            PutOperation op(testDoc.getBucket(),
+                            testDoc.getTimestamp(),
+                            testDoc.getDoc());
+            op.setDbDocumentId(DbDocumentId(subDb.getSubDBId(),
+                                       testDoc.getLid()));
+            _fh.storeOperation(op);
+            subDb.handlePut(op);
+        }
+    }
+}
+
+
+void
+MaintenanceControllerFixture::removeDocs(const test::UserDocuments &docs,
+        Timestamp timestamp)
+{
+
+    for (test::UserDocuments::Iterator itr = docs.begin();
+         itr != docs.end();
+         ++itr) {
+        const test::BucketDocuments &bucketDocs = itr->second;
+        for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) {
+            const test::Document &testDoc = bucketDocs.getDocs()[i];
+            RemoveOperation op(testDoc.getBucket(),
+                            timestamp,
+                            testDoc.getDoc()->getId());
+            op.setDbDocumentId(DbDocumentId(_removed.getSubDBId(),
+                                       testDoc.getLid()));
+            _fh.storeOperation(op);
+            _removed.handleRemove(op);
+        }
+    }
+}
+
+TEST_F("require that bucket move controller is active",
+       MaintenanceControllerFixture)
+{
+    f._builder.createDocs(1, 1, 4); // 3 docs
+    f._builder.createDocs(2, 4, 6); // 2 docs
+    test::UserDocuments readyDocs(f._builder.getDocs());
+    BucketId bucketId1(readyDocs.getBucket(1));
+    BucketId bucketId2(readyDocs.getBucket(2));
+    f.insertDocs(readyDocs, f._ready);
+    f._builder.clearDocs();
+    f._builder.createDocs(3, 1, 3); // 2 docs
+    f._builder.createDocs(4, 3, 6); // 3 docs
+    test::UserDocuments notReadyDocs(f._builder.getDocs());
+    BucketId bucketId3(notReadyDocs.getBucket(3));
+    BucketId bucketId4(notReadyDocs.getBucket(4));
+    f.insertDocs(notReadyDocs, f._notReady);
+    f._builder.clearDocs();
+    f.notifyClusterStateChanged();
+    EXPECT_TRUE(f._executor.isIdle());
+    EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
+    f.startMaintenance();
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(0u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(0u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(10u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(10u, f._notReady.getDocumentCount());
+    f._calc->addReady(bucketId1);
+    f.notifyClusterStateChanged();
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(3u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(3u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(7u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(7u, f._notReady.getDocumentCount());
+    MyFrozenBucket::UP frozen2(new MyFrozenBucket(f._mc, bucketId2));
+    f._calc->addReady(bucketId2);
+    f._calc->addReady(bucketId4);
+    f.notifyClusterStateChanged();
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(6u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(6u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(4u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(4u, f._notReady.getDocumentCount());
+    frozen2.reset();
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(8u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(8u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(2u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(2u, f._notReady.getDocumentCount());
+}
+
+TEST_F("require that document pruner is active",
+       MaintenanceControllerFixture)
+{
+    uint64_t tshz = 1000000;
+    uint64_t now = static_cast<uint64_t>(time(0)) * tshz;
+    Timestamp remTime(static_cast<Timestamp::Type>(now - 3600 * tshz));
+    Timestamp keepTime(static_cast<Timestamp::Type>(now + 3600 * tshz));
+    f._builder.createDocs(1, 1, 4); // 3 docs
+    f._builder.createDocs(2, 4, 6); // 2 docs
+    test::UserDocuments keepDocs(f._builder.getDocs());
+    BucketId bucketId1(keepDocs.getBucket(1));
+    BucketId bucketId2(keepDocs.getBucket(2));
+    f.removeDocs(keepDocs, keepTime);
+    f._builder.clearDocs();
+    f._builder.createDocs(3, 6, 8); // 2 docs
+    f._builder.createDocs(4, 8, 11); // 3 docs
+    test::UserDocuments removeDocs(f._builder.getDocs());
+    BucketId bucketId3(removeDocs.getBucket(3));
+    BucketId bucketId4(removeDocs.getBucket(4));
+    f.removeDocs(removeDocs, remTime);
+    f.notifyClusterStateChanged();
+    EXPECT_TRUE(f._executor.isIdle());
+    EXPECT_EQUAL(10u, f._removed.getNumUsedLids());
+    EXPECT_EQUAL(10u, f._removed.getDocumentCount());
+    f.startMaintenance();
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(10u, f._removed.getNumUsedLids());
+    EXPECT_EQUAL(10u, f._removed.getDocumentCount());
+    MyFrozenBucket::UP frozen3(new MyFrozenBucket(f._mc, bucketId3));
+    f.setPruneConfig(DocumentDBPruneRemovedDocumentsConfig(0.2, 900.0));
+    for (uint32_t i = 0; i < 6; ++i) {
+        FastOS_Thread::Sleep(100);
+        ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+        if (f._removed.getNumUsedLids() != 10u)
+            break;
+    }
+    EXPECT_EQUAL(10u, f._removed.getNumUsedLids());
+    EXPECT_EQUAL(10u, f._removed.getDocumentCount());
+    frozen3.reset();
+    for (uint32_t i = 0; i < 600; ++i) {
+        FastOS_Thread::Sleep(100);
+        ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+        if (f._removed.getNumUsedLids() != 10u)
+            break;
+    }
+    EXPECT_EQUAL(5u, f._removed.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._removed.getDocumentCount());
+}
+
+TEST_F("require that heartbeats are scheduled",
+       MaintenanceControllerFixture)
+{
+    f.notifyClusterStateChanged();
+    f.startMaintenance();
+    f.setHeartBeatConfig(DocumentDBHeartBeatConfig(0.2));
+    for (uint32_t i = 0; i < 600; ++i) {
+        FastOS_Thread::Sleep(100);
+        if (f._fh.getHeartBeats() != 0u)
+            break;
+    }
+    EXPECT_GREATER(f._fh.getHeartBeats(), 0u);
+}
+
+TEST_F("require that periodic session prunings are scheduled",
+       MaintenanceControllerFixture)
+{
+    ASSERT_FALSE(f._gsp.isInvoked);
+    f.notifyClusterStateChanged();
+    f.startMaintenance();
+    f.setGroupingSessionPruneInterval(0.2);
+    for (uint32_t i = 0; i < 600; ++i) {
+        FastOS_Thread::Sleep(100);
+        if (f._gsp.isInvoked) {
+            break;
+        }
+    }
+    ASSERT_TRUE(f._gsp.isInvoked);
+}
+
+TEST_F("require that wipe old removed fields are scheduled",
+       MaintenanceControllerFixture)
+{
+    f.notifyClusterStateChanged();
+    f.startMaintenance();
+    TimeStamp now0 = TimeStamp(ClockSystem::now());
+    f.setWipeOldRemovedFieldsConfig(DocumentDBWipeOldRemovedFieldsConfig(0.2, 100));
+    TimeStamp now = TimeStamp(ClockSystem::now());
+    TimeStamp expWipeTimeLimit = now - TimeStamp(100 * TimeStamp::SEC);
+    TimeStamp wtLim;
+    for (uint32_t i = 0; i < 600; ++i) {
+        FastOS_Thread::Sleep(100);
+        wtLim = f._fh.getWipeTimeLimit();
+        if (wtLim.sec() != 0u) {
+            break;
+        }
+    }
+    TimeStamp now1 = TimeStamp(ClockSystem::now());
+    double fuzz = now1.sec() - now0.sec();
+    LOG(info,
+        "WipeOldRemovedFields: "
+        "now(%" PRIu64 "), "
+        "expWipeTimeLimit(%" PRIu64 "), "
+        "actWipeTimeLimit(%" PRIu64 "), "
+        "fuzz(%05.3f)",
+        (uint64_t)now.sec(),
+        (uint64_t)expWipeTimeLimit.sec(),
+        (uint64_t)wtLim.sec(),
+        fuzz);
+    EXPECT_APPROX(expWipeTimeLimit.sec(), wtLim.sec(), 4u + fuzz);
+}
+
+TEST_F("require that active bucket is not moved until de-activated", MaintenanceControllerFixture)
+{
+    f._builder.createDocs(1, 1, 4); // 3 docs
+    f._builder.createDocs(2, 4, 6); // 2 docs
+    test::UserDocuments readyDocs(f._builder.getDocs());
+    f.insertDocs(readyDocs, f._ready);
+    f._builder.clearDocs();
+    f._builder.createDocs(3, 1, 3); // 2 docs
+    f._builder.createDocs(4, 3, 6); // 3 docs
+    test::UserDocuments notReadyDocs(f._builder.getDocs());
+    f.insertDocs(notReadyDocs, f._notReady);
+    f._builder.clearDocs();
+
+    // bucket 1 (active) should be moved from ready to not ready according to cluster state
+    f._calc->addReady(readyDocs.getBucket(2));
+    f._ready.setBucketState(readyDocs.getBucket(1), true);
+
+    f.notifyClusterStateChanged();
+    EXPECT_TRUE(f._executor.isIdle());
+    EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
+
+    f.startMaintenance();
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
+
+    // de-activate bucket 1
+    f._ready.setBucketState(readyDocs.getBucket(1), false);
+    f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::NOT_ACTIVE);
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(2u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(2u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(8u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(8u, f._notReady.getDocumentCount());
+
+    // re-activate bucket 1
+    f._ready.setBucketState(readyDocs.getBucket(1), true);
+    f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::ACTIVE);
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
+
+    // de-activate bucket 1
+    f._ready.setBucketState(readyDocs.getBucket(1), false);
+    f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::NOT_ACTIVE);
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(2u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(2u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(8u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(8u, f._notReady.getDocumentCount());
+
+    // re-activate bucket 1
+    f._ready.setBucketState(readyDocs.getBucket(1), true);
+    f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::ACTIVE);
+    ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC));
+    EXPECT_EQUAL(5u, f._ready.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._ready.getDocumentCount());
+    EXPECT_EQUAL(5u, f._notReady.getNumUsedLids());
+    EXPECT_EQUAL(5u, f._notReady.getDocumentCount());
+}
+
+TEST_F("require that a simple maintenance job is executed", MaintenanceControllerFixture)
+{
+    IMaintenanceJob::UP job(new MySimpleJob(0.2, 0.2, 3));
+    MySimpleJob &myJob = static_cast<MySimpleJob &>(*job);
+    f._mc.registerJob(std::move(job));
+    f._injectDefaultJobs = false;
+    f.startMaintenance();
+    bool done = myJob._latch.await(TIMEOUT_MS);
+    EXPECT_TRUE(done);
+    EXPECT_EQUAL(0u, myJob._latch.getCount());
+}
+
+TEST_F("require that a split maintenance job is executed", MaintenanceControllerFixture)
+{
+    IMaintenanceJob::UP job(new MySplitJob(0.2, TIMEOUT_SEC * 2, 3));
+    MySplitJob &myJob = static_cast<MySplitJob &>(*job);
+    f._mc.registerJob(std::move(job));
+    f._injectDefaultJobs = false;
+    f.startMaintenance();
+    bool done = myJob._latch.await(TIMEOUT_MS);
+    EXPECT_TRUE(done);
+    EXPECT_EQUAL(0u, myJob._latch.getCount());
+}
+
+TEST_F("require that a blocked job is unblocked and executed after thaw bucket",
+        MaintenanceControllerFixture)
+{
+    IMaintenanceJob::UP job1(new MySimpleJob(TIMEOUT_SEC * 2, TIMEOUT_SEC * 2, 1));
+    MySimpleJob &myJob1 = static_cast<MySimpleJob &>(*job1);
+    IMaintenanceJob::UP job2(new MySimpleJob(TIMEOUT_SEC * 2, TIMEOUT_SEC * 2, 0));
+    MySimpleJob &myJob2 = static_cast<MySimpleJob &>(*job2);
+    f._mc.registerJob(std::move(job1));
+    f._mc.registerJob(std::move(job2));
+    f._injectDefaultJobs = false;
+    f.startMaintenance();
+
+    myJob1.block();
+    EXPECT_TRUE(myJob1.isBlocked());
+    EXPECT_FALSE(myJob2.isBlocked());
+    IBucketFreezer &ibf = f._mc;
+    ibf.freezeBucket(BucketId(1));
+    ibf.thawBucket(BucketId(1));
+    EXPECT_TRUE(myJob1.isBlocked());
+    ibf.freezeBucket(BucketId(1));
+    IFrozenBucketHandler & fbh = f._mc;
+    // This is to simulate contention, as that is required for notification on thawed buckets.
+    EXPECT_FALSE(fbh.acquireExclusiveBucket(BucketId(1)));
+    ibf.thawBucket(BucketId(1));
+    f._executor.sync();
+    EXPECT_FALSE(myJob1.isBlocked());
+    EXPECT_FALSE(myJob2.isBlocked());
+    bool done1 = myJob1._latch.await(TIMEOUT_MS);
+    EXPECT_TRUE(done1);
+    FastOS_Thread::Sleep(2000);
+    EXPECT_EQUAL(0u, myJob2._runCnt);
+}
+
+TEST_F("require that blocked jobs are not executed", MaintenanceControllerFixture)
+{
+    IMaintenanceJob::UP job(new MySimpleJob(0.2, 0.2, 0));
+    MySimpleJob &myJob = static_cast<MySimpleJob &>(*job);
+    myJob.block();
+    f._mc.registerJob(std::move(job));
+    f._injectDefaultJobs = false;
+    f.startMaintenance();
+    FastOS_Thread::Sleep(2000);
+    EXPECT_EQUAL(0u, myJob._runCnt);
+}
+
+TEST_F("require that maintenance controller state list jobs", MaintenanceControllerFixture)
+{
+    {
+        IMaintenanceJob::UP job1(new MySimpleJob(TIMEOUT_SEC * 2, TIMEOUT_SEC * 2, 0));
+        IMaintenanceJob::UP job2(new MyLongRunningJob(0.2, 0.2));
+        MyLongRunningJob &longRunningJob = static_cast<MyLongRunningJob &>(*job2);
+        f._mc.registerJob(std::move(job1));
+        f._mc.registerJob(std::move(job2));
+        f._injectDefaultJobs = false;
+        f.startMaintenance();
+        longRunningJob._firstRun.await(TIMEOUT_MS);
+    }
+
+    MaintenanceControllerExplorer explorer(f._mc.getJobList());
+    Slime state;
+    SlimeInserter inserter(state);
+    explorer.get_state(inserter, true);
+
+    Inspector &runningJobs = state.get()["runningJobs"];
+    EXPECT_EQUAL(1u, runningJobs.children());
+    EXPECT_EQUAL("long_running_job", runningJobs[0]["name"].asString().make_string());
+
+    Inspector &allJobs = state.get()["allJobs"];
+    EXPECT_EQUAL(2u, allJobs.children());
+    EXPECT_EQUAL("my_job", allJobs[0]["name"].asString().make_string());
+    EXPECT_EQUAL("long_running_job", allJobs[1]["name"].asString().make_string());
+}
+
+TEST("Verify FrozenBucketsMap interface") {
+    FrozenBucketsMap m;
+    BucketId a(8, 6);
+    {
+        auto guard = m.acquireExclusiveBucket(a);
+        EXPECT_TRUE(bool(guard));
+        EXPECT_EQUAL(a, guard->getBucket());
+    }
+    m.freezeBucket(a);
+    EXPECT_FALSE(m.thawBucket(a));
+    m.freezeBucket(a);
+    {
+        auto guard = m.acquireExclusiveBucket(a);
+        EXPECT_FALSE(bool(guard));
+    }
+    EXPECT_TRUE(m.thawBucket(a));
+    m.freezeBucket(a);
+    m.freezeBucket(a);
+    m.freezeBucket(a);
+    {
+        auto guard = m.acquireExclusiveBucket(a);
+        EXPECT_FALSE(bool(guard));
+    }
+    EXPECT_FALSE(m.thawBucket(a));
+    EXPECT_FALSE(m.thawBucket(a));
+    EXPECT_TRUE(m.thawBucket(a));
+    {
+        auto guard = m.acquireExclusiveBucket(a);
+        EXPECT_TRUE(bool(guard));
+        EXPECT_EQUAL(a, guard->getBucket());
+    }
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/.gitignore b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/.gitignore
new file mode 100644
index 00000000000..eaabc7b9279
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/.gitignore
@@ -0,0 +1,4 @@
+Makefile
+.depend
+*_test
+searchcore_storeonlyfeedview_test_app
diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt
new file mode 100644
index 00000000000..294360b51ba
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_storeonlyfeedview_test_app
+    SOURCES
+    storeonlyfeedview_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_feedoperation
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_proton_metrics
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_storeonlyfeedview_test_app COMMAND searchcore_storeonlyfeedview_test_app)
diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
new file mode 100644
index 00000000000..fe9251e1193
--- /dev/null
+++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
@@ -0,0 +1,289 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for storeonlyfeedview.
+
+#include <vespa/log/log.h>
+LOG_SETUP("storeonlyfeedview_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/globalid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+#include <vespa/searchcore/proton/server/storeonlyfeedview.h>
+#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/test/thread_utils.h>
+#include <vespa/searchcore/proton/common/commit_time_tracker.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using document::BucketId;
+using document::DataType;
+using document::Document;
+using document::DocumentId;
+using document::DocumentTypeRepo;
+using document::DocumentUpdate;
+using document::GlobalId;
+using search::DocumentIdT;
+using search::index::DocBuilder;
+using search::index::Schema;
+using search::SerialNum;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+using namespace proton;
+
+namespace {
+
+class MySummaryAdapter : public ISummaryAdapter {
+    int &_rm_count;
+    int &_put_count;
+    int &_heartbeat_count;
+
+public:
+    MySummaryAdapter(int &remove_count, int &put_count, int &heartbeat_count)
+        : _rm_count(remove_count),
+          _put_count(put_count),
+          _heartbeat_count(heartbeat_count) {
+    }
+    virtual void put(SerialNum, const Document &, DocumentIdT)
+    { ++ _put_count; }
+    virtual void remove(SerialNum, DocumentIdT) { ++_rm_count; }
+    virtual void update(SerialNum, const DocumentUpdate &, DocumentIdT,
+                        const DocumentTypeRepo &) {}
+
+    virtual void heartBeat(SerialNum) { ++_heartbeat_count; }
+    virtual const search::IDocumentStore &getDocumentStore() const
+    { return *reinterpret_cast<const search::IDocumentStore *>(0); }
+
+    virtual Document::UP get(const DocumentIdT, const DocumentTypeRepo &)
+    { return Document::UP(); }
+};
+
+DocumentTypeRepo::SP myGetDocumentTypeRepo() {
+    Schema schema;
+    DocBuilder builder(schema);
+    DocumentTypeRepo::SP repo = builder.getDocumentTypeRepo();
+    ASSERT_TRUE(repo.get());
+    return repo;
+}
+
+struct MyMinimalFeedView : StoreOnlyFeedView {
+    typedef std::unique_ptr<MyMinimalFeedView> UP;
+
+    int removeAttributes_count;
+    int removeIndexedFields_count;
+    int heartBeatAttributes_count;
+    int heartBeatIndexedFields_count;
+
+    MyMinimalFeedView(const ISummaryAdapter::SP &summary_adapter,
+                      const DocumentMetaStore::SP &meta_store,
+                      searchcorespi::index::IThreadingService &writeService,
+                      documentmetastore::ILidReuseDelayer &lidReuseDelayer,
+                      CommitTimeTracker &commitTimeTracker,
+                      const PersistentParams &params) :
+        StoreOnlyFeedView(StoreOnlyFeedView::Context(summary_adapter,
+                        search::index::Schema::SP(),
+                        DocumentMetaStoreContext::SP(
+                                new DocumentMetaStoreContext(meta_store)),
+                        myGetDocumentTypeRepo(),
+                        writeService,
+                        lidReuseDelayer,
+                        commitTimeTracker),
+                        params),
+        removeAttributes_count(0),
+        removeIndexedFields_count(0),
+        heartBeatAttributes_count(0),
+        heartBeatIndexedFields_count(0) {
+    }
+    virtual void removeAttributes(SerialNum s, const LidVector &l,
+                                  bool immediateCommit, OnWriteDoneType onWriteDone) override {
+        StoreOnlyFeedView::removeAttributes(s, l, immediateCommit, onWriteDone);
+        ++removeAttributes_count;
+    }
+    virtual void removeIndexedFields(SerialNum s, const LidVector &l,
+                                     bool immediateCommit,
+                                     OnWriteDoneType onWriteDone) override {
+        StoreOnlyFeedView::removeIndexedFields(s, l,
+                                               immediateCommit, onWriteDone);
+        ++removeIndexedFields_count;
+    }
+    virtual void heartBeatIndexedFields(SerialNum s) override {
+        StoreOnlyFeedView::heartBeatIndexedFields(s);
+        ++heartBeatIndexedFields_count;
+    }
+    virtual void heartBeatAttributes(SerialNum s) override {
+        StoreOnlyFeedView::heartBeatAttributes(s);
+        ++heartBeatAttributes_count;
+    }
+};
+
+const uint32_t subdb_id = 0;
+
+struct Fixture {
+    int remove_count;
+    int put_count;
+    int heartbeat_count;
+    DocumentMetaStore::SP meta_store;
+    ExecutorThreadingService writeService;
+    documentmetastore::LidReuseDelayer _lidReuseDelayer;
+    CommitTimeTracker _commitTimeTracker;
+    MyMinimalFeedView::UP feedview;
+ 
+    Fixture(SubDbType subDbType = SubDbType::READY)
+        : remove_count(0),
+          put_count(0),
+          heartbeat_count(0),
+          meta_store(new DocumentMetaStore(std::make_shared<BucketDBOwner>(),
+                                           DocumentMetaStore::getFixedName(),
+                                           search::GrowStrategy(),
+                                           DocumentMetaStore::IGidCompare::SP(
+                                                   new DocumentMetaStore::
+                                                   DefaultGidCompare),
+                                           subDbType)),
+          writeService(),
+          _lidReuseDelayer(writeService, *meta_store),
+          _commitTimeTracker(fastos::TimeStamp()),
+          feedview() {
+        PerDocTypeFeedMetrics metrics(0);
+        StoreOnlyFeedView::PersistentParams
+            params(0, 0, DocTypeName("foo"), metrics, subdb_id,
+                   subDbType);
+        meta_store->constructFreeList();
+        ISummaryAdapter::SP adapter(new MySummaryAdapter(
+                        remove_count, put_count, heartbeat_count));
+        feedview.reset(new MyMinimalFeedView(adapter, meta_store, writeService,
+                                             _lidReuseDelayer,
+                                             _commitTimeTracker, params));
+    }
+
+    ~Fixture() {
+        writeService.sync();
+    }
+
+    void addSingleDocToMetaStore(uint32_t expected_lid) {
+        typedef DocumentMetaStore::Result Result;
+        DocumentId id(make_string("groupdoc:test:foo:%d", expected_lid));
+        Result inspect = meta_store->inspect(id.getGlobalId());
+        EXPECT_EQUAL(expected_lid,
+                     meta_store->put(id.getGlobalId(),
+                                     id.getGlobalId().convertToBucketId(),
+                                     Timestamp(10), inspect.getLid()).getLid());
+    }
+
+    void addDocsToMetaStore(int count) {
+        for (int i = 1; i <= count; ++i) {
+            addSingleDocToMetaStore(i);
+            EXPECT_TRUE(meta_store->validLid(i));
+        }
+    }
+
+    template <typename FunctionType>
+    void runInMaster(FunctionType func) {
+        test::runInMaster(writeService, func);
+    }
+
+};
+
+TEST_F("require that prepareMove sets target db document id", Fixture)
+{
+    Document::SP doc(new Document);
+    MoveOperation op(BucketId(20, 42), Timestamp(10), doc, 1, subdb_id + 1);
+    f.runInMaster([&] () { f.feedview->prepareMove(op); });
+
+    DbDocumentId target_id = op.getDbDocumentId();
+    EXPECT_EQUAL(subdb_id, target_id.getSubDbId());
+    EXPECT_EQUAL(1u, target_id.getLid());
+}
+
+TEST_F("require that handleMove adds document to target "
+       "and removes it from source", Fixture)
+{
+    Document::SP doc(new Document);
+    MoveOperation op(doc->getId().getGlobalId().convertToBucketId(),
+                     Timestamp(10), doc,
+                     DbDocumentId(subdb_id + 1, 1), subdb_id);
+    op.setSerialNum(1);
+    EXPECT_EQUAL(0, f.put_count);
+    f.runInMaster([&] () { f.feedview->prepareMove(op); });
+    f.runInMaster([&] () { f.feedview->handleMove(op); });
+    EXPECT_EQUAL(1, f.put_count);
+    uint32_t lid = op.getDbDocumentId().getLid();
+    EXPECT_TRUE(f.meta_store->validLid(lid));
+
+    // Change the MoveOperation so this is the source sub db.
+    op.setDbDocumentId(DbDocumentId(subdb_id + 1, lid));
+    op.setPrevDbDocumentId(DbDocumentId(subdb_id, lid));
+    EXPECT_EQUAL(0, f.remove_count);
+    f.runInMaster([&] () { f.feedview->handleMove(op); });
+    EXPECT_FALSE(f.meta_store->validLid(lid));
+    EXPECT_EQUAL(1, f.remove_count);
+}
+
+
+TEST_F("require that handleMove handles move within same subdb", Fixture)
+{
+    Document::SP doc(new Document);
+    DocumentId doc1id("groupdoc:test:foo:1");
+    f.runInMaster([&] () { f.meta_store->put(doc1id.getGlobalId(),
+                      doc1id.getGlobalId().convertToBucketId(),
+                      Timestamp(9), 1); });
+    f.runInMaster([&] () { f.meta_store->put(doc->getId().getGlobalId(),
+                      doc->getId().getGlobalId().convertToBucketId(),
+                      Timestamp(10), 2); });
+    f.runInMaster([&] () { f.meta_store->remove(1); });
+    f.meta_store->removeComplete(1);
+    MoveOperation op(doc->getId().getGlobalId().convertToBucketId(),
+                     Timestamp(10), doc,
+                     DbDocumentId(subdb_id, 2), subdb_id);
+    op.setTargetLid(1);
+    op.setSerialNum(1);
+    EXPECT_EQUAL(0, f.put_count); 
+    EXPECT_EQUAL(0, f.remove_count);
+    f.runInMaster([&] () { f.feedview->handleMove(op); });
+    EXPECT_EQUAL(1, f.put_count);
+    EXPECT_EQUAL(1, f.remove_count);
+    uint32_t lid = op.getDbDocumentId().getLid();
+    EXPECT_TRUE(f.meta_store->validLid(lid));
+}
+
+
+TEST_F("require that prune removed documents removes documents",
+       Fixture(SubDbType::REMOVED))
+{
+    f.addDocsToMetaStore(3);
+
+    LidVectorContext::LP lids(new LidVectorContext(4));
+    lids->addLid(1);
+    lids->addLid(3);
+    PruneRemovedDocumentsOperation op(lids->getDocIdLimit(), subdb_id);
+    op.setLidsToRemove(lids);
+    op.setSerialNum(1);  // allows use of meta store.
+    f.runInMaster([&] () { f.feedview->handlePruneRemovedDocuments(op); });
+
+    EXPECT_EQUAL(2, f.remove_count);
+    EXPECT_FALSE(f.meta_store->validLid(1));
+    EXPECT_TRUE(f.meta_store->validLid(2));
+    EXPECT_FALSE(f.meta_store->validLid(3));
+    EXPECT_EQUAL(0, f.feedview->removeAttributes_count);
+    EXPECT_EQUAL(0, f.feedview->removeIndexedFields_count);
+}
+
+TEST_F("require that heartbeat propagates and commits meta store", Fixture)
+{
+    EXPECT_EQUAL(0u, f.meta_store->getStatus().getLastSyncToken());
+    EXPECT_EQUAL(0, f.feedview->heartBeatIndexedFields_count);
+    EXPECT_EQUAL(0, f.feedview->heartBeatAttributes_count);
+    EXPECT_EQUAL(0, f.heartbeat_count);
+    f.runInMaster([&] () { f.feedview->heartBeat(2); });
+    EXPECT_EQUAL(2u, f.meta_store->getStatus().getLastSyncToken());
+    EXPECT_EQUAL(1, f.feedview->heartBeatIndexedFields_count);
+    EXPECT_EQUAL(1, f.feedview->heartBeatAttributes_count);
+    EXPECT_EQUAL(1, f.heartbeat_count);
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/documentmetastore/.gitignore b/searchcore/src/tests/proton/documentmetastore/.gitignore
new file mode 100644
index 00000000000..619f7adbc6c
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/.gitignore
@@ -0,0 +1,6 @@
+.depend
+Makefile
+gidmapattribute_test
+/documentmetastore2.dat
+/documentmetastore3.dat
+searchcore_documentmetastore_test_app
diff --git a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt
new file mode 100644
index 00000000000..fbaa86cafc8
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_documentmetastore_test_app
+    SOURCES
+    documentmetastore_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_attribute
+    searchcore_feedoperation
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_documentmetastore_test_app COMMAND sh documentmetastore_test.sh)
diff --git a/searchcore/src/tests/proton/documentmetastore/DESC b/searchcore/src/tests/proton/documentmetastore/DESC
new file mode 100644
index 00000000000..7a7bdaae267
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/DESC
@@ -0,0 +1 @@
+documentmetastore test. Take a look at documentmetastore_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentmetastore/FILES b/searchcore/src/tests/proton/documentmetastore/FILES
new file mode 100644
index 00000000000..29d56a32b24
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/FILES
@@ -0,0 +1 @@
+documentmetastore_test.cpp
diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
new file mode 100644
index 00000000000..e1e9f58fc14
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
@@ -0,0 +1,1878 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("documentmetastore_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h>
+#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
+#include <vespa/searchlib/attribute/attributefilesavetarget.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/simpleresult.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/searchcore/proton/server/itlssyncer.h>
+
+using namespace document;
+using search::AttributeVector;
+using search::AttributeGuard;
+using search::AttributeFileSaveTarget;
+using search::DocumentMetaData;
+using vespalib::GenerationHandler;
+using vespalib::GenerationHolder;
+using search::GrowStrategy;
+using search::LidUsageStats;
+using search::QueryTermSimple;
+using search::SingleValueBitNumericAttribute;
+using search::fef::MatchData;
+using search::fef::MatchDataLayout;
+using search::fef::TermFieldMatchData;
+using search::queryeval::Blueprint;
+using search::queryeval::SearchIterator;
+using search::queryeval::SimpleResult;
+using storage::spi::Timestamp;
+using storage::spi::BucketChecksum;
+using storage::spi::BucketInfo;
+using search::TuneFileAttributes;
+using search::index::DummyFileHeaderContext;
+using proton::bucketdb::BucketState;
+
+namespace proton {
+
+class DummyTlsSyncer : public ITlsSyncer
+{
+public:
+    virtual ~DummyTlsSyncer() = default;
+
+    virtual void sync() override { }
+};
+
+class ReverseGidCompare : public DocumentMetaStore::IGidCompare
+{
+    GlobalId::BucketOrderCmp _comp;
+public:
+    ReverseGidCompare(void)
+        : IGidCompare(),
+          _comp()
+    {
+    }
+
+    virtual bool
+    operator()(const GlobalId &lhs, const GlobalId &rhs) const
+    {
+        return _comp(rhs, lhs);
+    }
+};
+
+
+struct BoolVector : public std::vector<bool> {
+    BoolVector() : std::vector<bool>() {}
+    BoolVector(size_t sz) : std::vector<bool>(sz) {}
+    BoolVector &T() { push_back(true); return *this; }
+    BoolVector &F() { push_back(false); return *this; }
+
+    uint32_t
+    countTrue(void) const
+    {
+        uint32_t res(0);
+        for (uint32_t i = 0; i < size(); ++i)
+            if ((*this)[i])
+                ++res;
+        return res;
+    }
+};
+
+typedef DocumentMetaStore::Result PutRes;
+typedef DocumentMetaStore::Result Result;
+
+BucketDBOwner::SP
+createBucketDB()
+{
+    return std::make_shared<BucketDBOwner>();
+}
+
+bool
+assertPut(const BucketId &bucketId,
+          const Timestamp &timestamp,
+          uint32_t lid,
+          const GlobalId &gid,
+          DocumentMetaStore &dms)
+{
+    Result inspect = dms.inspect(gid);
+    PutRes putRes;
+    if (!EXPECT_TRUE((putRes = dms.put(gid, bucketId, timestamp, inspect.getLid())).
+            ok())) return false;
+    return EXPECT_EQUAL(lid, putRes.getLid());
+}
+
+bool
+compare(const GlobalId &lhs, const GlobalId &rhs)
+{
+    return EXPECT_EQUAL(lhs.toString(), rhs.toString());
+}
+
+bool
+assertGid(const GlobalId &exp, uint32_t lid, const DocumentMetaStore &dms)
+{
+    GlobalId act;
+    if (!EXPECT_TRUE(dms.getGid(lid, act))) return false;
+    return compare(exp, act);
+}
+
+bool
+assertGid(const GlobalId &exp,
+          uint32_t lid,
+          const DocumentMetaStore &dms,
+          const BucketId &expBucketId,
+          const Timestamp &expTimestamp)
+{
+    GlobalId act;
+    BucketId bucketId;
+    Timestamp timestamp(1);
+    if (!EXPECT_TRUE(dms.getGid(lid, act)))
+        return false;
+    if (!compare(exp, act))
+        return false;
+    DocumentMetaData meta = dms.getMetaData(act);
+    if (!EXPECT_TRUE(meta.valid()))
+        return false;
+    bucketId = meta.bucketId;
+    timestamp = meta.timestamp;
+    if (!EXPECT_EQUAL(expBucketId.getRawId(), bucketId.getRawId()))
+        return false;
+    if (!EXPECT_EQUAL(expBucketId.getId(), bucketId.getId()))
+        return false;
+    if (!EXPECT_EQUAL(expTimestamp, timestamp))
+        return false;
+    return true;
+}
+
+bool
+assertLid(uint32_t exp, const GlobalId &gid, const DocumentMetaStore &dms)
+{
+    uint32_t act;
+    if (!EXPECT_TRUE(dms.getLid(gid, act))) return false;
+    return EXPECT_EQUAL(exp, act);
+}
+
+bool
+assertMetaData(const DocumentMetaData &exp, const DocumentMetaData &act)
+{
+    if (!EXPECT_EQUAL(exp.lid, act.lid)) return false;
+    if (!EXPECT_EQUAL(exp.timestamp, act.timestamp)) return false;
+    if (!EXPECT_EQUAL(exp.bucketId, act.bucketId)) return false;
+    if (!EXPECT_EQUAL(exp.gid, act.gid)) return false;
+    if (!EXPECT_EQUAL(exp.removed, act.removed)) return false;
+    return true;
+}
+
+bool
+assertActiveLids(const BoolVector &exp, const SingleValueBitNumericAttribute &act)
+{
+    // lid 0 is reserved
+    if (!EXPECT_EQUAL(exp.size() + 1, act.getNumDocs())) return false;
+    for (size_t i = 0; i < exp.size(); ++i) {
+        if (!EXPECT_EQUAL((exp[i] ? 1 : 0), act.getInt(i + 1))) return false;
+    }
+    return true;
+}
+
+bool
+assertBlackList(const SimpleResult &exp, Blueprint::UP blackListBlueprint, bool strict)
+{
+    MatchDataLayout mdl;
+    MatchData::UP md = mdl.createMatchData();
+    blackListBlueprint->fetchPostings(strict);
+    SearchIterator::UP sb = blackListBlueprint->createSearch(*md, strict);
+    SimpleResult act;
+    act.search(*sb);
+    return EXPECT_EQUAL(exp, act);
+}
+
+bool
+assertSearchResult(const SimpleResult &exp, const DocumentMetaStore &dms,
+                   const vespalib::string &term, const QueryTermSimple::SearchTerm &termType,
+                   bool strict, uint32_t docIdLimit = 100)
+{
+    AttributeVector::SearchContext::UP sc =
+            dms.getSearch(QueryTermSimple::UP(new QueryTermSimple(term, termType)),
+                          AttributeVector::SearchContext::Params());
+    TermFieldMatchData tfmd;
+    SearchIterator::UP sb = sc->createIterator(&tfmd, strict);
+    SimpleResult act;
+    if (strict) {
+        act.search(*sb);
+    } else {
+        act.search(*sb, docIdLimit);
+    }
+    return EXPECT_EQUAL(exp, act);
+}
+
+bool
+assertBucketInfo(uint32_t expDocCount,
+                 uint32_t expMetaCount,
+                 const BucketInfo &act)
+{
+    if (!EXPECT_EQUAL(expDocCount, act.getDocumentCount()))
+        return false;
+    if (!EXPECT_EQUAL(expMetaCount, act.getEntryCount()))
+        return false;
+    return true;
+}
+
+GlobalId gid1("111111111111");
+GlobalId gid2("222222222222");
+GlobalId gid3("333333333333");
+GlobalId gid4("444444444444");
+GlobalId gid5("555555555555");
+const uint32_t minNumBits = 8u;
+BucketId bucketId1(minNumBits,
+                   gid1.convertToBucketId().getRawId());
+BucketId bucketId2(minNumBits,
+                   gid2.convertToBucketId().getRawId());
+BucketId bucketId3(minNumBits,
+                   gid3.convertToBucketId().getRawId());
+BucketId bucketId4(minNumBits,
+                   gid4.convertToBucketId().getRawId());
+BucketId bucketId5(minNumBits,
+                   gid5.convertToBucketId().getRawId());
+Timestamp time1(1u);
+Timestamp time2(2u);
+Timestamp time3(42u);
+Timestamp time4(82u);
+Timestamp time5(141u);
+
+uint32_t
+addGid(DocumentMetaStore &dms, const GlobalId &gid, const BucketId &bid, Timestamp timestamp = Timestamp())
+{
+    Result inspect = dms.inspect(gid);
+    PutRes putRes;
+    EXPECT_TRUE((putRes = dms.put(gid, bid, timestamp, inspect.getLid())).ok());
+    return putRes.getLid();
+}
+
+uint32_t
+addGid(DocumentMetaStore &dms, const GlobalId &gid, Timestamp timestamp = Timestamp())
+{
+    BucketId bid(minNumBits, gid.convertToBucketId().getRawId());
+    return addGid(dms, gid, bid, timestamp);
+}
+
+void
+putGid(DocumentMetaStore &dms, const GlobalId &gid, uint32_t lid, Timestamp timestamp = Timestamp())
+{
+    BucketId bid(minNumBits, gid.convertToBucketId().getRawId());
+    EXPECT_TRUE(dms.put(gid, bid, timestamp, lid).ok());
+}
+
+TEST("require that removed documents are bucketized to bucket 0")
+{
+    DocumentMetaStore dms(createBucketDB());
+    dms.constructFreeList();
+    EXPECT_EQUAL(1u, dms.getNumDocs());
+    EXPECT_EQUAL(0u, dms.getNumUsedLids());
+
+    vespalib::GenerationHandler::Guard guard = dms.getGuard();
+    EXPECT_EQUAL(0ul, dms.getBucketOf(guard, 1));
+    EXPECT_TRUE(assertPut(bucketId1, time1, 1, gid1, dms));
+    EXPECT_EQUAL(bucketId1.getId(), dms.getBucketOf(guard, 1));
+    EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, dms));
+    EXPECT_EQUAL(bucketId2.getId(), dms.getBucketOf(guard, 2));
+    EXPECT_TRUE(dms.remove(1));
+    EXPECT_EQUAL(0ul, dms.getBucketOf(guard, 1));
+    EXPECT_EQUAL(bucketId2.getId(), dms.getBucketOf(guard, 2));
+}
+
+TEST("requireThatGidsCanBeInsertedAndRetrieved")
+{
+    DocumentMetaStore dms(createBucketDB());
+    dms.constructFreeList();
+    // put()
+    EXPECT_EQUAL(1u, dms.getNumDocs());
+    EXPECT_EQUAL(0u, dms.getNumUsedLids());
+    EXPECT_TRUE(assertPut(bucketId1, time1, 1, gid1, dms));
+    EXPECT_EQUAL(2u, dms.getNumDocs());
+    EXPECT_EQUAL(1u, dms.getNumUsedLids());
+    EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, dms));
+    EXPECT_EQUAL(3u, dms.getNumDocs());
+    EXPECT_EQUAL(2u, dms.getNumUsedLids());
+    // gid1 already inserted
+    EXPECT_TRUE(assertPut(bucketId1, time1, 1, gid1, dms));
+    // gid2 already inserted
+    EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, dms));
+
+
+    // getGid()
+    GlobalId gid;
+    EXPECT_TRUE(assertGid(gid1, 1, dms));
+    EXPECT_TRUE(assertGid(gid2, 2, dms));
+    EXPECT_TRUE(!dms.getGid(3, gid));
+
+    // getLid()
+    uint32_t lid = 0;
+    EXPECT_TRUE(assertLid(1, gid1, dms));
+    EXPECT_TRUE(assertLid(2, gid2, dms));
+    EXPECT_TRUE(!dms.getLid(gid3, lid));
+}
+
+TEST("requireThatGidsCanBeCleared")
+{
+    DocumentMetaStore dms(createBucketDB());
+    GlobalId gid;
+    uint32_t lid = 0u;
+    dms.constructFreeList();
+    addGid(dms, gid1, bucketId1, time1);
+    EXPECT_TRUE(assertGid(gid1, 1, dms));
+    EXPECT_TRUE(assertLid(1, gid1, dms));
+    EXPECT_EQUAL(1u, dms.getNumUsedLids());
+    EXPECT_TRUE(dms.remove(1));
+    dms.removeComplete(1);
+    EXPECT_EQUAL(0u, dms.getNumUsedLids());
+    EXPECT_TRUE(!dms.getGid(1, gid));
+    EXPECT_TRUE(!dms.getLid(gid1, lid));
+    // reuse lid
+    addGid(dms, gid2, bucketId2, time2);
+    EXPECT_TRUE(assertGid(gid2, 1, dms));
+    EXPECT_TRUE(assertLid(1, gid2, dms));
+    EXPECT_EQUAL(1u, dms.getNumUsedLids());
+    EXPECT_TRUE(dms.remove(1));
+    dms.removeComplete(1);
+    EXPECT_EQUAL(0u, dms.getNumUsedLids());
+    EXPECT_TRUE(!dms.getGid(1, gid));
+    EXPECT_TRUE(!dms.getLid(gid2, lid));
+    EXPECT_TRUE(!dms.remove(1)); // not used
+    EXPECT_TRUE(!dms.remove(2)); // outside range
+}
+
+TEST("requireThatGenerationHandlingIsWorking")
+{
+    AttributeVector::SP av(new DocumentMetaStore(createBucketDB()));
+    DocumentMetaStore * dms = static_cast<DocumentMetaStore *>(av.get());
+    dms->constructFreeList();
+    const GenerationHandler & gh = dms->getGenerationHandler();
+    EXPECT_EQUAL(1u, gh.getCurrentGeneration());
+    addGid(*dms, gid1, bucketId1, time1);
+    EXPECT_EQUAL(2u, gh.getCurrentGeneration());
+    EXPECT_EQUAL(0u, gh.getGenerationRefCount());
+    {
+        AttributeGuard g1(av);
+        EXPECT_EQUAL(1u, gh.getGenerationRefCount());
+        {
+            AttributeGuard g2(av);
+            EXPECT_EQUAL(2u, gh.getGenerationRefCount());
+        }
+        EXPECT_EQUAL(1u, gh.getGenerationRefCount());
+    }
+    EXPECT_EQUAL(0u, gh.getGenerationRefCount());
+    dms->remove(1);
+    dms->removeComplete(1);
+    EXPECT_EQUAL(4u, gh.getCurrentGeneration());
+}
+
+TEST("requireThatBasicFreeListIsWorking")
+{
+    GenerationHolder genHold;
+    LidStateVector freeLids(100, 100, genHold, true, false);
+    LidHoldList list;
+    EXPECT_TRUE(freeLids.empty());
+    EXPECT_EQUAL(0u, freeLids.count());
+    EXPECT_EQUAL(0u, list.size());
+
+    list.add(10, 10);
+    EXPECT_TRUE(freeLids.empty());
+    EXPECT_EQUAL(0u, freeLids.count());
+    EXPECT_EQUAL(1u, list.size());
+
+    list.add(20, 20);
+    list.add(30, 30);
+    EXPECT_TRUE(freeLids.empty());
+    EXPECT_EQUAL(0u, freeLids.count());
+    EXPECT_EQUAL(3u, list.size());
+
+    list.trimHoldLists(20, freeLids);
+    EXPECT_FALSE(freeLids.empty());
+    EXPECT_EQUAL(1u, freeLids.count());
+
+    EXPECT_EQUAL(10u, freeLids.getLowest());
+    freeLids.clearBit(10);
+    EXPECT_TRUE(freeLids.empty());
+    EXPECT_EQUAL(0u, freeLids.count());
+    EXPECT_EQUAL(2u, list.size());
+
+    list.trimHoldLists(31, freeLids);
+    EXPECT_FALSE(freeLids.empty());
+    EXPECT_EQUAL(2u, freeLids.count());
+
+    EXPECT_EQUAL(20u, freeLids.getLowest());
+    freeLids.clearBit(20);
+    EXPECT_FALSE(freeLids.empty());
+    EXPECT_EQUAL(1u, freeLids.count());
+    EXPECT_EQUAL(0u, list.size());
+
+    EXPECT_EQUAL(30u, freeLids.getLowest());
+    freeLids.clearBit(30);
+    EXPECT_TRUE(freeLids.empty());
+    EXPECT_EQUAL(0u, list.size());
+    EXPECT_EQUAL(0u, freeLids.count());
+}
+
+void
+assertLidStateVector(const std::vector<uint32_t> &expLids, uint32_t lowest, uint32_t highest,
+                     const LidStateVector &actLids)
+{
+    if (!expLids.empty()) {
+        EXPECT_EQUAL(expLids.size(), actLids.count());
+        uint32_t trueBit = 0;
+        for (auto i : expLids) {
+            EXPECT_TRUE(actLids.testBit(i));
+            trueBit = actLids.getNextTrueBit(trueBit);
+            EXPECT_EQUAL(i, trueBit);
+            ++trueBit;
+        }
+        trueBit = actLids.getNextTrueBit(trueBit);
+        EXPECT_EQUAL(actLids.size(), trueBit);
+        EXPECT_EQUAL(lowest, actLids.getLowest());
+        EXPECT_EQUAL(highest, actLids.getHighest());
+    } else {
+        EXPECT_TRUE(actLids.empty());
+    }
+}
+
+TEST("requireThatLidStateVectorResizingIsWorking")
+{
+    GenerationHolder genHold;
+    LidStateVector lids(1000, 1000, genHold, true, true);
+    lids.setBit(3);
+    lids.setBit(150);
+    lids.setBit(270);
+    lids.setBit(310);
+    lids.setBit(440);
+    lids.setBit(780);
+    lids.setBit(930);
+    assertLidStateVector({3,150,270,310,440,780,930}, 3, 930, lids);
+
+    lids.resizeVector(1500, 1500);
+    assertLidStateVector({3,150,270,310,440,780,930}, 3, 930, lids);
+    lids.clearBit(3);
+    assertLidStateVector({150,270,310,440,780,930}, 150, 930, lids);
+    lids.clearBit(150);
+    assertLidStateVector({270,310,440,780,930}, 270, 930, lids);
+    lids.setBit(170);
+    assertLidStateVector({170,270,310,440,780,930}, 170, 930, lids);
+    lids.setBit(1490);
+    assertLidStateVector({170,270,310,440,780,930,1490}, 170, 1490, lids);
+
+    lids.resizeVector(2000, 2000);
+    assertLidStateVector({170,270,310,440,780,930,1490}, 170, 1490, lids);
+    lids.clearBit(170);
+    assertLidStateVector({270,310,440,780,930,1490}, 270, 1490, lids);
+    lids.clearBit(270);
+    assertLidStateVector({310,440,780,930,1490}, 310, 1490, lids);
+    lids.setBit(1990);
+    assertLidStateVector({310,440,780,930,1490,1990}, 310, 1990, lids);
+    lids.clearBit(310);
+    assertLidStateVector({440,780,930,1490,1990}, 440, 1990, lids);
+    lids.clearBit(440);
+    assertLidStateVector({780,930,1490,1990}, 780, 1990, lids);
+    lids.clearBit(780);
+    assertLidStateVector({930,1490,1990}, 930, 1990, lids);
+    lids.clearBit(930);
+    assertLidStateVector({1490,1990}, 1490, 1990, lids);
+    lids.clearBit(1490);
+    assertLidStateVector({1990}, 1990, 1990, lids);
+    lids.clearBit(1990);
+    assertLidStateVector({}, 0, 0, lids);
+
+    genHold.clearHoldLists();
+}
+
+TEST("requireThatLidAndGidSpaceIsReused")
+{
+    AttributeVector::SP av(new DocumentMetaStore(createBucketDB()));
+    DocumentMetaStore * dms = static_cast<DocumentMetaStore *>(av.get());
+    dms->constructFreeList();
+    EXPECT_EQUAL(1u, dms->getNumDocs());
+    EXPECT_EQUAL(0u, dms->getNumUsedLids());
+    EXPECT_TRUE(assertPut(bucketId1, time1, 1, gid1, *dms)); // -> gen 1
+    EXPECT_EQUAL(2u, dms->getNumDocs());
+    EXPECT_EQUAL(1u, dms->getNumUsedLids());
+    EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, *dms)); // -> gen 2
+    EXPECT_EQUAL(3u, dms->getNumDocs());
+    EXPECT_EQUAL(2u, dms->getNumUsedLids());
+    dms->remove(2); // -> gen 3
+    dms->removeComplete(2); // -> gen 4
+    EXPECT_EQUAL(3u, dms->getNumDocs());
+    EXPECT_EQUAL(1u, dms->getNumUsedLids());
+    // -> gen 5 (reuse of lid 2)
+    EXPECT_TRUE(assertPut(bucketId3, time3, 2, gid3, *dms));
+    EXPECT_EQUAL(3u, dms->getNumDocs());
+    EXPECT_EQUAL(2u, dms->getNumUsedLids()); // reuse
+    EXPECT_TRUE(assertGid(gid3, 2, *dms));
+    {
+        AttributeGuard g1(av); // guard on gen 5
+        dms->remove(2);
+        dms->removeComplete(2);
+        EXPECT_EQUAL(3u, dms->getNumDocs());
+        EXPECT_EQUAL(1u, dms->getNumUsedLids()); // lid 2 free but guarded
+        EXPECT_TRUE(assertPut(bucketId4, time4, 3, gid4, *dms));
+        EXPECT_EQUAL(4u, dms->getNumDocs()); // generation guarded, new lid
+        EXPECT_EQUAL(2u, dms->getNumUsedLids());
+        EXPECT_TRUE(assertGid(gid4, 3, *dms));
+    }
+    EXPECT_TRUE(assertPut(bucketId5, time5, 4, gid5, *dms));
+    EXPECT_EQUAL(5u, dms->getNumDocs()); // reuse blocked by previous guard. released at end of put()
+    EXPECT_EQUAL(3u, dms->getNumUsedLids());
+    EXPECT_TRUE(assertGid(gid5, 4, *dms));
+    EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, *dms)); // reuse of lid 2
+    EXPECT_EQUAL(5u, dms->getNumDocs());
+    EXPECT_EQUAL(4u, dms->getNumUsedLids());
+    EXPECT_TRUE(assertGid(gid2, 2, *dms));
+}
+
+GlobalId
+createGid(uint32_t lid)
+{
+    DocumentId docId(vespalib::make_string("doc:id:%u", lid));
+    return docId.getGlobalId();
+}
+
+GlobalId
+createGid(uint32_t userId, uint32_t lid)
+{
+    DocumentId docId(vespalib::make_string("userdoc:id:%u:%u", userId, lid));
+    return docId.getGlobalId();
+}
+
+TEST("requireThatWeCanStoreBucketIdAndTimestamp")
+{
+    DocumentMetaStore dms(createBucketDB());
+    uint32_t numLids = 1000;
+    uint32_t bkBits = UINT32_C(20);
+    uint64_t tsbias = UINT64_C(2000000000000);
+
+    dms.constructFreeList();
+    for (uint32_t lid = 1; lid <= numLids; ++lid) {
+        GlobalId gid = createGid(lid);
+        BucketId bucketId(gid.convertToBucketId());
+        bucketId.setUsedBits(bkBits);
+        uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(lid + tsbias));
+        EXPECT_EQUAL(lid, addLid);
+    }
+    for (uint32_t lid = 1; lid <= numLids; ++lid) {
+        GlobalId gid = createGid(lid);
+        BucketId bucketId(gid.convertToBucketId());
+        bucketId.setUsedBits(bkBits);
+        EXPECT_TRUE(assertGid(gid, lid, dms, bucketId,
+                              Timestamp(lid + tsbias)));
+        EXPECT_TRUE(assertLid(lid, gid, dms));
+    }
+}
+
+TEST("requireThatGidsCanBeSavedAndLoaded")
+{
+    DocumentMetaStore dms1(createBucketDB());
+    uint32_t numLids = 1000;
+    uint32_t bkBits = UINT32_C(20);
+    uint64_t tsbias = UINT64_C(2000000000000);
+    std::vector<uint32_t> removeLids;
+    removeLids.push_back(10);
+    removeLids.push_back(20);
+    removeLids.push_back(100);
+    removeLids.push_back(500);
+    dms1.constructFreeList();
+    for (uint32_t lid = 1; lid <= numLids; ++lid) {
+        GlobalId gid = createGid(lid);
+        BucketId bucketId(gid.convertToBucketId());
+        bucketId.setUsedBits(bkBits);
+        uint32_t addLid = addGid(dms1, gid, bucketId, Timestamp(lid + tsbias));
+        EXPECT_EQUAL(lid, addLid);
+    }
+    for (size_t i = 0; i < removeLids.size(); ++i) {
+        dms1.remove(removeLids[i]);
+        dms1.removeComplete(removeLids[i]);
+    }
+    uint64_t expSaveBytesSize = DocumentMetaStore::minHeaderLen +
+                                (1000 - 4) * DocumentMetaStore::entrySize;
+    EXPECT_EQUAL(expSaveBytesSize, dms1.getEstimatedSaveByteSize());
+    TuneFileAttributes tuneFileAttributes;
+    DummyFileHeaderContext fileHeaderContext;
+    AttributeFileSaveTarget saveTarget(tuneFileAttributes, fileHeaderContext);
+    EXPECT_TRUE(dms1.saveAs("documentmetastore2", saveTarget));
+
+    DocumentMetaStore dms2(createBucketDB(), "documentmetastore2");
+    EXPECT_TRUE(dms2.load());
+    dms2.constructFreeList();
+    EXPECT_EQUAL(numLids + 1, dms2.getNumDocs());
+    EXPECT_EQUAL(numLids - 4, dms2.getNumUsedLids()); // 4 removed
+    for (uint32_t lid = 1; lid <= numLids; ++lid) {
+        GlobalId gid = createGid(lid);
+        BucketId bucketId(gid.convertToBucketId());
+        bucketId.setUsedBits(bkBits);
+        if (std::count(removeLids.begin(), removeLids.end(), lid) == 0) {
+            EXPECT_TRUE(assertGid(gid, lid, dms2, bucketId,
+                                  Timestamp(lid + tsbias)));
+            EXPECT_TRUE(assertLid(lid, gid, dms2));
+        } else {
+            LOG(info, "Lid %u was removed before saving", lid);
+            uint32_t myLid;
+            GlobalId myGid;
+            EXPECT_TRUE(!dms2.getGid(lid, myGid));
+            EXPECT_TRUE(!dms2.getLid(gid, myLid));
+        }
+    }
+    // check we can re-use from free list after load
+    for (size_t i = 0; i < removeLids.size(); ++i) {
+        LOG(info, "Re-use remove lid %u", removeLids[i]);
+        GlobalId gid = createGid(removeLids[i]);
+        BucketId bucketId(bkBits,
+                          gid.convertToBucketId().getRawId());
+        // re-use removeLid[i]
+        uint32_t addLid = addGid(dms2, gid, bucketId, Timestamp(43u + i));
+        EXPECT_EQUAL(removeLids[i], addLid);
+        EXPECT_EQUAL(numLids + 1, dms2.getNumDocs());
+        EXPECT_EQUAL(numLids - (3 - i), dms2.getNumUsedLids());
+    }
+}
+
+TEST("requireThatStatsAreUpdated")
+{
+    DocumentMetaStore dms(createBucketDB());
+    dms.constructFreeList();
+    size_t perGidUsed = sizeof(uint32_t) + GlobalId::LENGTH;
+    EXPECT_EQUAL(1u, dms.getStatus().getNumDocs());
+    EXPECT_EQUAL(1u, dms.getStatus().getNumValues());
+    uint64_t lastAllocated = dms.getStatus().getAllocated();
+    uint64_t lastUsed = dms.getStatus().getUsed();
+    EXPECT_GREATER(lastAllocated, perGidUsed);
+    EXPECT_GREATER(lastUsed, perGidUsed);
+
+    FastOS_Thread::Sleep(2200);
+    addGid(dms, gid1, bucketId1, time1);
+    EXPECT_EQUAL(2u, dms.getStatus().getNumDocs());
+    EXPECT_EQUAL(2u, dms.getStatus().getNumValues());
+    EXPECT_GREATER_EQUAL(dms.getStatus().getAllocated(), lastAllocated);
+    EXPECT_GREATER_EQUAL(dms.getStatus().getAllocated(), lastUsed);
+    EXPECT_GREATER(dms.getStatus().getUsed(), lastUsed);
+    EXPECT_GREATER(dms.getStatus().getUsed(), 2 * perGidUsed);
+    lastAllocated = dms.getStatus().getAllocated();
+    lastUsed = dms.getStatus().getUsed();
+
+    addGid(dms, gid2, bucketId2, time2);
+    dms.commit(true);
+    EXPECT_EQUAL(3u, dms.getStatus().getNumDocs());
+    EXPECT_EQUAL(3u, dms.getStatus().getNumValues());
+    EXPECT_GREATER_EQUAL(dms.getStatus().getAllocated(), lastAllocated);
+    EXPECT_GREATER_EQUAL(dms.getStatus().getAllocated(), lastUsed);
+    EXPECT_GREATER(dms.getStatus().getUsed(), lastUsed);
+    EXPECT_GREATER(dms.getStatus().getUsed(), 3 * perGidUsed);
+    LOG(info,
+        "stats after 2 gids added: allocated %d, used is %d > %d (3 * %d)",
+        static_cast<int>(dms.getStatus().getAllocated()),
+        static_cast<int>(dms.getStatus().getUsed()),
+        static_cast<int>(3 * perGidUsed),
+        static_cast<int>(perGidUsed));
+}
+
+TEST("requireThatWeCanPutAndRemoveBeforeFreeListConstruct")
+{
+    DocumentMetaStore dms(createBucketDB());
+    EXPECT_TRUE(dms.put(gid4, bucketId4, time4, 4).ok());
+    EXPECT_TRUE(assertLid(4, gid4, dms));
+    EXPECT_TRUE(assertGid(gid4, 4, dms));
+    EXPECT_EQUAL(1u, dms.getNumUsedLids());
+    EXPECT_EQUAL(5u, dms.getNumDocs());
+    EXPECT_TRUE(dms.put(gid1, bucketId1, time1, 1).ok());
+    // already there, nothing changes
+    EXPECT_TRUE(dms.put(gid1, bucketId1, time1, 1).ok());
+    EXPECT_TRUE(assertLid(1, gid1, dms));
+    EXPECT_TRUE(assertGid(gid1, 1, dms));
+    EXPECT_EQUAL(2u, dms.getNumUsedLids());
+    EXPECT_EQUAL(5u, dms.getNumDocs());
+    // gid1 already there with lid 1
+    EXPECT_EXCEPTION(!dms.put(gid1, bucketId1, time1, 2).ok(),
+                     vespalib::IllegalStateException,
+                     "gid found, but using another lid");
+    EXPECT_EXCEPTION(!dms.put(gid5, bucketId5, time5, 1).ok(),
+                     vespalib::IllegalStateException,
+                     "gid not found, but lid is used by another gid");
+    EXPECT_TRUE(assertLid(1, gid1, dms));
+    EXPECT_TRUE(assertGid(gid1, 1, dms));
+    EXPECT_EQUAL(2u, dms.getNumUsedLids());
+    EXPECT_EQUAL(5u, dms.getNumDocs());
+    EXPECT_TRUE(dms.remove(4)); // -> goes to free list. cleared and re-applied in constructFreeList().
+    uint32_t lid;
+    GlobalId gid;
+    EXPECT_TRUE(!dms.getLid(gid4, lid));
+    EXPECT_TRUE(!dms.getGid(4, gid));
+    EXPECT_EQUAL(1u, dms.getNumUsedLids());
+    EXPECT_EQUAL(5u, dms.getNumDocs());
+    dms.constructFreeList();
+    EXPECT_EQUAL(1u, dms.getNumUsedLids());
+    EXPECT_EQUAL(5u, dms.getNumDocs());
+    EXPECT_TRUE(assertPut(bucketId2, time2, 2, gid2, dms));
+    EXPECT_TRUE(assertPut(bucketId3, time3, 3, gid3, dms));
+    EXPECT_EQUAL(3u, dms.getNumUsedLids());
+    EXPECT_EQUAL(5u, dms.getNumDocs());
+}
+
+TEST("requireThatWeCanSortGids")
+{
+    DocumentMetaStore dms(createBucketDB());
+    DocumentMetaStore rdms(createBucketDB(),
+                         DocumentMetaStore::getFixedName(),
+                         GrowStrategy(),
+                         DocumentMetaStore::IGidCompare::SP(
+                                 new ReverseGidCompare));
+
+    dms.constructFreeList();
+    rdms.constructFreeList();
+    uint32_t numLids = 1000;
+    for (uint32_t lid = 1; lid <= numLids; ++lid) {
+        GlobalId gid = createGid(lid);
+        Timestamp oldTimestamp;
+        BucketId bucketId(minNumBits,
+                          gid.convertToBucketId().getRawId());
+        uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(0u));
+        EXPECT_EQUAL(lid, addLid);
+        uint32_t addLid2 = addGid(rdms, gid, bucketId, Timestamp(0u));
+        EXPECT_EQUAL(lid, addLid2);
+    }
+    std::vector<uint32_t> lids;
+    std::vector<uint32_t> rlids;
+    for (DocumentMetaStore::ConstIterator it = dms.beginFrozen(); it.valid(); ++it)
+        lids.push_back(it.getKey());
+    for (DocumentMetaStore::ConstIterator rit = rdms.beginFrozen();
+         rit.valid(); ++rit)
+        rlids.push_back(rit.getKey());
+    EXPECT_EQUAL(numLids, lids.size());
+    EXPECT_EQUAL(numLids, rlids.size());
+    for (uint32_t i = 0; i < numLids; ++i) {
+        EXPECT_EQUAL(lids[numLids - 1 - i], rlids[i]);
+    }
+}
+
+TEST("requireThatBasicBucketInfoWorks")
+{
+    DocumentMetaStore dms(createBucketDB());
+    typedef std::pair<BucketId, GlobalId> Elem;
+    typedef std::map<Elem, Timestamp> Map;
+    Map m;
+    uint32_t numLids = 2000;
+    dms.constructFreeList();
+    for (uint32_t lid = 1; lid <= numLids; ++lid) {
+        GlobalId gid = createGid(lid);
+        Timestamp timestamp(UINT64_C(123456789) * lid);
+        Timestamp oldTimestamp;
+        BucketId bucketId(minNumBits,
+                          gid.convertToBucketId().getRawId());
+        uint32_t addLid = addGid(dms, gid, bucketId, timestamp);
+        EXPECT_EQUAL(lid, addLid);
+        m[std::make_pair(bucketId, gid)] = timestamp;
+    }
+    for (uint32_t lid = 2; lid <= numLids; lid += 7) {
+        GlobalId gid = createGid(lid);
+        Timestamp timestamp(UINT64_C(14735) * lid);
+        Timestamp oldTimestamp;
+        BucketId bucketId(minNumBits,
+                          gid.convertToBucketId().getRawId());
+        uint32_t addLid = addGid(dms, gid, bucketId, timestamp);
+        EXPECT_EQUAL(lid, addLid);
+        m[std::make_pair(bucketId, gid)] = timestamp;
+    }
+    for (uint32_t lid = 3; lid <= numLids; lid += 5) {
+        GlobalId gid = createGid(lid);
+        BucketId bucketId(minNumBits,
+                          gid.convertToBucketId().getRawId());
+        EXPECT_TRUE(dms.remove(lid));
+        dms.removeComplete(lid);
+        m.erase(std::make_pair(bucketId, gid));
+    }
+    assert(!m.empty());
+    BucketChecksum cksum;
+    BucketId prevBucket = m.begin()->first.first;
+    uint32_t cnt = 0u;
+    uint32_t maxcnt = 0u;
+    BucketDBOwner::Guard bucketDB = dms.getBucketDB().takeGuard();
+    for (Map::const_iterator i = m.begin(), ie = m.end(); i != ie; ++i) {
+        if (i->first.first == prevBucket) {
+            cksum = BucketChecksum(cksum +
+                                   BucketState::calcChecksum(i->first.second,
+                                           i->second));
+            ++cnt;
+        } else {
+            BucketInfo bi = bucketDB->get(prevBucket);
+            EXPECT_EQUAL(cnt, bi.getDocumentCount());
+            EXPECT_EQUAL(cksum, bi.getChecksum());
+            prevBucket = i->first.first;
+            cksum = BucketState::calcChecksum(i->first.second,
+                    i->second);
+            maxcnt = std::max(maxcnt, cnt);
+            cnt = 1u;
+        }
+    }
+    maxcnt = std::max(maxcnt, cnt);
+    BucketInfo bi = bucketDB->get(prevBucket);
+    EXPECT_EQUAL(cnt, bi.getDocumentCount());
+    EXPECT_EQUAL(cksum, bi.getChecksum());
+    LOG(info, "Largest bucket: %u elements", maxcnt);
+}
+
+TEST("requireThatWeCanRetrieveListOfLidsFromBucketId")
+{
+    typedef std::vector<uint32_t> LidVector;
+    typedef std::map<BucketId, LidVector> Map;
+    DocumentMetaStore dms(createBucketDB());
+    const uint32_t bucketBits = 2; // -> 4 buckets
+    uint32_t numLids = 1000;
+    Map m;
+
+    dms.constructFreeList();
+    // insert global ids
+    for (uint32_t lid = 1; lid <= numLids; ++lid) {
+        GlobalId gid = createGid(lid);
+        BucketId bucketId(bucketBits,
+                          gid.convertToBucketId().getRawId());
+        uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(0));
+        EXPECT_EQUAL(lid, addLid);
+        m[bucketId].push_back(lid);
+    }
+
+    // Verify that bucket id x has y lids
+    EXPECT_EQUAL(4u, m.size());
+    for (Map::const_iterator itr = m.begin(); itr != m.end(); ++itr) {
+        const BucketId &bucketId = itr->first;
+        const LidVector &expLids = itr->second;
+        LOG(info, "Verify that bucket id '%s' has %zu lids",
+            bucketId.toString().c_str(), expLids.size());
+        LidVector actLids;
+        dms.getLids(bucketId, actLids);
+        EXPECT_EQUAL(expLids.size(), actLids.size());
+        for (size_t i = 0; i < expLids.size(); ++i) {
+            EXPECT_TRUE(std::find(actLids.begin(), actLids.end(), expLids[i]) != actLids.end());
+        }
+    }
+
+    // Remove and verify empty buckets
+    for (Map::const_iterator itr = m.begin(); itr != m.end(); ++itr) {
+        const BucketId &bucketId = itr->first;
+        const LidVector &expLids = itr->second;
+        for (size_t i = 0; i < expLids.size(); ++i) {
+            EXPECT_TRUE(dms.remove(expLids[i]));
+            dms.removeComplete(expLids[i]);
+        }
+        LOG(info, "Verify that bucket id '%s' has 0 lids", bucketId.toString().c_str());
+        LidVector actLids;
+        dms.getLids(bucketId, actLids);
+        EXPECT_TRUE(actLids.empty());
+    }
+}
+
+struct Comparator {
+    bool operator() (const DocumentMetaData &lhs, const DocumentMetaData &rhs) const {
+        return lhs.lid < rhs.lid;
+    }
+};
+
+struct UserDocFixture {
+    std::shared_ptr<BucketDBOwner> _bucketDB;
+    DocumentMetaStore dms;
+    std::vector<GlobalId> gids;
+    BucketId bid1;
+    BucketId bid2;
+    BucketId bid3;
+    bucketdb::BucketDBHandler _bucketDBHandler;
+    UserDocFixture()
+        : _bucketDB(createBucketDB()),
+          dms(_bucketDB), gids(), bid1(), bid2(), bid3(),
+          _bucketDBHandler(*_bucketDB)
+    {
+        _bucketDBHandler.addDocumentMetaStore(&dms, 0);
+        gids.push_back(createGid(10, 1));
+        gids.push_back(createGid(10, 2));
+        gids.push_back(createGid(20, 3));
+        gids.push_back(createGid(10, 4));
+        gids.push_back(createGid(10, 5));
+        gids.push_back(createGid(20, 6));
+        gids.push_back(createGid(20, 7));
+        gids.push_back(createGid(30, 8)); // extra
+        gids.push_back(createGid(10, 9)); // extra
+        // 3 users -> 3 buckets
+        bid1 = BucketId(minNumBits, gids[0].convertToBucketId().getRawId());
+        bid2 = BucketId(minNumBits, gids[2].convertToBucketId().getRawId());
+        bid3 = BucketId(minNumBits, gids[7].convertToBucketId().getRawId());
+    }
+    void addGlobalId(const GlobalId &gid, uint32_t expLid, uint32_t timestampConst = 100) {
+        uint32_t actLid = addGid(dms, gid, Timestamp(expLid + timestampConst));
+        EXPECT_EQUAL(expLid, actLid);
+    }
+    void putGlobalId(const GlobalId &gid, uint32_t lid, uint32_t timestampConst = 100) {
+        putGid(dms, gid, lid, Timestamp(lid + timestampConst));
+    }
+    void addGlobalIds(size_t numGids=7) __attribute__((noinline));
+};
+
+void
+UserDocFixture::addGlobalIds(size_t numGids) {
+    for (size_t i = 0; i < numGids; ++i) {
+        uint32_t expLid = i + 1;
+        addGlobalId(gids[i], expLid);
+    }
+}
+
+TEST("requireThatWeCanRetrieveListOfMetaDataFromBucketId")
+{
+    UserDocFixture f;
+    { // empty bucket
+        DocumentMetaData::Vector result;
+        f.dms.getMetaData(f.bid1, result);
+        EXPECT_EQUAL(0u, result.size());
+    }
+    f.dms.constructFreeList();
+    f.addGlobalIds();
+    { // verify bucket 1
+        DocumentMetaData::Vector result;
+        f.dms.getMetaData(f.bid1, result);
+        std::sort(result.begin(), result.end(), Comparator());
+        EXPECT_EQUAL(4u, result.size());
+        EXPECT_TRUE(assertMetaData(DocumentMetaData(1, Timestamp(101), f.bid1,
+                                                    f.gids[0]), result[0]));
+        EXPECT_TRUE(assertMetaData(DocumentMetaData(2, Timestamp(102), f.bid1,
+                                                    f.gids[1]), result[1]));
+        EXPECT_TRUE(assertMetaData(DocumentMetaData(4, Timestamp(104), f.bid1,
+                                                    f.gids[3]), result[2]));
+        EXPECT_TRUE(assertMetaData(DocumentMetaData(5, Timestamp(105), f.bid1,
+                                                    f.gids[4]), result[3]));
+    }
+    { // verify bucket 2
+        DocumentMetaData::Vector result;
+        f.dms.getMetaData(f.bid2, result);
+        std::sort(result.begin(), result.end(), Comparator());
+        EXPECT_EQUAL(3u, result.size());
+        EXPECT_TRUE(assertMetaData(DocumentMetaData(3, Timestamp(103), f.bid2,
+                                                    f.gids[2]), result[0]));
+        EXPECT_TRUE(assertMetaData(DocumentMetaData(6, Timestamp(106), f.bid2,
+                                                    f.gids[5]), result[1]));
+        EXPECT_TRUE(assertMetaData(DocumentMetaData(7, Timestamp(107), f.bid2,
+                                                    f.gids[6]), result[2]));
+    }
+}
+
+TEST("requireThatBucketStateCanBeUpdated")
+{
+    UserDocFixture f;
+    f.dms.constructFreeList();
+    EXPECT_EQUAL(1u, f.dms.getActiveLids().getNumDocs()); // lid 0 is reserved
+
+    f.addGlobalIds();
+    EXPECT_TRUE(assertActiveLids(BoolVector().F().F().F().F().F().F().F(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid1).isActive());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid2).isActive());
+
+    f.dms.setBucketState(f.bid1, true);
+    EXPECT_TRUE(assertActiveLids(BoolVector().T().T().F().T().T().F().F(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(4u, f.dms.getNumActiveLids());
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->get(f.bid1).isActive());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid2).isActive());
+
+    f.dms.setBucketState(f.bid2, true);
+    EXPECT_TRUE(assertActiveLids(BoolVector().T().T().T().T().T().T().T(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(7u, f.dms.getNumActiveLids());
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->get(f.bid1).isActive());
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->get(f.bid2).isActive());
+
+    f.addGlobalId(createGid(30, 8), 8);
+    f.addGlobalId(createGid(10, 9), 9); // bid1 is active so added document should be active as well
+    EXPECT_TRUE(assertActiveLids(BoolVector().T().T().T().T().T().T().T().F().T(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(8u, f.dms.getNumActiveLids());
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->get(f.bid1).isActive());
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->get(f.bid2).isActive());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid3).isActive());
+
+    f.dms.setBucketState(f.bid1, false);
+    EXPECT_TRUE(assertActiveLids(BoolVector().F().F().T().F().F().T().T().F().F(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(3u, f.dms.getNumActiveLids());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid1).isActive());
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->get(f.bid2).isActive());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid3).isActive());
+
+    f.dms.setBucketState(f.bid2, false);
+    EXPECT_TRUE(assertActiveLids(BoolVector().F().F().F().F().F().F().F().F().F(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid1).isActive());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid2).isActive());
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->get(f.bid3).isActive());
+}
+
+
+TEST("requireThatRemovedLidsAreClearedAsActive")
+{
+    UserDocFixture f;
+    f.dms.constructFreeList();
+    f.addGlobalIds(2);
+    f.dms.setBucketState(f.bid1, true);
+    EXPECT_TRUE(assertActiveLids(BoolVector().T().T(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(2u, f.dms.getNumActiveLids());
+    f.dms.remove(2);
+    f.dms.removeComplete(2);
+    EXPECT_TRUE(assertActiveLids(BoolVector().T().F(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(1u, f.dms.getNumActiveLids());
+    f.addGlobalId(f.gids[2], 2); // from bid2
+    EXPECT_TRUE(assertActiveLids(BoolVector().T().F(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(1u, f.dms.getNumActiveLids());
+    f.dms.remove(2);
+    f.dms.removeComplete(2);
+    f.addGlobalId(f.gids[3], 2); // from bid1
+    EXPECT_TRUE(assertActiveLids(BoolVector().T().T(), f.dms.getActiveLids()));
+    EXPECT_EQUAL(2u, f.dms.getNumActiveLids());
+}
+
+TEST("requireThatBlackListBlueprintIsCreated")
+{
+    UserDocFixture f;
+    f.dms.constructFreeList();
+    f.addGlobalIds();
+
+    f.dms.setBucketState(f.bid1, true);
+    EXPECT_TRUE(assertBlackList(SimpleResult().addHit(3).addHit(6).addHit(7),
+                                f.dms.createBlackListBlueprint(), true));
+
+    f.dms.setBucketState(f.bid2, true);
+    EXPECT_TRUE(assertBlackList(SimpleResult(),
+                                f.dms.createBlackListBlueprint(), true));
+}
+
+TEST("requireThatDocumentAndMetaEntryCountIsUpdated")
+{
+    UserDocFixture f;
+    f.dms.constructFreeList();
+    EXPECT_EQUAL(0u, f.dms.getBucketDB().takeGuard()->get(f.bid1).getDocumentCount());
+    EXPECT_EQUAL(0u, f.dms.getBucketDB().takeGuard()->get(f.bid1).getEntryCount());
+    EXPECT_EQUAL(0u, f.dms.getBucketDB().takeGuard()->get(f.bid2).getDocumentCount());
+    EXPECT_EQUAL(0u, f.dms.getBucketDB().takeGuard()->get(f.bid2).getEntryCount());
+    f.addGlobalIds();
+    EXPECT_EQUAL(4u, f.dms.getBucketDB().takeGuard()->get(f.bid1).getDocumentCount());
+    EXPECT_EQUAL(4u, f.dms.getBucketDB().takeGuard()->get(f.bid1).getEntryCount());
+    EXPECT_EQUAL(3u, f.dms.getBucketDB().takeGuard()->get(f.bid2).getDocumentCount());
+    EXPECT_EQUAL(3u, f.dms.getBucketDB().takeGuard()->get(f.bid2).getEntryCount());
+    f.dms.remove(3); // from bid2
+    f.dms.removeComplete(3);
+    EXPECT_EQUAL(4u, f.dms.getBucketDB().takeGuard()->get(f.bid1).getDocumentCount());
+    EXPECT_EQUAL(4u, f.dms.getBucketDB().takeGuard()->get(f.bid1).getEntryCount());
+    EXPECT_EQUAL(2u, f.dms.getBucketDB().takeGuard()->get(f.bid2).getDocumentCount());
+    EXPECT_EQUAL(2u, f.dms.getBucketDB().takeGuard()->get(f.bid2).getEntryCount());
+}
+
+TEST("requireThatEmptyBucketsAreRemoved")
+{
+    UserDocFixture f;
+    f.dms.constructFreeList();
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid1));
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid2));
+    f.addGlobalIds(3);
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid1));
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid2));
+    f.dms.remove(3); // from bid2
+    f.dms.removeComplete(3);
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid1));
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid2));
+    EXPECT_EQUAL(0u, f.dms.getBucketDB().takeGuard()->get(f.bid2).getEntryCount());
+    f._bucketDBHandler.handleDeleteBucket(f.bid2);
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid2));
+    f.dms.remove(1); // from bid1
+    f.dms.removeComplete(1);
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid1));
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid2));
+    f.dms.remove(2); // from bid1
+    f.dms.removeComplete(2);
+    EXPECT_TRUE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid1));
+    EXPECT_EQUAL(0u, f.dms.getBucketDB().takeGuard()->get(f.bid1).getEntryCount());
+    f._bucketDBHandler.handleDeleteBucket(f.bid1);
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid1));
+    EXPECT_FALSE(f.dms.getBucketDB().takeGuard()->hasBucket(f.bid2));
+}
+
+
+struct GlobalIdEntry {
+    uint32_t lid;
+    GlobalId gid;
+    BucketId bid1;
+    BucketId bid2;
+    BucketId bid3;
+    GlobalIdEntry(uint32_t lid_) :
+        lid(lid_),
+        gid(createGid(lid_)),
+        bid1(1, gid.convertToBucketId().getRawId()),
+        bid2(2, gid.convertToBucketId().getRawId()),
+        bid3(3, gid.convertToBucketId().getRawId())
+    {}
+};
+
+typedef std::vector<GlobalIdEntry> GlobalIdVector;
+
+struct SplitAndJoinEmptyFixture
+{
+    DocumentMetaStore dms;
+    BucketId          bid10;
+    BucketId          bid11;
+    BucketId          bid20; // contained in bid10
+    BucketId          bid21; // contained in bid11
+    BucketId          bid22; // contained in bid10
+    BucketId          bid23; // contained in bid11
+    BucketId          bid30; // contained in bid10 and bid20
+    BucketId          bid32; // contained in bid10 and bid22
+    BucketId          bid34; // contained in bid10 and bid20
+    BucketId          bid36; // contained in bid10 and bid22
+    bucketdb::BucketDBHandler _bucketDBHandler;
+
+    SplitAndJoinEmptyFixture(void)
+        : dms(createBucketDB()),
+          bid10(1, 0), bid11(1, 1),
+          bid20(2, 0), bid21(2, 1), bid22(2, 2), bid23(2, 3),
+          bid30(3, 0), bid32(3, 2), bid34(3, 4), bid36(3, 6),
+          _bucketDBHandler(dms.getBucketDB())
+    {
+        _bucketDBHandler.addDocumentMetaStore(&dms, 0);
+    }
+
+    BucketInfo
+    getInfo(const BucketId &bid) const
+    {
+        return dms.getBucketDB().takeGuard()->get(bid);
+    }
+};
+
+struct SplitAndJoinFixture : public SplitAndJoinEmptyFixture
+{
+    typedef std::map<BucketId, GlobalIdVector> BucketMap;
+    GlobalIdVector    gids;
+    BucketMap         bid1s;
+    BucketMap         bid2s;
+    BucketMap         bid3s;
+    const GlobalIdVector *bid10Gids;
+    const GlobalIdVector *bid11Gids;
+    const GlobalIdVector *bid21Gids;
+    const GlobalIdVector *bid23Gids;
+    const GlobalIdVector *bid30Gids;
+    const GlobalIdVector *bid32Gids;
+    SplitAndJoinFixture()
+        : SplitAndJoinEmptyFixture(),
+          gids(),
+          bid1s(), bid2s(), bid3s(),
+          bid10Gids(), bid11Gids(), bid21Gids(), bid23Gids(),
+          bid30Gids(), bid32Gids()
+    {
+        for (uint32_t i = 1; i <= 31; ++i) {
+            gids.push_back(GlobalIdEntry(i));
+            bid1s[gids.back().bid1].push_back(gids.back());
+            bid2s[gids.back().bid2].push_back(gids.back());
+            bid3s[gids.back().bid3].push_back(gids.back());
+        }
+        ASSERT_EQUAL(2u, bid1s.size());
+        ASSERT_EQUAL(4u, bid2s.size());
+        ASSERT_EQUAL(8u, bid3s.size());
+        bid10Gids = &bid1s[bid10];
+        bid11Gids = &bid1s[bid11];
+        bid21Gids = &bid2s[bid21];
+        bid23Gids = &bid2s[bid23];
+        bid30Gids = &bid3s[bid30];
+        bid32Gids = &bid3s[bid32];
+    }
+    void insertGids1() {
+        for (size_t i = 0; i < gids.size(); ++i) {
+            EXPECT_TRUE(dms.put(gids[i].gid, gids[i].bid1, Timestamp(0),
+                                gids[i].lid).ok());
+        }
+    }
+    void insertGids2() {
+        for (size_t i = 0; i < gids.size(); ++i) {
+            EXPECT_TRUE(dms.put(gids[i].gid, gids[i].bid2, Timestamp(0),
+                                gids[i].lid).ok());
+        }
+    }
+
+    void
+    insertGids1Mostly(const BucketId &alt)
+    {
+        for (size_t i = 0; i < gids.size(); ++i) {
+            const GlobalIdEntry &g(gids[i]);
+            BucketId b(g.bid3 == alt ? g.bid2 : g.bid1);
+            EXPECT_TRUE(dms.put(g.gid, b, Timestamp(0),  g.lid).ok());
+        }
+    }
+
+    void
+    insertGids2Mostly(const BucketId &alt)
+    {
+        for (size_t i = 0; i < gids.size(); ++i) {
+            const GlobalIdEntry &g(gids[i]);
+            BucketId b(g.bid3 == alt ? g.bid1 : g.bid2);
+            EXPECT_TRUE(dms.put(g.gid, b, Timestamp(0),  g.lid).ok());
+        }
+    }
+};
+
+
+BoolVector
+getBoolVector(const GlobalIdVector &gids, size_t sz)
+{
+    BoolVector retval(sz);
+    for (size_t i = 0; i < gids.size(); ++i) {
+        uint32_t lid(gids[i].lid);
+        ASSERT_TRUE(lid <= sz && lid > 0u);
+        retval[lid - 1] = true;
+    }
+    return retval;
+}
+
+
+BoolVector
+getBoolVectorFiltered(const GlobalIdVector &gids, size_t sz,
+                      const BucketId &skip)
+{
+    BoolVector retval(sz);
+    for (size_t i = 0; i < gids.size(); ++i) {
+        const GlobalIdEntry &g(gids[i]);
+        uint32_t lid(g.lid);
+        ASSERT_TRUE(lid <= sz && lid > 0u);
+        if (g.bid3 == skip)
+            continue;
+        retval[lid - 1] = true;
+    }
+    return retval;
+}
+
+TEST("requireThatBucketInfoIsCorrectAfterSplit")
+{
+    SplitAndJoinFixture f;
+    f.insertGids1();
+    BucketInfo bi10 = f.getInfo(f.bid10);
+    BucketInfo bi11 = f.getInfo(f.bid11);
+    LOG(info, "%s: %s", f.bid10.toString().c_str(), bi10.toString().c_str());
+    LOG(info, "%s: %s", f.bid11.toString().c_str(), bi11.toString().c_str());
+    EXPECT_TRUE(assertBucketInfo(f.bid10Gids->size(), f.bid10Gids->size(), bi10));
+    EXPECT_TRUE(assertBucketInfo(f.bid11Gids->size(), f.bid11Gids->size(), bi11));
+    EXPECT_NOT_EQUAL(bi10.getEntryCount(), bi11.getEntryCount());
+    EXPECT_EQUAL(31u, bi10.getEntryCount() + bi11.getEntryCount());
+
+    f._bucketDBHandler.handleSplit(10, f.bid11, f.bid21, f.bid23);
+
+    BucketInfo nbi10 = f.getInfo(f.bid10);
+    BucketInfo nbi11 = f.getInfo(f.bid11);
+    BucketInfo bi21 = f.getInfo(f.bid21);
+    BucketInfo bi23 = f.getInfo(f.bid23);
+    LOG(info, "%s: %s", f.bid10.toString().c_str(), nbi10.toString().c_str());
+    LOG(info, "%s: %s", f.bid11.toString().c_str(), nbi11.toString().c_str());
+    LOG(info, "%s: %s", f.bid21.toString().c_str(), bi21.toString().c_str());
+    LOG(info, "%s: %s", f.bid23.toString().c_str(), bi23.toString().c_str());
+    EXPECT_TRUE(assertBucketInfo(f.bid10Gids->size(),
+                                 f.bid10Gids->size(),
+                                 nbi10));
+    EXPECT_TRUE(assertBucketInfo(0u, 0u, nbi11));
+    EXPECT_TRUE(assertBucketInfo(f.bid21Gids->size(),
+                                 f.bid21Gids->size(),
+                                 bi21));
+    EXPECT_TRUE(assertBucketInfo(f.bid23Gids->size(),
+                                 f.bid23Gids->size(),
+                                 bi23));
+    EXPECT_EQUAL(bi11.getEntryCount(),
+                 bi21.getEntryCount() + bi23.getEntryCount());
+    EXPECT_EQUAL(bi11.getDocumentCount(),
+                 bi21.getDocumentCount() +
+                 bi23.getDocumentCount());
+}
+
+TEST("requireThatActiveStateIsPreservedAfterSplit")
+{
+    { // non-active bucket
+        SplitAndJoinFixture f;
+        f.insertGids1();
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        f._bucketDBHandler.handleSplit(10, f.bid10, f.bid20, f.bid22);
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+        EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    }
+    { // active bucket
+        SplitAndJoinFixture f;
+        f.insertGids1();
+        f.dms.setBucketState(f.bid10, true);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        f._bucketDBHandler.handleSplit(10, f.bid10, f.bid20, f.bid22);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    }
+    { // non-active source, active overlapping target1
+        SplitAndJoinFixture f;
+        f.insertGids1Mostly(f.bid30);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+        EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+        f.dms.setBucketState(f.bid20, true);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        assertActiveLids(getBoolVector(*f.bid30Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid30Gids->size(), f.dms.getNumActiveLids());
+        f._bucketDBHandler.handleSplit(10, f.bid10, f.bid20, f.bid22);
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+        EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    }
+    { // non-active source, active overlapping target2
+        SplitAndJoinFixture f;
+        f.insertGids1Mostly(f.bid32);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+        EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+        f.dms.setBucketState(f.bid22, true);
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(getBoolVector(*f.bid32Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid32Gids->size(), f.dms.getNumActiveLids());
+        f._bucketDBHandler.handleSplit(10, f.bid10, f.bid20, f.bid22);
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+        EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    }
+    { // active source, non-active overlapping target1
+        SplitAndJoinFixture f;
+        f.insertGids1Mostly(f.bid30);
+        f.dms.setBucketState(f.bid10, true);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        BoolVector filtered(getBoolVectorFiltered(*f.bid10Gids, 31, f.bid30));
+        assertActiveLids(filtered, f.dms.getActiveLids());
+        EXPECT_EQUAL(filtered.countTrue(), f.dms.getNumActiveLids());
+        f._bucketDBHandler.handleSplit(10, f.bid10, f.bid20, f.bid22);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    }
+    { // active source, non-active overlapping target2
+        SplitAndJoinFixture f;
+        f.insertGids1Mostly(f.bid32);
+        f.dms.setBucketState(f.bid10, true);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        BoolVector filtered(getBoolVectorFiltered(*f.bid10Gids, 31, f.bid32));
+        assertActiveLids(filtered, f.dms.getActiveLids());
+        EXPECT_EQUAL(filtered.countTrue(), f.dms.getNumActiveLids());
+        f._bucketDBHandler.handleSplit(10, f.bid10, f.bid20, f.bid22);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    }
+}
+
+TEST("requireThatActiveStateIsPreservedAfterEmptySplit")
+{
+    { // non-active bucket
+        SplitAndJoinEmptyFixture f;
+        f._bucketDBHandler.handleCreateBucket(f.bid10);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        f._bucketDBHandler.handleSplit(10, f.bid10, f.bid20, f.bid22);
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+    }
+    { // active bucket
+        SplitAndJoinEmptyFixture f;
+        f._bucketDBHandler.handleCreateBucket(f.bid10);
+        f.dms.setBucketState(f.bid10, true);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        f._bucketDBHandler.handleSplit(10, f.bid10, f.bid20, f.bid22);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+    }
+}
+
+TEST("requireThatBucketInfoIsCorrectAfterJoin")
+{
+    SplitAndJoinFixture f;
+    f.insertGids2();
+    BucketInfo bi21 = f.getInfo(f.bid21);
+    BucketInfo bi23 = f.getInfo(f.bid23);
+    LOG(info, "%s: %s", f.bid21.toString().c_str(), bi21.toString().c_str());
+    LOG(info, "%s: %s", f.bid23.toString().c_str(), bi23.toString().c_str());
+    EXPECT_TRUE(assertBucketInfo(f.bid21Gids->size(), f.bid21Gids->size(), bi21));
+    EXPECT_TRUE(assertBucketInfo(f.bid23Gids->size(), f.bid23Gids->size(), bi23));
+    EXPECT_NOT_EQUAL(bi21.getEntryCount(), bi23.getEntryCount());
+    EXPECT_EQUAL(f.bid11Gids->size(), bi21.getEntryCount() + bi23.getEntryCount());
+
+    f._bucketDBHandler.handleJoin(10, f.bid21, f.bid23, f.bid11);
+    BucketInfo bi11 = f.getInfo(f.bid11);
+    BucketInfo nbi21 = f.getInfo(f.bid21);
+    BucketInfo nbi23 = f.getInfo(f.bid23);
+    LOG(info, "%s: %s", f.bid11.toString().c_str(), bi11.toString().c_str());
+    LOG(info, "%s: %s", f.bid21.toString().c_str(), nbi21.toString().c_str());
+    LOG(info, "%s: %s", f.bid23.toString().c_str(), nbi23.toString().c_str());
+    EXPECT_TRUE(assertBucketInfo(f.bid11Gids->size(),
+                                 f.bid11Gids->size(), bi11));
+    EXPECT_TRUE(assertBucketInfo(0u, 0u, nbi21));
+    EXPECT_TRUE(assertBucketInfo(0u, 0u, nbi23));
+    EXPECT_EQUAL(bi21.getEntryCount() + bi23.getEntryCount(),
+                 bi11.getEntryCount());
+    EXPECT_EQUAL(bi21.getDocumentCount() +
+                 bi23.getDocumentCount(),
+                 bi11.getDocumentCount());
+}
+
+TEST("requireThatActiveStateIsPreservedAfterJoin")
+{
+    { // non-active buckets
+        SplitAndJoinFixture f;
+        f.insertGids2();
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+        EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    }
+    { // active buckets
+        SplitAndJoinFixture f;
+        f.insertGids2();
+        f.dms.setBucketState(f.bid20, true);
+        f.dms.setBucketState(f.bid22, true);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    }
+    { // 1 active bucket
+        SplitAndJoinFixture f;
+        f.insertGids2();
+        f.dms.setBucketState(f.bid20, true);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    }
+    { // 1 active bucket
+        SplitAndJoinFixture f;
+        f.insertGids2();
+        f.dms.setBucketState(f.bid22, true);
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    }
+    { // non-active buckets, active target
+        SplitAndJoinFixture f;
+        f.insertGids2Mostly(f.bid30);
+        f.dms.setBucketState(f.bid10, true);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(getBoolVector(*f.bid30Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid30Gids->size(), f.dms.getNumActiveLids());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+        EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    }
+    { // non-active buckets, active target
+        SplitAndJoinFixture f;
+        f.insertGids2Mostly(f.bid32);
+        f.dms.setBucketState(f.bid10, true);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+        assertActiveLids(getBoolVector(*f.bid32Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid32Gids->size(), f.dms.getNumActiveLids());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+        EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    }
+    { // active buckets, non-active target
+        SplitAndJoinFixture f;
+        f.insertGids2Mostly(f.bid30);
+        f.dms.setBucketState(f.bid20, true);
+        f.dms.setBucketState(f.bid22, true);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+        BoolVector filtered(getBoolVectorFiltered(*f.bid10Gids, 31, f.bid30));
+        assertActiveLids(filtered, f.dms.getActiveLids());
+        EXPECT_EQUAL(filtered.countTrue(), f.dms.getNumActiveLids());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    }
+    { // active buckets, non-active target
+        SplitAndJoinFixture f;
+        f.insertGids2Mostly(f.bid32);
+        f.dms.setBucketState(f.bid20, true);
+        f.dms.setBucketState(f.bid22, true);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+        BoolVector filtered(getBoolVectorFiltered(*f.bid10Gids, 31, f.bid32));
+        assertActiveLids(filtered, f.dms.getActiveLids());
+        EXPECT_EQUAL(filtered.countTrue(), f.dms.getNumActiveLids());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+        assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                         f.dms.getActiveLids());
+        EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    }
+}
+
+TEST("requireThatActiveStateIsPreservedAfterEmptyJoin")
+{
+    { // non-active buckets
+        SplitAndJoinEmptyFixture f;
+        f._bucketDBHandler.handleCreateBucket(f.bid20);
+        f._bucketDBHandler.handleCreateBucket(f.bid22);
+        EXPECT_FALSE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_FALSE(f.getInfo(f.bid10).isActive());
+    }
+    { // active buckets
+        SplitAndJoinEmptyFixture f;
+        f._bucketDBHandler.handleCreateBucket(f.bid20);
+        f._bucketDBHandler.handleCreateBucket(f.bid22);
+        f.dms.setBucketState(f.bid20, true);
+        f.dms.setBucketState(f.bid22, true);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_TRUE(f.getInfo(f.bid22).isActive());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+    }
+    { // 1 active bucket
+        SplitAndJoinEmptyFixture f;
+        f._bucketDBHandler.handleCreateBucket(f.bid20);
+        f._bucketDBHandler.handleCreateBucket(f.bid22);
+        f.dms.setBucketState(f.bid20, true);
+        EXPECT_TRUE(f.getInfo(f.bid20).isActive());
+        EXPECT_FALSE(f.getInfo(f.bid22).isActive());
+
+        f._bucketDBHandler.handleJoin(10, f.bid20, f.bid22, f.bid10);
+        EXPECT_TRUE(f.getInfo(f.bid10).isActive());
+    }
+}
+
+TEST("requireThatOverlappingBucketActiveStateWorks")
+{
+    SplitAndJoinFixture f;
+    f.insertGids1Mostly(f.bid30);
+    assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+    EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+    f.dms.setBucketState(f.bid10, true);
+    BoolVector filtered(getBoolVectorFiltered(*f.bid10Gids, 31, f.bid30));
+    assertActiveLids(filtered, f.dms.getActiveLids());
+    EXPECT_EQUAL(filtered.countTrue(), f.dms.getNumActiveLids());
+    f.dms.setBucketState(f.bid20, true);
+    assertActiveLids(getBoolVector(*f.bid10Gids, 31),
+                     f.dms.getActiveLids());
+    EXPECT_EQUAL(f.bid10Gids->size(), f.dms.getNumActiveLids());
+    f.dms.setBucketState(f.bid10, false);
+    assertActiveLids(getBoolVector(*f.bid30Gids, 31),
+                     f.dms.getActiveLids());
+    EXPECT_EQUAL(f.bid30Gids->size(), f.dms.getNumActiveLids());
+    f.dms.setBucketState(f.bid20, false);
+    assertActiveLids(BoolVector(31), f.dms.getActiveLids());
+    EXPECT_EQUAL(0u, f.dms.getNumActiveLids());
+}
+
+struct RemovedFixture
+{
+    std::shared_ptr<BucketDBOwner> _bucketDB;
+    DocumentMetaStore dms;
+    bucketdb::BucketDBHandler _bucketDBHandler;
+
+    RemovedFixture(void)
+        : _bucketDB(createBucketDB()),
+          dms(_bucketDB,
+              DocumentMetaStore::getFixedName(),
+              search::GrowStrategy(),
+              DocumentMetaStore::IGidCompare::SP(new DocumentMetaStore::DefaultGidCompare),
+              SubDbType::REMOVED),
+          _bucketDBHandler(dms.getBucketDB())
+    {
+        _bucketDBHandler.addDocumentMetaStore(&dms, 0);
+    }
+
+    BucketInfo
+    getInfo(const BucketId &bid) const
+    {
+        return dms.getBucketDB().takeGuard()->get(bid);
+    }
+};
+
+TEST("requireThatRemoveChangedBucketWorks")
+{
+    RemovedFixture f;
+    GlobalIdEntry g(1);
+    f.dms.constructFreeList();
+    f._bucketDBHandler.handleCreateBucket(g.bid1);
+    uint32_t addLid1 = addGid(f.dms, g.gid, g.bid1, Timestamp(0));
+    EXPECT_EQUAL(1u, addLid1);
+    uint32_t addLid2 = addGid(f.dms, g.gid, g.bid2, Timestamp(0));
+    EXPECT_TRUE(1u == addLid2);
+    EXPECT_TRUE(f.dms.remove(1u));
+    f.dms.removeComplete(1u);
+}
+
+TEST("requireThatGetLidUsageStatsWorks")
+{
+    DocumentMetaStore dms(createBucketDB());
+    dms.constructFreeList();
+
+    LidUsageStats s = dms.getLidUsageStats();
+    EXPECT_EQUAL(1u, s.getLidLimit());
+    EXPECT_EQUAL(0u, s.getUsedLids());
+    EXPECT_EQUAL(1u, s.getLowestFreeLid());
+    EXPECT_EQUAL(0u, s.getHighestUsedLid());
+
+    putGid(dms, createGid(1), 1);
+    
+    s = dms.getLidUsageStats();
+    EXPECT_EQUAL(2u, s.getLidLimit());
+    EXPECT_EQUAL(1u, s.getUsedLids());
+    EXPECT_EQUAL(2u, s.getLowestFreeLid());
+    EXPECT_EQUAL(1u, s.getHighestUsedLid());
+
+    putGid(dms, createGid(2), 2);
+
+    s = dms.getLidUsageStats();
+    EXPECT_EQUAL(3u, s.getLidLimit());
+    EXPECT_EQUAL(2u, s.getUsedLids());
+    EXPECT_EQUAL(3u, s.getLowestFreeLid());
+    EXPECT_EQUAL(2u, s.getHighestUsedLid());
+
+
+    putGid(dms, createGid(3), 3);
+
+    s = dms.getLidUsageStats();
+    EXPECT_EQUAL(4u, s.getLidLimit());
+    EXPECT_EQUAL(3u, s.getUsedLids());
+    EXPECT_EQUAL(4u, s.getLowestFreeLid());
+    EXPECT_EQUAL(3u, s.getHighestUsedLid());
+
+    dms.remove(1);
+    dms.removeComplete(1);
+    
+    s = dms.getLidUsageStats();
+    EXPECT_EQUAL(4u, s.getLidLimit());
+    EXPECT_EQUAL(2u, s.getUsedLids());
+    EXPECT_EQUAL(1u, s.getLowestFreeLid());
+    EXPECT_EQUAL(3u, s.getHighestUsedLid());
+
+    dms.remove(3);
+    dms.removeComplete(3);
+    
+    s = dms.getLidUsageStats();
+    EXPECT_EQUAL(4u, s.getLidLimit());
+    EXPECT_EQUAL(1u, s.getUsedLids());
+    EXPECT_EQUAL(1u, s.getLowestFreeLid());
+    EXPECT_EQUAL(2u, s.getHighestUsedLid());
+
+    dms.remove(2);
+    dms.removeComplete(2);
+    
+    s = dms.getLidUsageStats();
+    EXPECT_EQUAL(4u, s.getLidLimit());
+    EXPECT_EQUAL(0u, s.getUsedLids());
+    EXPECT_EQUAL(1u, s.getLowestFreeLid());
+    EXPECT_EQUAL(0u, s.getHighestUsedLid());
+}
+
+bool
+assertLidBloat(uint32_t expBloat, uint32_t lidLimit, uint32_t usedLids)
+{
+    LidUsageStats stats(lidLimit, usedLids, 0, 0);
+    return  EXPECT_EQUAL(expBloat, stats.getLidBloat());
+}
+
+TEST("require that LidUsageStats::getLidBloat() works")
+{
+    assertLidBloat(4, 10, 5);
+    assertLidBloat(0, 1, 0);
+    assertLidBloat(0, 1, 1);
+}
+
+TEST("requireThatMoveWorks")
+{
+    DocumentMetaStore dms(createBucketDB());
+    GlobalId gid;
+    uint32_t lid = 0u;
+    dms.constructFreeList();
+    
+    EXPECT_EQUAL(1u, dms.getNumDocs());
+    EXPECT_EQUAL(0u, dms.getNumUsedLids());
+    EXPECT_TRUE(assertPut(bucketId1, time1, 1u, gid1, dms));
+    EXPECT_EQUAL(2u, dms.getNumDocs());
+    EXPECT_EQUAL(1u, dms.getNumUsedLids());
+    EXPECT_TRUE(assertPut(bucketId2, time2, 2u, gid2, dms));
+    EXPECT_EQUAL(3u, dms.getNumDocs());
+    EXPECT_EQUAL(2u, dms.getNumUsedLids());
+    EXPECT_TRUE(dms.getGid(1u, gid));
+    EXPECT_TRUE(dms.getLid(gid2, lid));
+    EXPECT_EQUAL(gid1, gid);
+    EXPECT_EQUAL(2u, lid);
+    EXPECT_TRUE(dms.remove(1));
+    dms.removeComplete(1u);
+    EXPECT_EQUAL(1u, dms.getNumUsedLids());
+    dms.move(2u, 1u);
+    dms.removeComplete(2u);
+    EXPECT_TRUE(dms.getGid(1u, gid));
+    EXPECT_TRUE(dms.getLid(gid2, lid));
+    EXPECT_EQUAL(gid2, gid);
+    EXPECT_EQUAL(1u, lid);
+}
+
+bool
+assertLidSpace(uint32_t numDocs,
+               uint32_t committedDocIdLimit,
+               uint32_t numUsedLids,
+               bool wantShrinkLidSpace,
+               bool canShrinkLidSpace,
+               const DocumentMetaStore &dms)
+{
+    if (!EXPECT_EQUAL(numDocs, dms.getNumDocs())) return false;
+    if (!EXPECT_EQUAL(committedDocIdLimit, dms.getCommittedDocIdLimit())) return false;
+    if (!EXPECT_EQUAL(numUsedLids, dms.getNumUsedLids())) return false;
+    if (!EXPECT_EQUAL(wantShrinkLidSpace, dms.wantShrinkLidSpace())) return false;
+    if (!EXPECT_EQUAL(canShrinkLidSpace, dms.canShrinkLidSpace())) return false;
+    return true;
+}
+
+void
+populate(uint32_t endLid, DocumentMetaStore &dms)
+{
+    for (uint32_t lid = 1; lid < endLid; ++lid) {
+        GlobalId gid = createGid(lid);
+        putGid(dms, gid, lid, Timestamp(10000 + lid));
+    }
+    EXPECT_TRUE(assertLidSpace(endLid, endLid, endLid - 1, false, false, dms));
+}
+
+void
+remove(uint32_t startLid, uint32_t shrinkTarget, DocumentMetaStore &dms)
+{
+    for (uint32_t lid = startLid; lid >= shrinkTarget; --lid) {
+        dms.remove(lid);
+        dms.removeComplete(lid);
+    }
+}
+
+TEST("requireThatShrinkWorks")
+{
+    DocumentMetaStore dms(createBucketDB());
+    dms.constructFreeList();
+
+    populate(10, dms);
+
+    uint32_t shrinkTarget = 5;
+    remove(9, shrinkTarget, dms);
+    EXPECT_TRUE(assertLidSpace(10, 10, shrinkTarget - 1, false, false, dms));
+
+    dms.compactLidSpace(shrinkTarget);
+    EXPECT_TRUE(assertLidSpace(10, shrinkTarget, shrinkTarget - 1, true, false, dms));
+
+    dms.holdUnblockShrinkLidSpace();
+    EXPECT_TRUE(assertLidSpace(10, shrinkTarget, shrinkTarget - 1, true, true, dms));
+
+    dms.shrinkLidSpace();
+    EXPECT_TRUE(assertLidSpace(shrinkTarget, shrinkTarget, shrinkTarget - 1, false, false, dms));
+}
+
+
+TEST("requireThatShrinkViaFlushTargetWorks")
+{
+    DocumentMetaStore::SP dms(new DocumentMetaStore(createBucketDB()));
+    dms->constructFreeList();
+    TuneFileAttributes tuneFileAttributes;
+    DummyFileHeaderContext fileHeaderContext;
+    DummyTlsSyncer dummyTlsSyncer;
+    vespalib::rmdir("dmsflush", true);
+    vespalib::mkdir("dmsflush");
+    IFlushTarget::SP ft(new DocumentMetaStoreFlushTarget(dms,
+                                                         dummyTlsSyncer,
+                                                         "dmsflush",
+                                                         tuneFileAttributes,
+                                                         fileHeaderContext));
+    
+    populate(10, *dms);
+
+    uint32_t shrinkTarget = 5;
+    remove(9, shrinkTarget, *dms);
+    EXPECT_TRUE(assertLidSpace(10, 10, shrinkTarget - 1, false, false, *dms));
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+
+    dms->compactLidSpace(shrinkTarget);
+    EXPECT_TRUE(assertLidSpace(10, shrinkTarget, shrinkTarget - 1, true, false, *dms));
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+    AttributeGuard::UP g(new AttributeGuard(dms));
+
+    dms->holdUnblockShrinkLidSpace();
+    EXPECT_TRUE(assertLidSpace(10, shrinkTarget, shrinkTarget - 1, true, false, *dms));
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+
+    g.reset();
+    dms->removeAllOldGenerations();
+    EXPECT_TRUE(assertLidSpace(10, shrinkTarget, shrinkTarget - 1, true, true, *dms));
+    EXPECT_TRUE(ft->getApproxMemoryGain().getBefore() >
+                ft->getApproxMemoryGain().getAfter());
+
+    vespalib::ThreadStackExecutor exec(1, 128 * 1024);
+    vespalib::Executor::Task::UP task = ft->initFlush(11);
+    exec.execute(std::move(task));
+    exec.sync();
+    exec.shutdown();
+    EXPECT_TRUE(assertLidSpace(shrinkTarget, shrinkTarget, shrinkTarget - 1, false, false, *dms));
+    EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
+                 ft->getApproxMemoryGain().getAfter());
+}
+
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh
new file mode 100644
index 00000000000..03fb18363f1
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+$VALGRIND ./searchcore_documentmetastore_test_app
+rm -rf documentmetastore*.dat
+rm -rf dmsflush
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/.gitignore b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/.gitignore
new file mode 100644
index 00000000000..3af6b83638f
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/.gitignore
@@ -0,0 +1 @@
+searchcore_lidreusedelayer_test_app
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt
new file mode 100644
index 00000000000..5e333c74b2f
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_lidreusedelayer_test_app
+    SOURCES
+    lidreusedelayer_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_documentmetastore
+)
+vespa_add_test(NAME searchcore_lidreusedelayer_test_app COMMAND searchcore_lidreusedelayer_test_app)
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/DESC b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/DESC
new file mode 100644
index 00000000000..73e755c73c6
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/DESC
@@ -0,0 +1 @@
+LID reuse delayer test. Take a look at lidreusedelayer_test.cpp for details.
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/FILES b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/FILES
new file mode 100644
index 00000000000..4965f6577f6
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/FILES
@@ -0,0 +1 @@
+lidreusedelayer_test.cpp
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
new file mode 100644
index 00000000000..ceecaad7d5b
--- /dev/null
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
@@ -0,0 +1,325 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("lidreusedelayer_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/documentmetastore/i_store.h>
+#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+#include <vespa/searchcore/proton/test/thread_utils.h>
+#include <vespa/searchcore/proton/test/threading_service_observer.h>
+#include <vespa/searchlib/common/lambdatask.h>
+
+using search::makeLambdaTask;
+
+namespace proton {
+
+namespace
+{
+
+bool
+assertThreadObserver(uint32_t masterExecuteCnt,
+                     uint32_t indexExecuteCnt,
+                     const test::ThreadingServiceObserver &observer)
+{
+    if (!EXPECT_EQUAL(masterExecuteCnt,
+                      observer.masterObserver().getExecuteCnt())) {
+        return false;
+    }
+    if (!EXPECT_EQUAL(indexExecuteCnt,
+                      observer.indexObserver().getExecuteCnt())) {
+        return false;
+    }
+    return true;
+}
+
+}
+
+class MyMetaStore : public documentmetastore::IStore
+{
+public:
+    bool     _freeListActive;
+    uint32_t _removeCompleteCount;
+    uint32_t _removeBatchCompleteCount;
+    uint32_t _removeCompleteLids;
+
+    MyMetaStore()
+        : _freeListActive(false),
+          _removeCompleteCount(0),
+          _removeBatchCompleteCount(0),
+          _removeCompleteLids(0)
+    {
+    }
+
+    virtual ~MyMetaStore() { }
+
+    virtual Result inspectExisting(const GlobalId &) const override
+    {
+        return Result();
+    }
+
+    virtual Result inspect(const GlobalId &) override
+    {
+        return Result();
+    }
+
+    virtual Result put(const GlobalId &, const BucketId &, const Timestamp &,
+                       DocId) override
+    {
+        return Result();
+    }
+
+    virtual bool updateMetaData(DocId, const BucketId &,
+                                const Timestamp &) override
+    {
+        return true;
+    }
+
+    virtual bool remove(DocId) override
+    {
+        return true;
+    }
+
+    virtual void removeComplete(DocId) override
+    {
+        ++_removeCompleteCount;
+        ++_removeCompleteLids;
+    }
+
+    virtual void move(DocId, DocId) override
+    {
+    }
+
+    virtual bool validLid(DocId) const override
+    {
+        return true;
+    }
+
+    virtual void removeBatch(const std::vector<DocId> &,
+                             const DocId) override
+    {
+    }
+
+    virtual void
+    removeBatchComplete(const std::vector<DocId> &lidsToRemove) override
+    {
+        ++_removeBatchCompleteCount;
+        _removeCompleteLids += lidsToRemove.size();
+    }
+
+    virtual const RawDocumentMetaData &getRawMetaData(DocId) const override
+    {
+        abort();
+    }
+
+    virtual bool getFreeListActive() const override
+    {
+        return _freeListActive;
+    }
+
+    bool
+    assertWork(uint32_t expRemoveCompleteCount,
+               uint32_t expRemoveBatchCompleteCount,
+               uint32_t expRemoveCompleteLids)
+    {
+        if (!EXPECT_EQUAL(expRemoveCompleteCount, _removeCompleteCount)) {
+            return false;
+        }
+        if (!EXPECT_EQUAL(expRemoveBatchCompleteCount,
+                          _removeBatchCompleteCount)) {
+            return false;
+        }
+        if (!EXPECT_EQUAL(expRemoveCompleteLids, _removeCompleteLids)) {
+            return false;
+        }
+        return true;
+    }
+};
+
+class Fixture
+{
+public:
+    ExecutorThreadingService _writeServiceReal;
+    test::ThreadingServiceObserver _writeService;
+    MyMetaStore _store;
+    documentmetastore::LidReuseDelayer _lidReuseDelayer;
+
+    Fixture()
+        : _writeServiceReal(),
+          _writeService(_writeServiceReal),
+          _store(),
+          _lidReuseDelayer(_writeService, _store)
+    {
+    }
+
+    template <typename FunctionType>
+    void runInMaster(FunctionType func) {
+        test::runInMaster(_writeService, func);
+    }
+
+    void
+    cycledLids(const std::vector<uint32_t> &lids)
+    {
+        if (lids.size() == 1) {
+            _store.removeComplete(lids[0]);
+        } else {
+            _store.removeBatchComplete(lids);
+        }
+    }
+
+    void
+    performCycleLids(const std::vector<uint32_t> &lids)
+    {
+        _writeService.master().execute(
+                makeLambdaTask([=]() { cycledLids(lids);}));
+    }
+
+    void
+    cycleLids(const std::vector<uint32_t> &lids)
+    {
+        if (lids.empty())
+            return;
+        _writeService.index().execute(
+                makeLambdaTask([=]() { performCycleLids(lids);}));
+    }
+
+    bool
+    delayReuse(uint32_t lid)
+    {
+        bool res = false;
+        runInMaster([&] () { res = _lidReuseDelayer.delayReuse(lid); } );
+        return res;
+    }
+
+    bool
+    delayReuse(const std::vector<uint32_t> &lids)
+    {
+        bool res = false;
+        runInMaster([&] () { res = _lidReuseDelayer.delayReuse(lids); });
+        return res;
+    }
+
+    void setImmediateCommit(bool immediateCommit) {
+        runInMaster([&] () { _lidReuseDelayer.
+                                    setImmediateCommit(immediateCommit); } );
+    }
+
+    void setHasIndexedFields(bool hasIndexedFields) {
+        runInMaster([&] () { _lidReuseDelayer.
+                                    setHasIndexedFields(hasIndexedFields); } );
+    }
+
+    void commit() {
+        runInMaster([&] () { cycleLids(_lidReuseDelayer.getReuseLids()); });
+    }
+
+    void
+    sync()
+    {
+        _writeService.sync();
+    }
+
+    void
+    scheduleDelayReuseLid(uint32_t lid)
+    {
+        runInMaster([&] () { cycleLids({ lid }); });
+    }
+
+    void
+    scheduleDelayReuseLids(const std::vector<uint32_t> &lids)
+    {
+        runInMaster([&] () { cycleLids(lids); });
+    }
+};
+
+
+TEST_F("require that nothing happens before free list is active", Fixture)
+{
+    f.setHasIndexedFields(true);
+    EXPECT_FALSE(f.delayReuse(4));
+    EXPECT_FALSE(f.delayReuse({ 5, 6}));
+    EXPECT_TRUE(f._store.assertWork(0, 0, 0));
+    EXPECT_TRUE(assertThreadObserver(3, 0, f._writeService));
+}
+
+
+TEST_F("require that single lid is delayed", Fixture)
+{
+    f._store._freeListActive = true;
+    f.setHasIndexedFields(true);
+    EXPECT_TRUE(f.delayReuse(4));
+    f.scheduleDelayReuseLid(4);
+    EXPECT_TRUE(f._store.assertWork(1, 0, 1));
+    EXPECT_TRUE(assertThreadObserver(4, 1, f._writeService));
+}
+
+
+TEST_F("require that lid vector is delayed", Fixture)
+{
+    f._store._freeListActive = true;
+    f.setHasIndexedFields(true);
+    EXPECT_TRUE(f.delayReuse({ 5, 6, 7}));
+    f.scheduleDelayReuseLids({ 5, 6, 7});
+    EXPECT_TRUE(f._store.assertWork(0, 1, 3));
+    EXPECT_TRUE(assertThreadObserver(4, 1, f._writeService));
+}
+
+
+TEST_F("require that reuse can be batched", Fixture)
+{
+    f._store._freeListActive = true;
+    f.setHasIndexedFields(true);
+    f.setImmediateCommit(false);
+    EXPECT_FALSE(f.delayReuse(4));
+    EXPECT_FALSE(f.delayReuse({ 5, 6, 7}));
+    EXPECT_TRUE(f._store.assertWork(0, 0, 0));
+    EXPECT_TRUE(assertThreadObserver(4, 0, f._writeService));
+    f.commit();
+    EXPECT_TRUE(f._store.assertWork(0, 1, 4));
+    EXPECT_TRUE(assertThreadObserver(6, 1, f._writeService));
+    EXPECT_FALSE(f.delayReuse(8));
+    EXPECT_FALSE(f.delayReuse({ 9, 10}));
+    EXPECT_TRUE(f._store.assertWork(0, 1, 4));
+    EXPECT_TRUE(assertThreadObserver(8, 1, f._writeService));
+}
+
+
+TEST_F("require that single element array is optimized", Fixture)
+{
+    f._store._freeListActive = true;
+    f.setHasIndexedFields(true);
+    f.setImmediateCommit(false);
+    EXPECT_FALSE(f.delayReuse({ 4}));
+    EXPECT_TRUE(f._store.assertWork(0, 0, 0));
+    EXPECT_TRUE(assertThreadObserver(3, 0, f._writeService));
+    f.commit();
+    f.setImmediateCommit(true);
+    EXPECT_TRUE(f._store.assertWork(1, 0, 1));
+    EXPECT_TRUE(assertThreadObserver(6, 1, f._writeService));
+    EXPECT_TRUE(f.delayReuse({ 8}));
+    f.scheduleDelayReuseLids({ 8});
+    EXPECT_TRUE(f._store.assertWork(2, 0, 2));
+    EXPECT_TRUE(assertThreadObserver(9, 2, f._writeService));
+}
+
+
+TEST_F("require that lids are reused faster with no indexed fields", Fixture)
+{
+    f._store._freeListActive = true;
+    f.setHasIndexedFields(false);
+    EXPECT_FALSE(f.delayReuse(4));
+    EXPECT_TRUE(f._store.assertWork(1, 0, 1));
+    EXPECT_TRUE(assertThreadObserver(2, 0, f._writeService));
+    EXPECT_FALSE(f.delayReuse({ 5, 6, 7}));
+    EXPECT_TRUE(f._store.assertWork(1, 1, 4));
+    EXPECT_TRUE(assertThreadObserver(3, 0, f._writeService));
+}
+
+}
+
+
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/feed_and_search/.gitignore b/searchcore/src/tests/proton/feed_and_search/.gitignore
new file mode 100644
index 00000000000..83587d9de8b
--- /dev/null
+++ b/searchcore/src/tests/proton/feed_and_search/.gitignore
@@ -0,0 +1,8 @@
+.depend
+Makefile
+feed_and_search_test
+test_index
+test_index2
+test_index3
+test_index4
+searchcore_feed_and_search_test_app
diff --git a/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt b/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt
new file mode 100644
index 00000000000..b6d1258a386
--- /dev/null
+++ b/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_feed_and_search_test_app
+    SOURCES
+    feed_and_search.cpp
+    DEPENDS
+    searchcore_util
+)
+vespa_add_test(NAME searchcore_feed_and_search_test_app COMMAND searchcore_feed_and_search_test_app)
diff --git a/searchcore/src/tests/proton/feed_and_search/DESC b/searchcore/src/tests/proton/feed_and_search/DESC
new file mode 100644
index 00000000000..0ae466eb2c9
--- /dev/null
+++ b/searchcore/src/tests/proton/feed_and_search/DESC
@@ -0,0 +1 @@
+feed_and_search test. Take a look at feed_and_search.cpp for details.
diff --git a/searchcore/src/tests/proton/feed_and_search/FILES b/searchcore/src/tests/proton/feed_and_search/FILES
new file mode 100644
index 00000000000..cd160110672
--- /dev/null
+++ b/searchcore/src/tests/proton/feed_and_search/FILES
@@ -0,0 +1 @@
+feed_and_search.cpp
diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
new file mode 100644
index 00000000000..ccd80f6bdf3
--- /dev/null
+++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
@@ -0,0 +1,241 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("feed_and_search_test");
+
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/searchlib/memoryindex/memoryindex.h>
+#include <vespa/searchlib/diskindex/diskindex.h>
+#include <vespa/searchlib/diskindex/indexbuilder.h>
+#include <vespa/searchlib/fef/fef.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/searchlib/query/tree/simplequery.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/fake_requestcontext.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <sstream>
+#include <vespa/searchlib/diskindex/fusion.h>
+#include <vespa/searchlib/common/documentsummary.h>
+#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+
+using document::DataType;
+using document::Document;
+using document::FieldValue;
+using search::DocumentIdT;
+using search::TuneFileIndexing;
+using search::TuneFileSearch;
+using search::diskindex::DiskIndex;
+using search::diskindex::IndexBuilder;
+using search::diskindex::SelectorArray;
+using search::fef::FieldPositionsIterator;
+using search::fef::MatchData;
+using search::fef::MatchDataLayout;
+using search::fef::TermFieldHandle;
+using search::fef::TermFieldMatchData;
+using search::index::DocBuilder;
+using search::index::Schema;
+using search::index::DummyFileHeaderContext;
+using search::memoryindex::MemoryIndex;
+using search::query::SimpleStringTerm;
+using search::queryeval::Blueprint;
+using search::queryeval::FieldSpec;
+using search::queryeval::FieldSpecList;
+using search::queryeval::SearchIterator;
+using search::queryeval::Searchable;
+using search::queryeval::FakeRequestContext;
+using std::ostringstream;
+using vespalib::string;
+using search::docsummary::DocumentSummary;
+
+namespace {
+
+class Test : public vespalib::TestApp {
+    const char *current_state;
+    void DumpState(bool) {
+      fprintf(stderr, "%s: ERROR: in %s\n", GetName(), current_state);
+    }
+
+    void requireThatMemoryIndexCanBeDumpedAndSearched();
+
+    void testSearch(Searchable &source,
+                    const string &term, uint32_t doc_id);
+
+public:
+    int Main();
+};
+
+#define TEST_CALL(func) \
+    current_state = #func; \
+    func();
+
+int
+Test::Main()
+{
+    TEST_INIT("feed_and_search_test");
+
+    if (_argc > 0) {
+        DummyFileHeaderContext::setCreator(_argv[0]);
+    }
+    TEST_CALL(requireThatMemoryIndexCanBeDumpedAndSearched);
+
+    TEST_DONE();
+}
+
+const string field_name = "string_field";
+const string noise = "noise";
+const string word1 = "foo";
+const string word2 = "bar";
+const DocumentIdT doc_id1 = 1;
+const DocumentIdT doc_id2 = 2;
+const uint32_t field_id = 1;
+
+Schema getSchema() {
+    Schema schema;
+    schema.addIndexField(Schema::IndexField(field_name,
+                                            Schema::STRING));
+    return schema;
+}
+
+Document::UP buildDocument(DocBuilder & doc_builder, int id,
+                           const string &word) {
+    ostringstream ost;
+    ost << "doc::" << id;
+    doc_builder.startDocument(ost.str());
+    doc_builder.startIndexField(field_name)
+        .addStr(noise).addStr(word).endField();
+    return doc_builder.endDocument();
+}
+
+// Performs a search using a Searchable.
+void Test::testSearch(Searchable &source,
+                      const string &term, uint32_t doc_id)
+{
+    FakeRequestContext requestContext;
+    uint32_t fieldId = 0;
+    MatchDataLayout mdl;
+    TermFieldHandle handle = mdl.allocTermField(fieldId);
+    MatchData::UP match_data = mdl.createMatchData();
+
+    SimpleStringTerm node(term, field_name, 0, search::query::Weight(0));
+    Blueprint::UP result = source.createBlueprint(requestContext,
+            FieldSpecList().add(FieldSpec(field_name, 0, handle)), node);
+    result->fetchPostings(true);
+    SearchIterator::UP search_iterator =
+        result->createSearch(*match_data, true);
+    search_iterator->initFullRange();
+    ASSERT_TRUE(search_iterator.get());
+    ASSERT_TRUE(search_iterator->seek(doc_id));
+    EXPECT_EQUAL(doc_id, search_iterator->getDocId());
+    search_iterator->unpack(doc_id);
+    FieldPositionsIterator it =
+        match_data->resolveTermField(handle)->getIterator();
+    ASSERT_TRUE(it.valid());
+    EXPECT_EQUAL(1u, it.size());
+    EXPECT_EQUAL(1u, it.getPosition());  // All hits are at pos 1 in this index
+
+    EXPECT_TRUE(!search_iterator->seek(doc_id + 1));
+    EXPECT_TRUE(search_iterator->isAtEnd());
+}
+
+// Creates a memory index, inserts documents, performs a few
+// searches, dumps the index to disk, and performs the searches
+// again.
+void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
+    Schema schema = getSchema();
+    search::SequencedTaskExecutor indexFieldInverter(2);
+    search::SequencedTaskExecutor indexFieldWriter(2);
+    MemoryIndex memory_index(schema, indexFieldInverter, indexFieldWriter);
+    DocBuilder doc_builder(schema);
+
+    Document::UP doc = buildDocument(doc_builder, doc_id1, word1);
+    memory_index.insertDocument(doc_id1, *doc.get());
+
+    doc = buildDocument(doc_builder, doc_id2, word2);
+    memory_index.insertDocument(doc_id2, *doc.get());
+    memory_index.commit(std::shared_ptr<search::IDestructorCallback>());
+    indexFieldWriter.sync();
+
+    testSearch(memory_index, word1, doc_id1);
+    testSearch(memory_index, word2, doc_id2);
+
+    const string index_dir = "test_index";
+    IndexBuilder index_builder(schema);
+    index_builder.setPrefix(index_dir);
+    const uint32_t docIdLimit = memory_index.getDocIdLimit();
+    const uint64_t num_words = memory_index.getNumWords();
+    search::TuneFileIndexing tuneFileIndexing;
+    DummyFileHeaderContext fileHeaderContext;
+    index_builder.open(docIdLimit, num_words, tuneFileIndexing,
+                       fileHeaderContext);
+    memory_index.dump(index_builder);
+    index_builder.close();
+
+    // Fusion test.  Keep all documents to get an "indentical" copy.
+    const string index_dir2 = "test_index2";
+    std::vector<string> fusionInputs;
+    fusionInputs.push_back(index_dir);
+    uint32_t fusionDocIdLimit = 0;
+    typedef search::diskindex::Fusion FastS_Fusion;
+    bool fret1 = DocumentSummary::readDocIdLimit(index_dir, fusionDocIdLimit);
+    ASSERT_TRUE(fret1);
+    SelectorArray selector(fusionDocIdLimit, 0);
+    bool fret2 = FastS_Fusion::merge(schema,
+                                    index_dir2,
+                                    fusionInputs,
+                                    selector,
+                                    false /* dynamicKPosOccFormat */,
+                                     tuneFileIndexing,
+                                     fileHeaderContext);
+    ASSERT_TRUE(fret2);
+
+    // Fusion test with all docs removed in output (doesn't affect word list)
+    const string index_dir3 = "test_index3";
+    fusionInputs.clear();
+    fusionInputs.push_back(index_dir);
+    fusionDocIdLimit = 0;
+    bool fret3 = DocumentSummary::readDocIdLimit(index_dir, fusionDocIdLimit);
+    ASSERT_TRUE(fret3);
+    SelectorArray selector2(fusionDocIdLimit, 1);
+    bool fret4 = FastS_Fusion::merge(schema,
+                                    index_dir3,
+                                    fusionInputs,
+                                    selector2,
+                                    false /* dynamicKPosOccFormat */,
+                                     tuneFileIndexing,
+                                     fileHeaderContext);
+    ASSERT_TRUE(fret4);
+
+    // Fusion test with all docs removed in input (affects word list)
+    const string index_dir4 = "test_index4";
+    fusionInputs.clear();
+    fusionInputs.push_back(index_dir3);
+    fusionDocIdLimit = 0;
+    bool fret5 = DocumentSummary::readDocIdLimit(index_dir3, fusionDocIdLimit);
+    ASSERT_TRUE(fret5);
+    SelectorArray selector3(fusionDocIdLimit, 0);
+    bool fret6 = FastS_Fusion::merge(schema,
+                                    index_dir4,
+                                    fusionInputs,
+                                    selector3,
+                                    false /* dynamicKPosOccFormat */,
+                                     tuneFileIndexing,
+                                     fileHeaderContext);
+    ASSERT_TRUE(fret6);
+
+    DiskIndex disk_index(index_dir);
+    ASSERT_TRUE(disk_index.setup(TuneFileSearch()));
+    testSearch(disk_index, word1, doc_id1);
+    testSearch(disk_index, word2, doc_id2);
+    DiskIndex disk_index2(index_dir2);
+    ASSERT_TRUE(disk_index2.setup(TuneFileSearch()));
+    testSearch(disk_index2, word1, doc_id1);
+    testSearch(disk_index2, word2, doc_id2);
+}
+}  // namespace
+
+TEST_APPHOOK(Test);
diff --git a/searchcore/src/tests/proton/feedoperation/.gitignore b/searchcore/src/tests/proton/feedoperation/.gitignore
new file mode 100644
index 00000000000..695cdf3365d
--- /dev/null
+++ b/searchcore/src/tests/proton/feedoperation/.gitignore
@@ -0,0 +1,5 @@
+*_test
+.depend
+Makefile
+
+searchcore_feedoperation_test_app
diff --git a/searchcore/src/tests/proton/feedoperation/CMakeLists.txt b/searchcore/src/tests/proton/feedoperation/CMakeLists.txt
new file mode 100644
index 00000000000..fc47a3bde85
--- /dev/null
+++ b/searchcore/src/tests/proton/feedoperation/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_feedoperation_test_app
+    SOURCES
+    feedoperation_test.cpp
+    DEPENDS
+    searchcore_feedoperation
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_feedoperation_test_app COMMAND searchcore_feedoperation_test_app)
diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
new file mode 100644
index 00000000000..804927a0e03
--- /dev/null
+++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
@@ -0,0 +1,172 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for feedoperation.
+
+#include <vespa/log/log.h>
+LOG_SETUP("feedoperation_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <persistence/spi/types.h>
+#include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h>
+#include <vespa/searchcore/proton/feedoperation/deletebucketoperation.h>
+#include <vespa/searchcore/proton/feedoperation/joinbucketsoperation.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchcore/proton/feedoperation/newconfigoperation.h>
+#include <vespa/searchcore/proton/feedoperation/noopoperation.h>
+#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h>
+#include <vespa/searchcore/proton/feedoperation/putoperation.h>
+#include <vespa/searchcore/proton/feedoperation/removeoperation.h>
+#include <vespa/searchcore/proton/feedoperation/splitbucketoperation.h>
+#include <vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h>
+#include <vespa/searchcore/proton/feedoperation/updateoperation.h>
+#include <vespa/searchcore/proton/feedoperation/wipehistoryoperation.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using document::BucketId;
+using document::DataType;
+using document::Document;
+using document::DocumentId;
+using document::DocumentUpdate;
+using search::DocumentIdT;
+using storage::spi::Timestamp;
+using namespace proton;
+
+namespace {
+
+struct MyStreamHandler : NewConfigOperation::IStreamHandler {
+    typedef NewConfigOperation::SerialNum SerialNum;
+    virtual void serializeConfig(SerialNum, vespalib::nbostream &) {}
+    virtual void deserializeConfig(SerialNum, vespalib::nbostream &) {}
+};
+
+TEST("require that toString() on derived classes are meaningful")
+{
+    BucketId bucket_id1(42);
+    BucketId bucket_id2(43);
+    BucketId bucket_id3(44);
+    Timestamp timestamp(10);
+    Document::SP doc(new Document);
+    DbDocumentId db_doc_id;
+    uint32_t sub_db_id = 1;
+    MyStreamHandler stream_handler;
+    DocumentIdT doc_id_limit = 15;
+    DocumentId doc_id("doc:foo:bar");
+    DocumentUpdate::SP update(new DocumentUpdate(*DataType::DOCUMENT, doc_id));
+
+    EXPECT_EQUAL("DeleteBucket(BucketId(0x0000000000000000), serialNum=0)",
+                 DeleteBucketOperation().toString());
+    EXPECT_EQUAL("DeleteBucket(BucketId(0x000000000000002a), serialNum=0)",
+                 DeleteBucketOperation(bucket_id1).toString());
+
+    EXPECT_EQUAL("JoinBuckets("
+                 "source1=BucketId(0x0000000000000000), "
+                 "source2=BucketId(0x0000000000000000), "
+                 "target=BucketId(0x0000000000000000), serialNum=0)",
+                 JoinBucketsOperation().toString());
+    EXPECT_EQUAL("JoinBuckets("
+                 "source1=BucketId(0x000000000000002a), "
+                 "source2=BucketId(0x000000000000002b), "
+                 "target=BucketId(0x000000000000002c), serialNum=0)",
+                 JoinBucketsOperation(bucket_id1, bucket_id2, bucket_id3)
+                 .toString());
+
+    EXPECT_EQUAL("Move(NULL, BucketId(0x0000000000000000), timestamp=0, dbdId=(subDbId=0, lid=0), "
+                 "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
+                 MoveOperation().toString());
+    EXPECT_EQUAL("Move(null::, BucketId(0x000000000000002a), timestamp=10, dbdId=(subDbId=1, lid=0), "
+                 "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
+                 MoveOperation(bucket_id1, timestamp, doc,
+                               db_doc_id, sub_db_id).toString());
+
+    EXPECT_EQUAL("NewConfig(serialNum=64)",
+                 NewConfigOperation(64, stream_handler).toString());
+
+    EXPECT_EQUAL("Noop(serialNum=32)", NoopOperation(32).toString());
+
+    EXPECT_EQUAL("PruneRemovedDocuments(limitLid=0, subDbId=0, "
+                 "serialNum=0)",
+                 PruneRemovedDocumentsOperation().toString());
+    EXPECT_EQUAL("PruneRemovedDocuments(limitLid=15, subDbId=1, "
+                 "serialNum=0)",
+                 PruneRemovedDocumentsOperation(
+                         doc_id_limit, sub_db_id).toString());
+
+    EXPECT_EQUAL("Put(NULL, BucketId(0x0000000000000000), timestamp=0, dbdId=(subDbId=0, lid=0), "
+                 "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
+                 PutOperation().toString());
+    EXPECT_EQUAL("Put(null::, BucketId(0x000000000000002a), timestamp=10, dbdId=(subDbId=0, lid=0), "
+                 "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
+                 PutOperation(bucket_id1, timestamp, doc).toString());
+
+    EXPECT_EQUAL("Remove(null::, BucketId(0x0000000000000000), timestamp=0, dbdId=(subDbId=0, lid=0), "
+                 "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
+                 RemoveOperation().toString());
+    EXPECT_EQUAL("Remove(doc:foo:bar, BucketId(0x000000000000002a), timestamp=10, dbdId=(subDbId=0, lid=0), "
+                 "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
+                 RemoveOperation(bucket_id1, timestamp, doc_id).toString());
+
+    EXPECT_EQUAL("SplitBucket("
+                 "source=BucketId(0x0000000000000000), "
+                 "target1=BucketId(0x0000000000000000), "
+                 "target2=BucketId(0x0000000000000000), serialNum=0)",
+                 SplitBucketOperation().toString());
+    EXPECT_EQUAL("SplitBucket("
+                 "source=BucketId(0x000000000000002a), "
+                 "target1=BucketId(0x000000000000002b), "
+                 "target2=BucketId(0x000000000000002c), serialNum=0)",
+                 SplitBucketOperation(bucket_id1, bucket_id2, bucket_id3)
+                 .toString());
+
+    EXPECT_EQUAL("SpoolerReplayStart(spoolerSerialNum=0, serialNum=0)",
+                 SpoolerReplayStartOperation().toString());
+    EXPECT_EQUAL("SpoolerReplayStart(spoolerSerialNum=20, serialNum=10)",
+                 SpoolerReplayStartOperation(10, 20).toString());
+
+    EXPECT_EQUAL("SpoolerReplayComplete(spoolerSerialNum=0, serialNum=0)",
+                 SpoolerReplayCompleteOperation().toString());
+    EXPECT_EQUAL("SpoolerReplayComplete(spoolerSerialNum=2, serialNum=1)",
+                 SpoolerReplayCompleteOperation(1, 2).toString());
+
+    EXPECT_EQUAL("Update(NULL, BucketId(0x0000000000000000), timestamp=0, dbdId=(subDbId=0, lid=0), "
+                 "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
+                 UpdateOperation().toString());
+    EXPECT_EQUAL("Update(doc:foo:bar, BucketId(0x000000000000002a), timestamp=10, dbdId=(subDbId=0, lid=0), "
+                 "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
+                 UpdateOperation(bucket_id1, timestamp, update).toString());
+
+    EXPECT_EQUAL("WipeHistory(wipeTimeLimit=0, serialNum=0)",
+                 WipeHistoryOperation().toString());
+    EXPECT_EQUAL("WipeHistory(wipeTimeLimit=20, serialNum=10)",
+                 WipeHistoryOperation(10, 20).toString());
+    EXPECT_EQUAL("CompactLidSpace(subDbId=2, lidLimit=99, serialNum=0)",
+                 CompactLidSpaceOperation(2, 99).toString());
+}
+
+TEST("require that serialize/deserialize works for CompactLidSpaceOperation")
+{
+    vespalib::nbostream stream;
+    {
+        CompactLidSpaceOperation op(2, 99);
+        EXPECT_EQUAL(FeedOperation::COMPACT_LID_SPACE, op.getType());
+        EXPECT_EQUAL(2u, op.getSubDbId());
+        EXPECT_EQUAL(99u, op.getLidLimit());
+        op.serialize(stream);
+    }
+    {
+        const document::DocumentTypeRepo *repo = NULL;
+        CompactLidSpaceOperation op;
+        op.deserialize(stream, *repo);
+        EXPECT_EQUAL(FeedOperation::COMPACT_LID_SPACE, op.getType());
+        EXPECT_EQUAL(2u, op.getSubDbId());
+        EXPECT_EQUAL(99u, op.getLidLimit());
+    }
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/feedtoken/.gitignore b/searchcore/src/tests/proton/feedtoken/.gitignore
new file mode 100644
index 00000000000..eee829c3cbf
--- /dev/null
+++ b/searchcore/src/tests/proton/feedtoken/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+feedtoken_test
+searchcore_feedtoken_test_app
diff --git a/searchcore/src/tests/proton/feedtoken/CMakeLists.txt b/searchcore/src/tests/proton/feedtoken/CMakeLists.txt
new file mode 100644
index 00000000000..328d872f668
--- /dev/null
+++ b/searchcore/src/tests/proton/feedtoken/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_feedtoken_test_app
+    SOURCES
+    feedtoken.cpp
+    DEPENDS
+    searchcore_pcommon
+    searchcore_proton_metrics
+)
+vespa_add_test(NAME searchcore_feedtoken_test_app COMMAND searchcore_feedtoken_test_app)
diff --git a/searchcore/src/tests/proton/feedtoken/DESC b/searchcore/src/tests/proton/feedtoken/DESC
new file mode 100644
index 00000000000..6fd2921a8c4
--- /dev/null
+++ b/searchcore/src/tests/proton/feedtoken/DESC
@@ -0,0 +1 @@
+feedtoken test. Take a look at feedtoken.cpp for details.
diff --git a/searchcore/src/tests/proton/feedtoken/FILES b/searchcore/src/tests/proton/feedtoken/FILES
new file mode 100644
index 00000000000..052aab3d388
--- /dev/null
+++ b/searchcore/src/tests/proton/feedtoken/FILES
@@ -0,0 +1 @@
+feedtoken.cpp
diff --git a/searchcore/src/tests/proton/feedtoken/feedtoken.cpp b/searchcore/src/tests/proton/feedtoken/feedtoken.cpp
new file mode 100644
index 00000000000..bd7d0b9cf6c
--- /dev/null
+++ b/searchcore/src/tests/proton/feedtoken/feedtoken.cpp
@@ -0,0 +1,158 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("feedtoken_test");
+
+#include <vespa/messagebus/emptyreply.h>
+#include <vespa/messagebus/testlib/receptor.h>
+#include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace proton;
+
+class LocalTransport : public FeedToken::ITransport {
+private:
+    mbus::Receptor _receptor;
+    double         _latency_ms;
+
+public:
+    LocalTransport()
+        : _receptor(),
+          _latency_ms(0.0)
+    {
+        // empty
+    }
+
+    void send(mbus::Reply::UP reply,
+              ResultUP,
+              bool,
+              double latency_ms) {
+        _receptor.handleReply(std::move(reply));
+        _latency_ms = latency_ms;
+    }
+
+    mbus::Reply::UP getReply() {
+        return _receptor.getReply();
+    }
+
+    double getLatencyMs() const {
+        return _latency_ms;
+    }
+};
+
+class Test : public vespalib::TestApp {
+private:
+    void testAck();
+    void testAutoReply();
+    void testFail();
+    void testHandover();
+    void testIntegrity();
+    void testTrace();
+
+public:
+    int Main() {
+        TEST_INIT("feedtoken_test");
+
+        testAck();       TEST_FLUSH();
+//        testAutoReply(); TEST_FLUSH();
+        testFail();      TEST_FLUSH();
+        testHandover();  TEST_FLUSH();
+//        testIntegrity(); TEST_FLUSH();
+        testTrace();     TEST_FLUSH();
+
+        TEST_DONE();
+    }
+};
+
+TEST_APPHOOK(Test);
+
+void
+Test::testAck()
+{
+    LocalTransport transport;
+    mbus::Reply::UP msg(new documentapi::RemoveDocumentReply());
+    FeedToken token(transport, std::move(msg));
+    token.ack();
+    mbus::Reply::UP reply = transport.getReply();
+    ASSERT_TRUE(reply.get() != NULL);
+    EXPECT_TRUE(!reply->hasErrors());
+}
+
+void
+Test::testAutoReply()
+{
+    mbus::Receptor receptor;
+    mbus::Reply::UP reply(new documentapi::RemoveDocumentReply());
+    reply->pushHandler(receptor);
+    {
+        LocalTransport transport;
+        FeedToken token(transport, std::move(reply));
+    }
+    reply = receptor.getReply(0);
+    ASSERT_TRUE(reply.get() != NULL);
+    EXPECT_TRUE(reply->hasErrors());
+}
+
+void
+Test::testFail()
+{
+    LocalTransport transport;
+    mbus::Reply::UP reply(new documentapi::RemoveDocumentReply());
+    FeedToken token(transport, std::move(reply));
+    token.fail(69, "6699");
+    reply = transport.getReply();
+    ASSERT_TRUE(reply.get() != NULL);
+    EXPECT_EQUAL(1u, reply->getNumErrors());
+    EXPECT_EQUAL(69u, reply->getError(0).getCode());
+    EXPECT_EQUAL("6699", reply->getError(0).getMessage());
+}
+
+void
+Test::testHandover()
+{
+    struct MyHandover {
+        static FeedToken handover(FeedToken token) {
+            return token;
+        }
+    };
+
+    LocalTransport transport;
+    mbus::Reply::UP reply(new documentapi::RemoveDocumentReply());
+
+    FeedToken token(transport, std::move(reply));
+    token = MyHandover::handover(token);
+    token.ack();
+    reply = transport.getReply();
+    ASSERT_TRUE(reply.get() != NULL);
+    EXPECT_TRUE(!reply->hasErrors());
+}
+
+void
+Test::testIntegrity()
+{
+    LocalTransport transport;
+    try {
+        FeedToken token(transport, mbus::Reply::UP());
+        EXPECT_TRUE(false); // should throw an exception
+    } catch (vespalib::IllegalArgumentException &e) {
+        (void)e; // expected
+    }
+}
+
+void
+Test::testTrace()
+{
+    LocalTransport transport;
+    mbus::Reply::UP reply(new documentapi::RemoveDocumentReply());
+
+    FeedToken token(transport, std::move(reply));
+    token.trace(0, "foo");
+    token.ack();
+    reply = transport.getReply();
+    ASSERT_TRUE(reply.get() != NULL);
+    EXPECT_TRUE(!reply->hasErrors());
+    std::string trace = reply->getTrace().toString();
+    fprintf(stderr, "%s", trace.c_str());
+    EXPECT_TRUE(trace.find("foo") != std::string::npos);
+}
diff --git a/searchcore/src/tests/proton/flushengine/.gitignore b/searchcore/src/tests/proton/flushengine/.gitignore
new file mode 100644
index 00000000000..65d6633a4d1
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/.gitignore
@@ -0,0 +1,2 @@
+/vlog.txt
+searchcore_flushengine_test_app
diff --git a/searchcore/src/tests/proton/flushengine/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/CMakeLists.txt
new file mode 100644
index 00000000000..4fc59180946
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_flushengine_test_app
+    SOURCES
+    flushengine.cpp
+    DEPENDS
+    searchcore_flushengine
+    searchcore_pcommon
+)
+vespa_add_test(
+    NAME searchcore_flushengine_test_app
+    COMMAND searchcore_flushengine_test_app
+    ENVIRONMENT "VESPA_LOG_LEVEL=all;VESPA_LOG_TARGET=file:vlog.txt"
+)
diff --git a/searchcore/src/tests/proton/flushengine/DESC b/searchcore/src/tests/proton/flushengine/DESC
new file mode 100644
index 00000000000..87a9f388463
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/DESC
@@ -0,0 +1 @@
+flushengine test. Take a look at flushengine.cpp for details.
diff --git a/searchcore/src/tests/proton/flushengine/FILES b/searchcore/src/tests/proton/flushengine/FILES
new file mode 100644
index 00000000000..5156cba47e7
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/FILES
@@ -0,0 +1 @@
+flushengine.cpp
diff --git a/searchcore/src/tests/proton/flushengine/flushengine.cpp b/searchcore/src/tests/proton/flushengine/flushengine.cpp
new file mode 100644
index 00000000000..59b86671a0d
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/flushengine.cpp
@@ -0,0 +1,605 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("flushengine_test");
+
+#include <vespa/searchcore/proton/flushengine/cachedflushtarget.h>
+#include <vespa/searchcore/proton/flushengine/flush_engine_explorer.h>
+#include <vespa/searchcore/proton/flushengine/flushengine.h>
+#include <vespa/searchcore/proton/flushengine/threadedflushtarget.h>
+#include <vespa/searchcore/proton/flushengine/tls_stats_map.h>
+#include <vespa/searchcore/proton/flushengine/i_tls_stats_factory.h>
+#include <vespa/searchcore/proton/server/igetserialnum.h>
+#include <vespa/searchcore/proton/test/dummy_flush_handler.h>
+#include <vespa/searchcore/proton/test/dummy_flush_target.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/util/sync.h>
+#include <memory>
+
+// --------------------------------------------------------------------------------
+//
+// Setup.
+//
+// --------------------------------------------------------------------------------
+
+using namespace proton;
+using namespace vespalib::slime;
+using searchcorespi::FlushTask;
+using vespalib::Slime;
+
+const long LONG_TIMEOUT = 66666;
+const long SHORT_TIMEOUT = 1;
+const uint32_t IINTERVAL = 1000;
+
+class SimpleExecutor : public vespalib::Executor {
+public:
+    vespalib::Gate _done;
+
+public:
+    SimpleExecutor()
+        : _done()
+    {
+        // empty
+    }
+
+    Task::UP
+    execute(Task::UP task)
+    {
+        task->run();
+        _done.countDown();
+        return Task::UP();
+    }
+};
+
+class SimpleGetSerialNum : public IGetSerialNum
+{
+    virtual search::SerialNum getSerialNum() const override {
+        return 0u;
+    }
+};
+
+class SimpleTlsStatsFactory : public flushengine::ITlsStatsFactory
+{
+    virtual flushengine::TlsStatsMap create() override {
+        vespalib::hash_map<vespalib::string, flushengine::TlsStats> map;
+        return flushengine::TlsStatsMap(std::move(map));
+    }
+};
+
+typedef std::vector<IFlushTarget::SP> Targets;
+
+class SimpleHandler : public test::DummyFlushHandler {
+public:
+    Targets                   _targets;
+    search::SerialNum         _oldestSerial;
+    search::SerialNum         _currentSerial;
+    vespalib::CountDownLatch  _done;
+
+public:
+    typedef std::shared_ptr<SimpleHandler> SP;
+
+    SimpleHandler(const Targets &targets, const std::string &name = "anon",
+                  search::SerialNum currentSerial = -1)
+        : test::DummyFlushHandler(name),
+          _targets(targets),
+          _oldestSerial(0),
+          _currentSerial(currentSerial),
+          _done(targets.size())
+    {
+        // empty
+    }
+
+    search::SerialNum
+    getCurrentSerialNumber() const override
+    {
+        LOG(info, "SimpleHandler(%s)::getCurrentSerialNumber()",
+            getName().c_str());
+        return _currentSerial;
+    }
+
+    std::vector<IFlushTarget::SP>
+    getFlushTargets() override
+    {
+        LOG(info, "SimpleHandler(%s)::getFlushTargets()",
+            getName().c_str());
+        return _targets;
+    }
+
+    void
+    flushDone(search::SerialNum oldestSerial) override
+    {
+        LOG(info, "SimpleHandler(%s)::flushDone(%" PRIu64 ")",
+            getName().c_str(), oldestSerial);
+        _oldestSerial = std::max(_oldestSerial, oldestSerial);
+        _done.countDown();
+   }
+
+};
+
+class SimpleTask : public searchcorespi::FlushTask {
+public:
+    vespalib::Gate &_start;
+    vespalib::Gate &_done;
+    vespalib::Gate *_proceed;
+
+public:
+    SimpleTask(vespalib::Gate &start,
+               vespalib::Gate &done,
+               vespalib::Gate *proceed)
+        : _start(start), _done(done), _proceed(proceed)
+    {
+        // empty
+    }
+
+    void run() {
+        _start.countDown();
+        if (_proceed != NULL) {
+            _proceed->await();
+        }
+        _done.countDown();
+    }
+
+    virtual search::SerialNum
+    getFlushSerial(void) const
+    {
+        return 0u;
+    }
+};
+
+class SimpleTarget : public test::DummyFlushTarget {
+public:
+    search::SerialNum _flushedSerial;
+    vespalib::Gate    _proceed;
+    vespalib::Gate    _initDone;
+    vespalib::Gate    _taskStart;
+    vespalib::Gate    _taskDone;
+    Task::UP          _task;
+
+public:
+    typedef std::shared_ptr<SimpleTarget> SP;
+
+    SimpleTarget(Task::UP task, const std::string &name) :
+        test::DummyFlushTarget(name),
+        _flushedSerial(0),
+        _proceed(),
+        _initDone(),
+        _taskStart(),
+        _taskDone(),
+        _task(std::move(task))
+    {
+    }
+
+    SimpleTarget(const std::string &name = "anon", search::SerialNum flushedSerial = 0, bool proceedImmediately = true) :
+        test::DummyFlushTarget(name),
+        _flushedSerial(flushedSerial),
+        _proceed(),
+        _initDone(),
+        _taskStart(),
+        _taskDone(),
+        _task(new SimpleTask(_taskStart, _taskDone, &_proceed))
+    {
+        if (proceedImmediately) {
+            _proceed.countDown();
+        }
+    }
+
+    virtual Time
+    getLastFlushTime() const override { return fastos::ClockSystem::now(); }
+
+    virtual SerialNum
+    getFlushedSerialNum() const override
+    {
+        LOG(info, "SimpleTarget(%s)::getFlushedSerialNum()",
+            getName().c_str());
+        return _flushedSerial;
+    }
+
+    virtual Task::UP
+    initFlush(SerialNum currentSerial) override
+    {
+        LOG(info, "SimpleTarget(%s)::initFlush(%" PRIu64 ")",
+            getName().c_str(), currentSerial);
+        _initDone.countDown();
+        return std::move(_task);
+    }
+
+};
+
+class AssertedTarget : public SimpleTarget {
+public:
+    mutable bool _mgain;
+    mutable bool _serial;
+
+public:
+    typedef std::shared_ptr<AssertedTarget> SP;
+
+    AssertedTarget(const std::string &name = "anon")
+        : SimpleTarget(name),
+          _mgain(false),
+          _serial(false)
+    {
+        // empty
+    }
+
+    virtual MemoryGain
+    getApproxMemoryGain() const
+    {
+        LOG_ASSERT(_mgain == false);
+        _mgain = true;
+        return SimpleTarget::getApproxMemoryGain();
+    }
+
+    virtual search::SerialNum
+    getFlushedSerialNum() const
+    {
+        LOG_ASSERT(_serial == false);
+        _serial = true;
+        return SimpleTarget::getFlushedSerialNum();
+    }
+};
+
+class SimpleStrategy : public IFlushStrategy {
+public:
+    std::vector<IFlushTarget::SP> _targets;
+
+    struct CompareTarget {
+        CompareTarget(const SimpleStrategy &flush) : _flush(flush) { }
+        bool operator () (const FlushContext::SP &lhs, const FlushContext::SP &rhs) const {
+            return _flush.compare(lhs->getTarget(), rhs->getTarget());
+        }
+        const SimpleStrategy &_flush;
+    };
+
+    virtual FlushContext::List getFlushTargets(const FlushContext::List &targetList,
+                                               const flushengine::TlsStatsMap &) const override {
+        FlushContext::List fv(targetList);
+        std::sort(fv.begin(), fv.end(), CompareTarget(*this));
+        return fv;
+    }
+
+    bool
+    compare(const IFlushTarget::SP &lhs, const IFlushTarget::SP &rhs) const
+    {
+        LOG(info, "SimpleStrategy::compare(%p, %p)", lhs.get(), rhs.get());
+        return indexOf(lhs) < indexOf(rhs);
+    }
+
+
+public:
+    typedef std::shared_ptr<SimpleStrategy> SP;
+
+    SimpleStrategy()
+    {
+        // empty
+    }
+
+    uint32_t
+    indexOf(const IFlushTarget::SP &target) const
+    {
+        IFlushTarget *raw = target.get();
+        CachedFlushTarget *cached = dynamic_cast<CachedFlushTarget*>(raw);
+        if (cached != NULL) {
+            raw = cached->getFlushTarget().get();
+        }
+        for (uint32_t i = 0, len = _targets.size(); i < len; ++i) {
+            if (raw == _targets[i].get()) {
+                LOG(info, "Index of target %p is %d.", raw, i);
+                return i;
+            }
+        }
+        LOG(info, "Target %p not found.", raw);
+        return -1;
+    }
+};
+
+class ConstantFlushStrategy : public SimpleStrategy {
+public:
+    uint64_t _millis;
+
+public:
+    ConstantFlushStrategy(uint64_t millis) : SimpleStrategy(), _millis(millis) { }
+    typedef std::shared_ptr<ConstantFlushStrategy> SP;
+};
+
+// --------------------------------------------------------------------------------
+//
+// Tests.
+//
+// --------------------------------------------------------------------------------
+
+class AppendTask : public FlushTask
+{
+public:
+    AppendTask(const vespalib::string & name, std::vector<vespalib::string> & list, vespalib::Gate & done) :
+        _list(list),
+        _done(done),
+        _name(name)
+    { }
+    void run() {
+        _list.push_back(_name);
+        _done.countDown();
+    }
+    virtual search::SerialNum
+    getFlushSerial(void) const
+    {
+        return 0u;
+    }
+    std::vector<vespalib::string> & _list;
+    vespalib::Gate    & _done;
+    vespalib::string    _name;
+};
+
+
+struct Fixture
+{
+    std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory;
+    SimpleStrategy::SP strategy;
+    FlushEngine engine;
+
+    Fixture(uint32_t numThreads, uint32_t idleIntervalMS)
+        : tlsStatsFactory(std::make_shared<SimpleTlsStatsFactory>()),
+          strategy(std::make_shared<SimpleStrategy>()),
+          engine(tlsStatsFactory, strategy, numThreads, idleIntervalMS, false)
+    {
+    }
+};
+
+
+TEST_F("require that strategy controls flush target", Fixture(1, IINTERVAL))
+{
+    vespalib::Gate fooG, barG;
+    std::vector<vespalib::string> order;
+    FlushTask::UP fooT(new AppendTask("foo", order, fooG));
+    FlushTask::UP barT(new AppendTask("bar", order, barG));
+    SimpleTarget::SP foo(new SimpleTarget(std::move(fooT), "foo"));
+    SimpleTarget::SP bar(new SimpleTarget(std::move(barT), "bar"));
+    f.strategy->_targets.push_back(foo);
+    f.strategy->_targets.push_back(bar);
+
+    SimpleHandler::SP handler(new SimpleHandler({bar, foo}));
+    DocTypeName dtnvanon("anon");
+    f.engine.putFlushHandler(dtnvanon, handler);
+    f.engine.start();
+
+    EXPECT_TRUE(fooG.await(LONG_TIMEOUT));
+    EXPECT_TRUE(barG.await(LONG_TIMEOUT));
+    EXPECT_EQUAL(2u, order.size());
+    EXPECT_EQUAL("foo", order[0]);
+    EXPECT_EQUAL("bar", order[1]);
+}
+
+TEST_F("require that zero handlers does not core", Fixture(2, 50))
+{
+    f.engine.start();
+}
+
+TEST_F("require that zero targets does not core", Fixture(2, 50))
+{
+    DocTypeName dtnvfoo("foo");
+    DocTypeName dtnvbar("bar");
+    f.engine.putFlushHandler(dtnvfoo,
+                             IFlushHandler::SP(new SimpleHandler({}, "foo")));
+    f.engine.putFlushHandler(dtnvbar,
+                             IFlushHandler::SP(new SimpleHandler({}, "bar")));
+    f.engine.start();
+}
+
+TEST_F("require that oldest serial is found", Fixture(1, IINTERVAL))
+{
+    SimpleTarget::SP foo(new SimpleTarget("foo", 10));
+    SimpleTarget::SP bar(new SimpleTarget("bar", 20));
+    f.strategy->_targets.push_back(foo);
+    f.strategy->_targets.push_back(bar);
+
+    SimpleHandler::SP handler(new SimpleHandler({foo, bar}, "anon", 25));
+    DocTypeName dtnvanon("anon");
+    f.engine.putFlushHandler(dtnvanon, handler);
+    f.engine.start();
+
+    EXPECT_TRUE(handler->_done.await(LONG_TIMEOUT));
+    EXPECT_EQUAL(20ul, handler->_oldestSerial);
+}
+
+TEST_F("require that oldest serial is found in group", Fixture(2, IINTERVAL))
+{
+    SimpleTarget::SP fooT1(new SimpleTarget("fooT1", 10));
+    SimpleTarget::SP fooT2(new SimpleTarget("fooT2", 20));
+    SimpleTarget::SP barT1(new SimpleTarget("barT1",  5));
+    SimpleTarget::SP barT2(new SimpleTarget("barT2", 15));
+    f.strategy->_targets.push_back(fooT1);
+    f.strategy->_targets.push_back(fooT2);
+    f.strategy->_targets.push_back(barT1);
+    f.strategy->_targets.push_back(barT2);
+
+    SimpleHandler::SP fooH(new SimpleHandler({fooT1, fooT2}, "fooH", 25));
+    DocTypeName dtnvfoo("foo");
+    f.engine.putFlushHandler(dtnvfoo, fooH);
+
+    SimpleHandler::SP barH(new SimpleHandler({barT1, barT2}, "barH", 20));
+    DocTypeName dtnvbar("bar");
+    f.engine.putFlushHandler(dtnvbar, barH);
+
+    f.engine.start();
+
+    EXPECT_TRUE(fooH->_done.await(LONG_TIMEOUT));
+    EXPECT_EQUAL(20ul, fooH->_oldestSerial);
+    EXPECT_TRUE(barH->_done.await(LONG_TIMEOUT));
+    EXPECT_EQUAL(15ul, barH->_oldestSerial);
+}
+
+TEST_F("require that target can refuse flush", Fixture(2, IINTERVAL))
+{
+    SimpleTarget::SP target(new SimpleTarget());
+    SimpleHandler::SP handler(new SimpleHandler({target}));
+    target->_task = searchcorespi::FlushTask::UP();
+    DocTypeName dtnvanon("anon");
+    f.engine.putFlushHandler(dtnvanon, handler);
+    f.engine.start();
+
+    EXPECT_TRUE(target->_initDone.await(LONG_TIMEOUT));
+    EXPECT_TRUE(!target->_taskDone.await(SHORT_TIMEOUT));
+    EXPECT_TRUE(!handler->_done.await(SHORT_TIMEOUT));
+}
+
+TEST_F("require that targets are flushed when nothing new to flush",
+       Fixture(2, IINTERVAL))
+{
+    SimpleTarget::SP target(new SimpleTarget("anon", 5)); // oldest unflushed serial num = 5
+    SimpleHandler::SP handler(new SimpleHandler({target}, "anon", 4)); // current serial num = 4
+    DocTypeName dtnvanon("anon");
+    f.engine.putFlushHandler(dtnvanon, handler);
+    f.engine.start();
+
+    EXPECT_TRUE(target->_initDone.await(LONG_TIMEOUT));
+    EXPECT_TRUE(target->_taskDone.await(LONG_TIMEOUT));
+    EXPECT_TRUE(handler->_done.await(LONG_TIMEOUT));
+}
+
+TEST_F("require that flushing targets are skipped", Fixture(2, IINTERVAL))
+{
+    SimpleTarget::SP foo(new SimpleTarget("foo"));
+    SimpleTarget::SP bar(new SimpleTarget("bar"));
+    f.strategy->_targets.push_back(foo);
+    f.strategy->_targets.push_back(bar);
+
+    SimpleHandler::SP handler(new SimpleHandler({bar, foo}));
+    DocTypeName dtnvanon("anon");
+    f.engine.putFlushHandler(dtnvanon, handler);
+    f.engine.start();
+
+    EXPECT_TRUE(foo->_taskDone.await(LONG_TIMEOUT));
+    EXPECT_TRUE(bar->_taskDone.await(LONG_TIMEOUT)); /* this is the key check */
+}
+
+TEST_F("require that updated targets are not skipped", Fixture(2, IINTERVAL))
+{
+    SimpleTarget::SP target(new SimpleTarget("target", 1));
+    f.strategy->_targets.push_back(target);
+
+    SimpleHandler::SP handler(new SimpleHandler({target}, "handler", 0));
+    DocTypeName dtnvhandler("handler");
+    f.engine.putFlushHandler(dtnvhandler, handler);
+    f.engine.start();
+
+    EXPECT_TRUE(target->_taskDone.await(LONG_TIMEOUT));
+}
+
+TEST("require that threaded target works")
+{
+    SimpleExecutor executor;
+    SimpleGetSerialNum getSerialNum;
+    IFlushTarget::SP target(new SimpleTarget());
+    target.reset(new ThreadedFlushTarget(executor, getSerialNum, target));
+
+    EXPECT_FALSE(executor._done.await(SHORT_TIMEOUT));
+    EXPECT_TRUE(target->initFlush(0).get() != NULL);
+    EXPECT_TRUE(executor._done.await(LONG_TIMEOUT));
+}
+
+TEST("require that cached target works")
+{
+    IFlushTarget::SP target(new AssertedTarget());
+    target.reset(new CachedFlushTarget(target));
+    for (uint32_t i = 0; i < 2; ++i) {
+        EXPECT_EQUAL(0l, target->getApproxMemoryGain().getBefore());
+        EXPECT_EQUAL(0l, target->getApproxMemoryGain().getAfter());
+        EXPECT_EQUAL(0ul, target->getFlushedSerialNum());
+    }
+}
+
+TEST_F("require that trigger flush works", Fixture(2, IINTERVAL))
+{
+    SimpleTarget::SP target(new SimpleTarget("target", 1));
+    f.strategy->_targets.push_back(target);
+
+    SimpleHandler::SP handler(new SimpleHandler({target}, "handler", 9));
+    DocTypeName dtnvhandler("handler");
+    f.engine.putFlushHandler(dtnvhandler, handler);
+    f.engine.start();
+    f.engine.triggerFlush();
+    EXPECT_TRUE(target->_initDone.await(LONG_TIMEOUT));
+    EXPECT_TRUE(target->_taskDone.await(LONG_TIMEOUT));
+}
+
+bool
+asserCorrectHandlers(const FlushEngine::FlushMetaSet & current1, const std::vector<const char *> & targets)
+{
+    bool retval(targets.size() == current1.size());
+    FlushEngine::FlushMetaSet::const_iterator curr(current1.begin());
+    if (retval) {
+        for (const char * target : targets) {
+            if (target != (curr++)->getName()) {
+                return false;
+            }
+        }
+    }
+    return retval;
+}
+
+void
+assertThatHandlersInCurrentSet(FlushEngine & engine, const std::vector<const char *> & targets)
+{
+    FlushEngine::FlushMetaSet current1 = engine.getCurrentlyFlushingSet();
+    while ((current1.size() < targets.size()) || !asserCorrectHandlers(current1, targets)) {
+        FastOS_Thread::Sleep(1);
+        current1 = engine.getCurrentlyFlushingSet();
+    }
+}
+
+TEST_F("require that concurrency works", Fixture(2, 1))
+{
+    SimpleTarget::SP target1(new SimpleTarget("target1", 1, false));
+    SimpleTarget::SP target2(new SimpleTarget("target2", 2, false));
+    SimpleTarget::SP target3(new SimpleTarget("target3", 3, false));
+    SimpleHandler::SP handler(new SimpleHandler({target1, target2, target3}, "handler", 9));
+    DocTypeName dtnvhandler("handler");
+    f.engine.putFlushHandler(dtnvhandler, handler);
+    f.engine.start();
+    EXPECT_TRUE(target1->_initDone.await(LONG_TIMEOUT));
+    EXPECT_TRUE(target2->_initDone.await(LONG_TIMEOUT));
+    EXPECT_TRUE(!target3->_initDone.await(SHORT_TIMEOUT));
+    assertThatHandlersInCurrentSet(f.engine, {"handler.target1", "handler.target2"});
+    EXPECT_TRUE(!target3->_initDone.await(SHORT_TIMEOUT));
+    target1->_proceed.countDown();
+    EXPECT_TRUE(target1->_taskDone.await(LONG_TIMEOUT));
+    assertThatHandlersInCurrentSet(f.engine, {"handler.target2", "handler.target3"});
+    target3->_proceed.countDown();
+    target2->_proceed.countDown();
+}
+
+TEST_F("require that state explorer can list flush targets", Fixture(1, 1))
+{
+    SimpleTarget::SP target = std::make_shared<SimpleTarget>("target1", 100, false);
+    f.engine.putFlushHandler(DocTypeName("handler"),
+            std::make_shared<SimpleHandler>(
+                    Targets({target, std::make_shared<SimpleTarget>("target2", 50, true)}),
+                             "handler", 9));
+    f.engine.start();
+    target->_initDone.await(LONG_TIMEOUT);
+    target->_taskStart.await(LONG_TIMEOUT);
+
+    FlushEngineExplorer explorer(f.engine);
+    Slime state;
+    SlimeInserter inserter(state);
+    explorer.get_state(inserter, true);
+
+    Inspector &all = state.get()["allTargets"];
+    EXPECT_EQUAL(2u, all.children());
+    EXPECT_EQUAL("handler.target2", all[0]["name"].asString().make_string());
+    EXPECT_EQUAL(50, all[0]["flushedSerialNum"].asLong());
+    EXPECT_EQUAL("handler.target1", all[1]["name"].asString().make_string());
+    EXPECT_EQUAL(100, all[1]["flushedSerialNum"].asLong());
+
+    Inspector &flushing = state.get()["flushingTargets"];
+    EXPECT_EQUAL(1u, flushing.children());
+    EXPECT_EQUAL("handler.target1", flushing[0]["name"].asString().make_string());
+
+    target->_proceed.countDown();
+    target->_taskDone.await(LONG_TIMEOUT);
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt
new file mode 100644
index 00000000000..a4bff892ffa
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt
@@ -0,0 +1,12 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_flushengine_prepare_restart_flush_strategy_test_app
+    SOURCES
+    prepare_restart_flush_strategy_test.cpp
+    DEPENDS
+    searchcorespi
+    searchcore_flushengine
+)
+vespa_add_test(
+    NAME searchcore_flushengine_prepare_restart_flush_strategy_test_app
+    COMMAND searchcore_flushengine_prepare_restart_flush_strategy_test_app
+)
diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/DESC b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/DESC
new file mode 100644
index 00000000000..77474a51fbb
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/DESC
@@ -0,0 +1 @@
+prepare_restart_flush_strategy test. Take a look at prepare_restart_flush_strategy_test.cpp for details.
diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/FILES b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/FILES
new file mode 100644
index 00000000000..35ad6c54f3c
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/FILES
@@ -0,0 +1 @@
+prepare_restart_flush_strategy_test.cpp
diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp
new file mode 100644
index 00000000000..ac3dbb8fed2
--- /dev/null
+++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp
@@ -0,0 +1,297 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h>
+#include <vespa/searchcore/proton/flushengine/flush_target_candidates.h>
+#include <vespa/searchcore/proton/flushengine/tls_stats_map.h>
+#include <vespa/searchcore/proton/test/dummy_flush_handler.h>
+#include <vespa/searchcore/proton/test/dummy_flush_target.h>
+
+using namespace proton;
+using search::SerialNum;
+
+using SimpleFlushHandler = test::DummyFlushHandler;
+using FlushCandidatesList = std::vector<FlushTargetCandidates>;
+using Config = PrepareRestartFlushStrategy::Config;
+
+const Config DEFAULT_CFG(2.0, 4.0);
+
+struct SimpleFlushTarget : public test::DummyFlushTarget
+{
+    SerialNum flushedSerial;
+    uint64_t approxDiskBytes;
+    SimpleFlushTarget(const vespalib::string &name,
+                      SerialNum flushedSerial_,
+                      uint64_t approxDiskBytes_)
+        : test::DummyFlushTarget(name),
+          flushedSerial(flushedSerial_),
+          approxDiskBytes(approxDiskBytes_)
+    {}
+    SimpleFlushTarget(const vespalib::string &name,
+                      const Type &type,
+                      SerialNum flushedSerial_,
+                      uint64_t approxDiskBytes_)
+        : test::DummyFlushTarget(name, type, Component::OTHER),
+          flushedSerial(flushedSerial_),
+          approxDiskBytes(approxDiskBytes_)
+    {}
+    virtual SerialNum getFlushedSerialNum() const override {
+        return flushedSerial;
+    }
+    virtual uint64_t getApproxBytesToWriteToDisk() const override {
+        return approxDiskBytes;
+    }
+};
+
+class ContextsBuilder
+{
+private:
+    FlushContext::List _result;
+    std::map<vespalib::string, IFlushHandler::SP> _handlers;
+
+    IFlushHandler::SP createAndGetHandler(const vespalib::string &handlerName) {
+        auto itr = _handlers.find(handlerName);
+        if (itr != _handlers.end()) {
+            return itr->second;
+        }
+        IFlushHandler::SP handler = std::make_shared<SimpleFlushHandler>(handlerName);
+        _handlers.insert(std::make_pair(handlerName, handler));
+        return handler;
+    }
+
+public:
+    ContextsBuilder() : _result(), _handlers() {}
+    ContextsBuilder &add(const vespalib::string &handlerName,
+                         const vespalib::string &targetName,
+                         IFlushTarget::Type targetType,
+                         SerialNum flushedSerial,
+                         uint64_t approxDiskBytes) {
+        IFlushHandler::SP handler = createAndGetHandler(handlerName);
+        IFlushTarget::SP target = std::make_shared<SimpleFlushTarget>(targetName,
+                                                                      targetType,
+                                                                      flushedSerial,
+                                                                      approxDiskBytes);
+        _result.push_back(std::make_shared<FlushContext>(handler, target, 0, 0));
+        return *this;
+    }
+    ContextsBuilder &add(const vespalib::string &handlerName,
+                         const vespalib::string &targetName,
+                         SerialNum flushedSerial,
+                         uint64_t approxDiskBytes) {
+        return add(handlerName, targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes);
+    }
+    ContextsBuilder &add(const vespalib::string &targetName,
+                         SerialNum flushedSerial,
+                         uint64_t approxDiskBytes) {
+        return add("handler1", targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes);
+    }
+    ContextsBuilder &addGC(const vespalib::string &targetName,
+                           SerialNum flushedSerial,
+                           uint64_t approxDiskBytes) {
+        return add("handler1", targetName, IFlushTarget::Type::GC, flushedSerial, approxDiskBytes);
+    }
+    FlushContext::List build() const { return _result; }
+};
+
+class CandidatesBuilder
+{
+private:
+    const FlushContext::List *_sortedFlushContexts;
+    size_t _numCandidates;
+    flushengine::TlsStats _tlsStats;
+    Config _cfg;
+
+public:
+    CandidatesBuilder(const FlushContext::List &sortedFlushContexts)
+        : _sortedFlushContexts(&sortedFlushContexts),
+          _numCandidates(sortedFlushContexts.size()),
+          _tlsStats(1000, 11, 110),
+          _cfg(DEFAULT_CFG)
+    {}
+    CandidatesBuilder &flushContexts(const FlushContext::List &sortedFlushContexts) {
+        _sortedFlushContexts = &sortedFlushContexts;
+        _numCandidates = sortedFlushContexts.size();
+        return *this;
+    }
+    CandidatesBuilder &numCandidates(size_t numCandidates) {
+        _numCandidates = numCandidates;
+        return *this;
+    }
+    CandidatesBuilder &replayEnd(SerialNum replayEndSerial) {
+        flushengine::TlsStats oldTlsStats = _tlsStats;
+        _tlsStats = flushengine::TlsStats(oldTlsStats.getNumBytes(),
+                                          oldTlsStats.getFirstSerial(),
+                                          replayEndSerial);
+        return *this;
+    }
+    FlushTargetCandidates build() const {
+        return FlushTargetCandidates(*_sortedFlushContexts,
+                                     _numCandidates,
+                                     _tlsStats,
+                                     _cfg);
+    }
+};
+
+struct CandidatesFixture
+{
+    FlushContext::List emptyContexts;
+    CandidatesBuilder builder;
+    CandidatesFixture() : emptyContexts(), builder(emptyContexts) {}
+};
+
+TEST_F("require that tls replay cost is correct for 100% replay", CandidatesFixture)
+{
+    EXPECT_EQUAL(2000, f.builder.replayEnd(110).build().getTlsReplayCost());
+}
+
+TEST_F("require that tls replay cost is correct for 75% replay", CandidatesFixture)
+{
+    FlushContext::List contexts = ContextsBuilder().add("target1", 10, 0).add("target2", 35, 0).build();
+    EXPECT_EQUAL(1500, f.builder.flushContexts(contexts).numCandidates(1).replayEnd(110).
+            build().getTlsReplayCost());
+}
+
+TEST_F("require that tls replay cost is correct for 25% replay", CandidatesFixture)
+{
+    FlushContext::List contexts = ContextsBuilder().add("target1", 10, 0).add("target2", 85, 0).build();
+    EXPECT_EQUAL(500, f.builder.flushContexts(contexts).numCandidates(1).replayEnd(110).
+            build().getTlsReplayCost());
+}
+
+TEST_F("require that tls replay cost is correct for zero operations to replay", CandidatesFixture)
+{
+    EXPECT_EQUAL(0, f.builder.replayEnd(10).build().getTlsReplayCost());
+}
+
+TEST_F("require that flush cost is correct for zero flush targets", CandidatesFixture)
+{
+    EXPECT_EQUAL(0, f.builder.build().getFlushTargetsWriteCost());
+}
+
+TEST_F("require that flush cost is sum of flush targets", CandidatesFixture)
+{
+    FlushContext::List contexts = ContextsBuilder().add("target1", 20, 1000).add("target2", 30, 2000).build();
+    EXPECT_EQUAL(12000, f.builder.flushContexts(contexts).build().getFlushTargetsWriteCost());
+}
+
+
+flushengine::TlsStatsMap
+defaultTransactionLogStats()
+{
+    flushengine::TlsStatsMap::Map result;
+    result.insert(std::make_pair("handler1", flushengine::TlsStats(1000, 11, 110)));
+    result.insert(std::make_pair("handler2", flushengine::TlsStats(2000, 11, 110)));
+    return std::move(result);
+}
+
+struct FlushStrategyFixture
+{
+    flushengine::TlsStatsMap _tlsStatsMap;
+    PrepareRestartFlushStrategy strategy;
+    FlushStrategyFixture()
+        : _tlsStatsMap(defaultTransactionLogStats()),
+          strategy(DEFAULT_CFG)
+    {}
+    FlushContext::List getFlushTargets(const FlushContext::List &targetList,
+                                       const flushengine::TlsStatsMap &tlsStatsMap) const {
+        return strategy.getFlushTargets(targetList, tlsStatsMap);
+    }
+};
+
+vespalib::string
+toString(const FlushContext::List &flushContexts)
+{
+    std::ostringstream oss;
+    oss << "[";
+    bool comma = false;
+    for (const auto &flushContext : flushContexts) {
+        if (comma) {
+            oss << ",";
+        }
+        oss << flushContext->getTarget()->getName();
+        comma = true;
+    }
+    oss << "]";
+    return oss.str();
+}
+
+void
+assertFlushContexts(const vespalib::string &expected, const FlushContext::List &actual)
+{
+    EXPECT_EQUAL(expected, toString(actual));
+}
+
+/**
+ * For the following tests the content of the TLS is as follows:
+ *   - handler1: serial numbers 10 -> 110, 1000 bytes
+ *   - handler2: serial numbers 10 -> 110, 2000 bytes
+ *
+ * The cost config is: tlsReplayCost=2.0, flushTargetsWriteCost=4.0.
+ * The cost of replaying the complete TLS is then:
+ *   - handler1: 1000*2.0 = 2000
+ *   - handler2: 2000*2.0 = 4000
+ *
+ * With 3 flush targets that has getApproxBytesToWriteToDisk=167,
+ * the total write cost is 3*167*4.0 = 2004.
+ *
+ * This should give the baseline for understanding the following tests:
+ */
+
+TEST_F("require that the best strategy is flushing 0 targets", FlushStrategyFixture)
+{
+    FlushContext::List targets = f.getFlushTargets(ContextsBuilder().
+            add("foo", 10, 167).add("bar", 10, 167).add("baz", 10, 167).build(), f._tlsStatsMap);
+    TEST_DO(assertFlushContexts("[]", targets));
+}
+
+TEST_F("require that the best strategy is flushing all targets", FlushStrategyFixture)
+{
+    FlushContext::List targets = f.getFlushTargets(ContextsBuilder().
+            add("foo", 10, 166).add("bar", 10, 166).add("baz", 10, 166).build(), f._tlsStatsMap);
+    TEST_DO(assertFlushContexts("[bar,baz,foo]", targets));
+}
+
+TEST_F("require that the best strategy is flushing all targets (with different unflushed serial)", FlushStrategyFixture)
+{
+    FlushContext::List targets = f.getFlushTargets(ContextsBuilder().
+            add("foo", 10, 166).add("bar", 11, 166).add("baz", 12, 166).build(), f._tlsStatsMap);
+    TEST_DO(assertFlushContexts("[foo,bar,baz]", targets));
+}
+
+TEST_F("require that the best strategy is flushing 1 target", FlushStrategyFixture)
+{
+    FlushContext::List targets = f.getFlushTargets(ContextsBuilder().
+            add("foo", 10, 249).add("bar", 60, 125).add("baz", 60, 125).build(), f._tlsStatsMap);
+    TEST_DO(assertFlushContexts("[foo]", targets));
+}
+
+TEST_F("require that the best strategy is flushing 2 targets", FlushStrategyFixture)
+{
+    FlushContext::List targets = f.getFlushTargets(ContextsBuilder().
+            add("foo", 10, 124).add("bar", 11, 124).add("baz", 60, 251).build(), f._tlsStatsMap);
+    TEST_DO(assertFlushContexts("[foo,bar]", targets));
+}
+
+TEST_F("require that GC flush targets are removed", FlushStrategyFixture)
+{
+    FlushContext::List targets = f.getFlushTargets(ContextsBuilder().
+            addGC("foo", 10, 124).add("bar", 11, 124).add("baz", 60, 251).build(), f._tlsStatsMap);
+    TEST_DO(assertFlushContexts("[bar]", targets));
+}
+
+TEST_F("require that flush targets for different flush handlers are treated independently", FlushStrategyFixture)
+{
+    // best strategy for handler1 is flushing 1 target (foo)
+    // best strategy for handler2 is flushing 2 targets (baz,quz)
+    FlushContext::List targets = f.getFlushTargets(ContextsBuilder().
+            add("handler1", "foo", 10, 249).add("handler1", "bar", 60, 251).
+            add("handler2", "baz", 10, 499).add("handler2", "quz", 60, 499).build(), f._tlsStatsMap);
+    TEST_DO(assertFlushContexts("[foo,baz,quz]", targets));
+}
+
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/index/.gitignore b/searchcore/src/tests/proton/index/.gitignore
new file mode 100644
index 00000000000..00f61bc687d
--- /dev/null
+++ b/searchcore/src/tests/proton/index/.gitignore
@@ -0,0 +1,4 @@
+searchcore_diskindexcleaner_test_app
+searchcore_fusionrunner_test_app
+searchcore_indexcollection_test_app
+searchcore_indexmanager_test_app
diff --git a/searchcore/src/tests/proton/index/CMakeLists.txt b/searchcore/src/tests/proton/index/CMakeLists.txt
new file mode 100644
index 00000000000..46cbf4117ac
--- /dev/null
+++ b/searchcore/src/tests/proton/index/CMakeLists.txt
@@ -0,0 +1,33 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_indexmanager_test_app
+    SOURCES
+    indexmanager_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_index
+    searchcore_flushengine
+    searchcore_pcommon
+    searchcore_util
+)
+vespa_add_executable(searchcore_fusionrunner_test_app
+    SOURCES
+    fusionrunner_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_index
+    searchcore_pcommon
+    searchcore_util
+)
+vespa_add_executable(searchcore_diskindexcleaner_test_app
+    SOURCES
+    diskindexcleaner_test.cpp
+    DEPENDS
+    searchcore_index
+)
+vespa_add_executable(searchcore_indexcollection_test_app
+    SOURCES
+    indexcollection_test.cpp
+    DEPENDS
+    searchcore_index
+)
+vespa_add_test(NAME searchcore_index_test COMMAND sh index_test.sh)
diff --git a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp
new file mode 100644
index 00000000000..e462ba17dba
--- /dev/null
+++ b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for diskindexcleaner.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("diskindexcleaner_test");
+
+#include <vespa/searchcorespi/index/activediskindexes.h>
+#include <vespa/searchcorespi/index/diskindexcleaner.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <string>
+#include <vector>
+
+using std::string;
+using std::vector;
+using namespace searchcorespi::index;
+
+namespace {
+
+class Test : public vespalib::TestApp {
+    void requireThatAllIndexesOlderThanLastFusionIsRemoved();
+    void requireThatIndexesInUseAreNotRemoved();
+    void requireThatInvalidFlushIndexesAreRemoved();
+    void requireThatInvalidFusionIndexesAreRemoved();
+    void requireThatRemoveDontTouchNewIndexes();
+
+public:
+    int Main();
+};
+
+const string index_dir = "test_data";
+
+void removeTestData() {
+    FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str());
+}
+
+int
+Test::Main()
+{
+    TEST_INIT("diskindexcleaner_test");
+
+    TEST_DO(removeTestData());
+
+    TEST_DO(requireThatAllIndexesOlderThanLastFusionIsRemoved());
+    TEST_DO(requireThatIndexesInUseAreNotRemoved());
+    TEST_DO(requireThatInvalidFlushIndexesAreRemoved());
+    TEST_DO(requireThatInvalidFusionIndexesAreRemoved());
+    TEST_DO(requireThatRemoveDontTouchNewIndexes());
+
+    TEST_DO(removeTestData());
+
+    TEST_DONE();
+}
+
+void createIndex(const string &name) {
+    FastOS_FileInterface::MakeDirIfNotPresentOrExit(index_dir.c_str());
+    const string dir_name = index_dir + "/" + name;
+    FastOS_FileInterface::MakeDirIfNotPresentOrExit(dir_name.c_str());
+    const string serial_file = dir_name + "/serial.dat";
+    FastOS_File file(serial_file.c_str());
+    file.OpenWriteOnlyTruncate();
+}
+
+vector<string> readIndexes() {
+    vector<string> indexes;
+    FastOS_DirectoryScan dir_scan(index_dir.c_str());
+    while (dir_scan.ReadNext()) {
+        string name = dir_scan.GetName();
+        if (!dir_scan.IsDirectory() || name.find("index.") != 0) {
+            continue;
+        }
+        indexes.push_back(name);
+    }
+    return indexes;
+}
+
+template <class Container>
+bool contains(Container c, typename Container::value_type v) {
+    return find(c.begin(), c.end(), v) != c.end();
+}
+
+void createIndexes() {
+    createIndex("index.flush.0");
+    createIndex("index.flush.1");
+    createIndex("index.fusion.1");
+    createIndex("index.flush.2");
+    createIndex("index.fusion.2");
+    createIndex("index.flush.3");
+    createIndex("index.flush.4");
+}
+
+void Test::requireThatAllIndexesOlderThanLastFusionIsRemoved() {
+    createIndexes();
+    ActiveDiskIndexes active_indexes;
+    DiskIndexCleaner::clean(index_dir, active_indexes);
+    vector<string> indexes = readIndexes();
+    EXPECT_EQUAL(3u, indexes.size());
+    EXPECT_TRUE(contains(indexes, "index.fusion.2"));
+    EXPECT_TRUE(contains(indexes, "index.flush.3"));
+    EXPECT_TRUE(contains(indexes, "index.flush.4"));
+}
+
+void Test::requireThatIndexesInUseAreNotRemoved() {
+    createIndexes();
+    ActiveDiskIndexes active_indexes;
+    active_indexes.setActive(index_dir + "/index.fusion.1");
+    active_indexes.setActive(index_dir + "/index.flush.2");
+    DiskIndexCleaner::clean(index_dir, active_indexes);
+    vector<string> indexes = readIndexes();
+    EXPECT_TRUE(contains(indexes, "index.fusion.1"));
+    EXPECT_TRUE(contains(indexes, "index.flush.2"));
+
+    active_indexes.notActive(index_dir + "/index.fusion.1");
+    active_indexes.notActive(index_dir + "/index.flush.2");
+    DiskIndexCleaner::clean(index_dir, active_indexes);
+    indexes = readIndexes();
+    EXPECT_TRUE(!contains(indexes, "index.fusion.1"));
+    EXPECT_TRUE(!contains(indexes, "index.flush.2"));
+}
+
+void Test::requireThatInvalidFlushIndexesAreRemoved() {
+    createIndexes();
+    FastOS_File((index_dir + "/index.flush.4/serial.dat").c_str()).Delete();
+    ActiveDiskIndexes active_indexes;
+    DiskIndexCleaner::clean(index_dir, active_indexes);
+    vector<string> indexes = readIndexes();
+    EXPECT_EQUAL(2u, indexes.size());
+    EXPECT_TRUE(contains(indexes, "index.fusion.2"));
+    EXPECT_TRUE(contains(indexes, "index.flush.3"));
+}
+
+void Test::requireThatInvalidFusionIndexesAreRemoved() {
+    createIndexes();
+    FastOS_File((index_dir + "/index.fusion.2/serial.dat").c_str()).Delete();
+    ActiveDiskIndexes active_indexes;
+    DiskIndexCleaner::clean(index_dir, active_indexes);
+    vector<string> indexes = readIndexes();
+    EXPECT_EQUAL(4u, indexes.size());
+    EXPECT_TRUE(contains(indexes, "index.fusion.1"));
+    EXPECT_TRUE(contains(indexes, "index.flush.2"));
+    EXPECT_TRUE(contains(indexes, "index.flush.3"));
+    EXPECT_TRUE(contains(indexes, "index.flush.4"));
+}
+
+void Test::requireThatRemoveDontTouchNewIndexes() {
+    createIndexes();
+    FastOS_File((index_dir + "/index.flush.4/serial.dat").c_str()).Delete();
+    ActiveDiskIndexes active_indexes;
+    DiskIndexCleaner::removeOldIndexes(index_dir, active_indexes);
+    vector<string> indexes = readIndexes();
+    EXPECT_EQUAL(3u, indexes.size());
+    EXPECT_TRUE(contains(indexes, "index.fusion.2"));
+    EXPECT_TRUE(contains(indexes, "index.flush.3"));
+    EXPECT_TRUE(contains(indexes, "index.flush.4"));
+}
+
+}  // namespace
+
+TEST_APPHOOK(Test);
diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp
new file mode 100644
index 00000000000..dd2bcee97b2
--- /dev/null
+++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp
@@ -0,0 +1,328 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for fusionrunner.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("fusionrunner_test");
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/searchlib/memoryindex/memoryindex.h>
+#include <vespa/searchcore/proton/index/indexmanager.h>
+#include <vespa/searchcorespi/index/fusionrunner.h>
+#include <vespa/searchcorespi/index/fusionspec.h>
+#include <vespa/searchlib/attribute/fixedsourceselector.h>
+#include <vespa/searchlib/diskindex/diskindex.h>
+#include <vespa/searchlib/diskindex/indexbuilder.h>
+#include <vespa/searchlib/fef/matchdata.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/searchlib/fef/termfieldmatchdata.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/query/tree/simplequery.h>
+#include <vespa/searchlib/queryeval/fake_requestcontext.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <set>
+
+using document::Document;
+using document::FieldValue;
+using search::FixedSourceSelector;
+using search::diskindex::DiskIndex;
+using search::diskindex::IndexBuilder;
+using search::fef::MatchData;
+using search::fef::MatchDataLayout;
+using search::fef::TermFieldHandle;
+using search::fef::TermFieldMatchData;
+using search::index::DocBuilder;
+using search::index::Schema;
+using search::index::DummyFileHeaderContext;
+using search::memoryindex::MemoryIndex;
+using search::query::SimpleStringTerm;
+using search::queryeval::Blueprint;
+using search::queryeval::FieldSpec;
+using search::queryeval::FieldSpecList;
+using search::queryeval::ISourceSelector;
+using search::queryeval::SearchIterator;
+using search::queryeval::FakeRequestContext;
+using std::set;
+using std::string;
+using namespace proton;
+using search::TuneFileAttributes;
+using search::TuneFileIndexing;
+using search::TuneFileIndexManager;
+using search::TuneFileSearch;
+using searchcorespi::index::FusionRunner;
+using searchcorespi::index::FusionSpec;
+using proton::ExecutorThreadingService;
+
+namespace {
+
+#define TEST_CALL(func) \
+    setUp(); \
+    func; \
+    tearDown()
+
+class Test : public vespalib::TestApp {
+    std::unique_ptr<FusionRunner> _fusion_runner;
+    FixedSourceSelector::UP _selector;
+    FusionSpec _fusion_spec;
+    DummyFileHeaderContext _fileHeaderContext;
+    ExecutorThreadingService _threadingService;
+    IndexManager::MaintainerOperations _ops;
+
+    void setUp();
+    void tearDown();
+
+    void createIndex(const string &dir, uint32_t id, bool fusion = false);
+    void checkResults(uint32_t fusion_id, const uint32_t *ids, size_t size);
+
+    void requireThatNoDiskIndexesGiveId0();
+    void requireThatOneDiskIndexCausesCopy();
+    void requireThatTwoDiskIndexesCauseFusion();
+    void requireThatFusionCanRunOnMultipleDiskIndexes();
+    void requireThatOldFusionIndexCanBePartOfNewFusion();
+    void requireThatSelectorsCanBeRebased();
+
+public:
+    Test()
+        : _fusion_runner(),
+          _selector(),
+          _fusion_spec(),
+          _fileHeaderContext(),
+          _threadingService(),
+          _ops(_fileHeaderContext,
+               TuneFileIndexManager(), 0,
+               _threadingService)
+    {
+    }
+    int Main();
+};
+
+int
+Test::Main()
+{
+    TEST_INIT("fusionrunner_test");
+
+    if (_argc > 0) {
+        DummyFileHeaderContext::setCreator(_argv[0]);
+    }
+    TEST_CALL(requireThatNoDiskIndexesGiveId0());
+    TEST_CALL(requireThatOneDiskIndexCausesCopy());
+    TEST_CALL(requireThatTwoDiskIndexesCauseFusion());
+    TEST_CALL(requireThatFusionCanRunOnMultipleDiskIndexes());
+    TEST_CALL(requireThatOldFusionIndexCanBePartOfNewFusion());
+    TEST_CALL(requireThatSelectorsCanBeRebased());
+
+    TEST_DONE();
+}
+
+const string base_dir = "fusion_test_data";
+const string field_name = "field_name";
+const string term = "foo";
+const uint32_t disk_id[] = { 1, 2, 21, 42 };
+
+Schema getSchema() {
+    Schema schema;
+    schema.addIndexField(
+            Schema::IndexField(field_name, Schema::STRING));
+    return schema;
+}
+
+void Test::setUp() {
+    FastOS_FileInterface::EmptyAndRemoveDirectory(base_dir.c_str());
+    _fusion_runner.reset(new FusionRunner(base_dir, getSchema(),
+                                 TuneFileAttributes(),
+                                 _fileHeaderContext));
+    const string selector_base = base_dir + "/index.flush.0/selector";
+    _selector.reset(new FixedSourceSelector(0, selector_base));
+    _fusion_spec = FusionSpec();
+}
+
+void Test::tearDown() {
+    FastOS_FileInterface::EmptyAndRemoveDirectory(base_dir.c_str());
+    _selector.reset(0);
+}
+
+Document::UP buildDocument(DocBuilder & doc_builder, int id, const string &word) {
+    vespalib::asciistream ost;
+    ost << "doc::" << id;
+    doc_builder.startDocument(ost.str());
+    doc_builder.startIndexField(field_name).addStr(word).endField();
+    return doc_builder.endDocument();
+}
+
+void addDocument(DocBuilder & doc_builder, MemoryIndex &index, ISourceSelector &selector,
+                 uint8_t index_id, uint32_t docid, const string &word) {
+    Document::UP doc = buildDocument(doc_builder, docid, word);
+    index.insertDocument(docid, *doc);
+    index.commit(std::shared_ptr<search::IDestructorCallback>());
+    selector.setSource(docid, index_id);
+}
+
+void Test::createIndex(const string &dir, uint32_t id, bool fusion) {
+    FastOS_FileInterface::MakeDirIfNotPresentOrExit(dir.c_str());
+    vespalib::asciistream ost;
+    if (fusion) {
+        ost << dir << "/index.fusion." << id;
+        _fusion_spec.last_fusion_id = id;
+    } else {
+        ost << dir << "/index.flush." << id;
+        _fusion_spec.flush_ids.push_back(id);
+    }
+    const string index_dir = ost.str();
+
+    Schema schema = getSchema();
+    DocBuilder doc_builder(schema);
+    MemoryIndex memory_index(schema, _threadingService.indexFieldInverter(),
+                             _threadingService.indexFieldWriter());
+    addDocument(doc_builder, memory_index, *_selector, id, id + 0, term);
+    addDocument(doc_builder, memory_index, *_selector, id, id + 1, "bar");
+    addDocument(doc_builder, memory_index, *_selector, id, id + 2, "baz");
+    addDocument(doc_builder, memory_index, *_selector, id, id + 3, "qux");
+    _threadingService.indexFieldWriter().sync();
+
+    const uint32_t docIdLimit =
+        std::min(memory_index.getDocIdLimit(), _selector->getDocIdLimit());
+    IndexBuilder index_builder(schema);
+    index_builder.setPrefix(index_dir);
+    TuneFileIndexing tuneFileIndexing;
+    TuneFileAttributes tuneFileAttributes;
+    index_builder.open(docIdLimit, memory_index.getNumWords(),
+                       tuneFileIndexing,
+                       _fileHeaderContext);
+    memory_index.dump(index_builder);
+    index_builder.close();
+
+    _selector->extractSaveInfo(index_dir + "/selector")->
+        save(tuneFileAttributes, _fileHeaderContext);
+}
+
+set<uint32_t> readFusionIds(const string &dir) {
+    set<uint32_t> ids;
+    FastOS_DirectoryScan dir_scan(dir.c_str());
+    while (dir_scan.ReadNext()) {
+        if (!dir_scan.IsDirectory()) {
+            continue;
+        }
+        vespalib::string name = dir_scan.GetName();
+        const vespalib::string prefix("index.fusion.");
+        vespalib::string::size_type pos = name.find(prefix);
+        if (pos != 0) {
+            continue;
+        }
+        vespalib::string idString = name.substr(prefix.size());
+        vespalib::asciistream ist(idString);
+        uint32_t id;
+        ist >> id;
+        ids.insert(id);
+    }
+    return ids;
+}
+
+vespalib::string getFusionIndexName(uint32_t fusion_id) {
+   vespalib::asciistream ost;
+   ost << base_dir << "/index.fusion." << fusion_id;
+   return ost.str();
+}
+
+void Test::checkResults(uint32_t fusion_id, const uint32_t *ids, size_t size) {
+    FakeRequestContext requestContext;
+    DiskIndex disk_index(getFusionIndexName(fusion_id));
+    ASSERT_TRUE(disk_index.setup(TuneFileSearch()));
+    uint32_t fieldId = 0;
+
+    MatchDataLayout mdl;
+    TermFieldHandle handle = mdl.allocTermField(fieldId);
+    MatchData::UP match_data = mdl.createMatchData();
+
+    FieldSpec field(field_name, fieldId, handle);
+    FieldSpecList fields;
+    fields.add(field);
+
+    search::queryeval::Searchable &searchable = disk_index;
+    SimpleStringTerm node(term, field_name, fieldId, search::query::Weight(0));
+    Blueprint::UP blueprint = searchable.createBlueprint(requestContext, fields, node);
+    blueprint->fetchPostings(true);
+    SearchIterator::UP search = blueprint->createSearch(*match_data, true);
+    search->initFullRange();
+    for (size_t i = 0; i < size; ++i) {
+        EXPECT_TRUE(search->seek(ids[i]));
+    }
+}
+
+void Test::requireThatNoDiskIndexesGiveId0() {
+    uint32_t fusion_id = _fusion_runner->fuse(_fusion_spec, 0u, _ops);
+    EXPECT_EQUAL(0u, fusion_id);
+}
+
+void Test::requireThatOneDiskIndexCausesCopy() {
+    createIndex(base_dir, disk_id[0]);
+    uint32_t fusion_id = _fusion_runner->fuse(_fusion_spec, 0u, _ops);
+    EXPECT_EQUAL(disk_id[0], fusion_id);
+    set<uint32_t> fusion_ids = readFusionIds(base_dir);
+    ASSERT_TRUE(!fusion_ids.empty());
+    EXPECT_EQUAL(1u, fusion_ids.size());
+    EXPECT_EQUAL(fusion_id, *fusion_ids.begin());
+
+    checkResults(fusion_id, disk_id, 1);
+}
+
+void Test::requireThatTwoDiskIndexesCauseFusion() {
+    createIndex(base_dir, disk_id[0]);
+    createIndex(base_dir, disk_id[1]);
+    uint32_t fusion_id = _fusion_runner->fuse(_fusion_spec, 0u, _ops);
+    EXPECT_EQUAL(disk_id[1], fusion_id);
+    set<uint32_t> fusion_ids = readFusionIds(base_dir);
+    ASSERT_TRUE(!fusion_ids.empty());
+    EXPECT_EQUAL(1u, fusion_ids.size());
+    EXPECT_EQUAL(fusion_id, *fusion_ids.begin());
+
+    checkResults(fusion_id, disk_id, 2);
+}
+
+void Test::requireThatFusionCanRunOnMultipleDiskIndexes() {
+    createIndex(base_dir, disk_id[0]);
+    createIndex(base_dir, disk_id[1]);
+    createIndex(base_dir, disk_id[2]);
+    createIndex(base_dir, disk_id[3]);
+    uint32_t fusion_id = _fusion_runner->fuse(_fusion_spec, 0u, _ops);
+    EXPECT_EQUAL(disk_id[3], fusion_id);
+    set<uint32_t> fusion_ids = readFusionIds(base_dir);
+    ASSERT_TRUE(!fusion_ids.empty());
+    EXPECT_EQUAL(1u, fusion_ids.size());
+    EXPECT_EQUAL(fusion_id, *fusion_ids.begin());
+
+    checkResults(fusion_id, disk_id, 4);
+}
+
+void Test::requireThatOldFusionIndexCanBePartOfNewFusion() {
+    createIndex(base_dir, disk_id[0], true);
+    createIndex(base_dir, disk_id[1]);
+    uint32_t fusion_id = _fusion_runner->fuse(_fusion_spec, 0u, _ops);
+    EXPECT_EQUAL(disk_id[1], fusion_id);
+    set<uint32_t> fusion_ids = readFusionIds(base_dir);
+    ASSERT_TRUE(!fusion_ids.empty());
+    EXPECT_EQUAL(2u, fusion_ids.size());
+    EXPECT_EQUAL(disk_id[0], *fusion_ids.begin());
+    EXPECT_EQUAL(fusion_id, *(++fusion_ids.begin()));
+
+    checkResults(fusion_id, disk_id, 2);
+}
+
+void Test::requireThatSelectorsCanBeRebased() {
+    createIndex(base_dir, disk_id[0]);
+    createIndex(base_dir, disk_id[1]);
+    uint32_t fusion_id = _fusion_runner->fuse(_fusion_spec, 0u, _ops);
+
+    _fusion_spec.flush_ids.clear();
+    _fusion_spec.last_fusion_id = fusion_id;
+    createIndex(base_dir, disk_id[2]);
+    fusion_id = _fusion_runner->fuse(_fusion_spec, 0u, _ops);
+
+    checkResults(fusion_id, disk_id, 3);
+}
+
+}  // namespace
+
+TEST_APPHOOK(Test);
diff --git a/searchcore/src/tests/proton/index/index_test.sh b/searchcore/src/tests/proton/index/index_test.sh
new file mode 100644
index 00000000000..91c271128fe
--- /dev/null
+++ b/searchcore/src/tests/proton/index/index_test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+$VALGRIND ./searchcore_diskindexcleaner_test_app
+$VALGRIND ./searchcore_fusionrunner_test_app
+$VALGRIND ./searchcore_indexcollection_test_app
+$VALGRIND ./searchcore_indexmanager_test_app
diff --git a/searchcore/src/tests/proton/index/index_writer/.gitignore b/searchcore/src/tests/proton/index/index_writer/.gitignore
new file mode 100644
index 00000000000..bbada541cf0
--- /dev/null
+++ b/searchcore/src/tests/proton/index/index_writer/.gitignore
@@ -0,0 +1 @@
+searchcore_index_writer_test_app
diff --git a/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt b/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt
new file mode 100644
index 00000000000..88db11dbdba
--- /dev/null
+++ b/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_index_writer_test_app
+    SOURCES
+    index_writer_test.cpp
+    DEPENDS
+    searchcore_index
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_index_writer_test_app COMMAND searchcore_index_writer_test_app)
diff --git a/searchcore/src/tests/proton/index/index_writer/DESC b/searchcore/src/tests/proton/index/index_writer/DESC
new file mode 100644
index 00000000000..ea5596d3039
--- /dev/null
+++ b/searchcore/src/tests/proton/index/index_writer/DESC
@@ -0,0 +1 @@
+index writer test. Take a look at index_writer_test.cpp for details.
diff --git a/searchcore/src/tests/proton/index/index_writer/FILES b/searchcore/src/tests/proton/index/index_writer/FILES
new file mode 100644
index 00000000000..3b26c7f84ad
--- /dev/null
+++ b/searchcore/src/tests/proton/index/index_writer/FILES
@@ -0,0 +1 @@
+index_writer_test.cpp
diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp
new file mode 100644
index 00000000000..1ff5a6e8649
--- /dev/null
+++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp
@@ -0,0 +1,117 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("index_writer_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/index/index_writer.h>
+#include <vespa/searchcore/proton/test/mock_index_manager.h>
+#include <vespa/searchlib/index/docbuilder.h>
+
+using namespace proton;
+using namespace search;
+using namespace search::index;
+using namespace searchcorespi;
+
+using document::Document;
+
+std::string
+toString(const std::vector<SerialNum> &vec)
+{
+    std::ostringstream oss;
+    for (size_t i = 0; i < vec.size(); ++i) {
+        if (i > 0) oss << ",";
+        oss << vec[i];
+    }
+    return oss.str();
+}
+
+struct MyIndexManager : public test::MockIndexManager
+{
+    typedef std::map<uint32_t, std::vector<SerialNum> > LidMap;
+    LidMap puts;
+    LidMap removes;
+    SerialNum current;
+    SerialNum flushed;
+    SerialNum commitSerial;
+    MyIndexManager() : puts(), removes(), current(0), flushed(0),
+                       commitSerial(0)
+    {
+    }
+    std::string getPut(uint32_t lid) {
+        return toString(puts[lid]);
+    }
+    std::string getRemove(uint32_t lid) {
+        return toString(removes[lid]);
+    }
+    // Implements IIndexManager
+    virtual void putDocument(uint32_t lid, const Document &,
+                             SerialNum serialNum) override {
+        puts[lid].push_back(serialNum);
+    }
+    virtual void removeDocument(uint32_t lid,
+                                SerialNum serialNum) override {
+        removes[lid].push_back(serialNum);
+    }
+    virtual void commit(SerialNum serialNum,
+                        OnWriteDoneType) override {
+        commitSerial = serialNum;
+    }
+    virtual SerialNum getCurrentSerialNum() const override {
+        return current;
+    }
+    virtual SerialNum getFlushedSerialNum() const override {
+        return flushed;
+    }
+};
+
+struct Fixture
+{
+    IIndexManager::SP iim;
+    MyIndexManager   &mim;
+    IndexWriter      iw;
+    Schema            schema;
+    DocBuilder        builder;
+    Document::UP      dummyDoc;
+    Fixture()
+        : iim(new MyIndexManager()),
+          mim(static_cast<MyIndexManager &>(*iim)),
+          iw(iim),
+          schema(),
+          builder(schema),
+          dummyDoc(createDoc(1234)) // This content of this is not used
+    {
+    }
+    Document::UP createDoc(uint32_t lid) {
+        builder.startDocument(vespalib::make_string("doc:test:%u", lid));
+        return builder.endDocument();
+    }
+    void put(SerialNum serialNum, const search::DocumentIdT lid) {
+        iw.put(serialNum, *dummyDoc, lid);
+        iw.commit(serialNum, std::shared_ptr<IDestructorCallback>());
+    }
+    void remove(SerialNum serialNum, const search::DocumentIdT lid) {
+        iw.remove(serialNum, lid);
+        iw.commit(serialNum, std::shared_ptr<IDestructorCallback>());
+    }
+};
+
+TEST_F("require that index adapter ignores old operations", Fixture)
+{
+    f.mim.flushed = 10;
+    f.put(8, 1);
+    f.remove(9, 2);
+    EXPECT_EQUAL("", f.mim.getPut(1));
+    EXPECT_EQUAL("", f.mim.getRemove(2));
+}
+
+TEST_F("require that commit is forwarded to index manager", Fixture)
+{
+    f.iw.commit(10, std::shared_ptr<IDestructorCallback>());
+    EXPECT_EQUAL(10u, f.mim.commitSerial);
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/index/indexcollection_test.cpp b/searchcore/src/tests/proton/index/indexcollection_test.cpp
new file mode 100644
index 00000000000..f27b9c86260
--- /dev/null
+++ b/searchcore/src/tests/proton/index/indexcollection_test.cpp
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("indexcollection_test");
+
+#include <vespa/searchcore/proton/matching/fakesearchcontext.h>
+#include <vespa/searchcorespi/index/indexcollection.h>
+#include <vespa/searchcorespi/index/warmupindexcollection.h>
+#include <vespa/searchlib/queryeval/fake_searchable.h>
+#include <vespa/searchlib/attribute/fixedsourceselector.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+
+using search::queryeval::ISourceSelector;
+using search::queryeval::FakeSearchable;
+using search::FixedSourceSelector;
+using namespace proton;
+using namespace searchcorespi;
+
+namespace {
+
+class Test : public vespalib::TestApp,
+             public IWarmupDone
+{
+    std::shared_ptr<ISourceSelector> _selector;
+    std::shared_ptr<IndexSearchable> _source1;
+    std::shared_ptr<IndexSearchable> _source2;
+    std::shared_ptr<IndexSearchable> _fusion_source;
+    vespalib::ThreadStackExecutor    _executor;
+    std::shared_ptr<IndexSearchable> _warmup;
+
+    void requireThatSearchablesCanBeAppended(IndexCollection::UP fsc);
+    void requireThatSearchablesCanBeReplaced(IndexCollection::UP fsc);
+    void requireThatReplaceAndRenumberUpdatesCollectionAfterFusion();
+    IndexCollection::UP createWarmup(const IndexCollection::SP & prev, const IndexCollection::SP & next);
+    virtual void warmupDone(ISearchableIndexCollection::SP current) {
+        (void) current;
+    }
+
+public:
+    Test() : _selector(new FixedSourceSelector(0, "fs1")),
+             _source1(new FakeIndexSearchable),
+             _source2(new FakeIndexSearchable),
+             _fusion_source(new FakeIndexSearchable),
+             _executor(1, 128*1024),
+             _warmup(new FakeIndexSearchable)
+    {}
+
+    int Main();
+};
+
+
+IndexCollection::UP
+Test::createWarmup(const IndexCollection::SP & prev, const IndexCollection::SP & next)
+{
+    return IndexCollection::UP(new WarmupIndexCollection(1.0, prev, next, *_warmup, _executor, *this));
+}
+
+int
+Test::Main()
+{
+    TEST_INIT("indexcollection_test");
+
+    TEST_DO(requireThatSearchablesCanBeAppended(IndexCollection::UP(new IndexCollection(_selector))));
+    TEST_DO(requireThatSearchablesCanBeReplaced(IndexCollection::UP(new IndexCollection(_selector))));
+    TEST_DO(requireThatReplaceAndRenumberUpdatesCollectionAfterFusion());
+    {
+        IndexCollection::SP prev(new IndexCollection(_selector));
+        IndexCollection::SP next(new IndexCollection(_selector));
+        requireThatSearchablesCanBeAppended(createWarmup(prev, next));
+        EXPECT_EQUAL(0u, prev->getSourceCount());
+        EXPECT_EQUAL(1u, next->getSourceCount());
+    }
+    {
+        IndexCollection::SP prev(new IndexCollection(_selector));
+        IndexCollection::SP next(new IndexCollection(_selector));
+        requireThatSearchablesCanBeReplaced(createWarmup(prev, next));
+        EXPECT_EQUAL(0u, prev->getSourceCount());
+        EXPECT_EQUAL(1u, next->getSourceCount());
+    }
+
+    TEST_DONE();
+}
+
+void Test::requireThatSearchablesCanBeAppended(IndexCollection::UP fsc) {
+    const uint32_t id = 42;
+
+    fsc->append(id, _source1);
+    EXPECT_EQUAL(1u, fsc->getSourceCount());
+    EXPECT_EQUAL(id, fsc->getSourceId(0));
+}
+
+void Test::requireThatSearchablesCanBeReplaced(IndexCollection::UP fsc) {
+    const uint32_t id = 42;
+
+    fsc->append(id, _source1);
+    EXPECT_EQUAL(1u, fsc->getSourceCount());
+    EXPECT_EQUAL(id, fsc->getSourceId(0));
+    EXPECT_EQUAL(_source1.get(), &fsc->getSearchable(0));
+
+    fsc->replace(id, _source2);
+    EXPECT_EQUAL(1u, fsc->getSourceCount());
+    EXPECT_EQUAL(id, fsc->getSourceId(0));
+    EXPECT_EQUAL(_source2.get(), &fsc->getSearchable(0));
+}
+
+void Test::requireThatReplaceAndRenumberUpdatesCollectionAfterFusion() {
+    IndexCollection fsc(_selector);
+
+    fsc.append(0, _source1);
+    fsc.append(1, _source1);
+    fsc.append(2, _source1);
+    fsc.append(3, _source2);
+    EXPECT_EQUAL(4u, fsc.getSourceCount());
+
+    const uint32_t id_diff = 2;
+    IndexCollection::UP new_fsc =
+        IndexCollection::replaceAndRenumber(
+                _selector, fsc, id_diff, _fusion_source);
+    EXPECT_EQUAL(2u, new_fsc->getSourceCount());
+    EXPECT_EQUAL(0u, new_fsc->getSourceId(0));
+    EXPECT_EQUAL(_fusion_source.get(), &new_fsc->getSearchable(0));
+    EXPECT_EQUAL(1u, new_fsc->getSourceId(1));
+    EXPECT_EQUAL(_source2.get(), &new_fsc->getSearchable(1));
+}
+
+}  // namespace
+
+TEST_APPHOOK(Test);
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp
new file mode 100644
index 00000000000..95f5a3b50ce
--- /dev/null
+++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp
@@ -0,0 +1,690 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for IndexManager.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("indexmanager_test");
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/searchcore/proton/index/indexmanager.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+#include <vespa/searchcorespi/index/indexcollection.h>
+#include <vespa/searchcorespi/index/indexflushtarget.h>
+#include <vespa/searchcorespi/index/indexfusiontarget.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/memoryindex/dictionary.h>
+#include <vespa/searchlib/memoryindex/documentinverter.h>
+#include <vespa/searchlib/memoryindex/fieldinverter.h>
+#include <vespa/searchlib/memoryindex/ordereddocumentinserter.h>
+#include <vespa/searchlib/memoryindex/compact_document_words_store.h>
+#include <vespa/searchlib/queryeval/isourceselector.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/searchlib/util/dirtraverse.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <set>
+
+using document::Document;
+using document::FieldValue;
+using search::btree::EntryRef;
+using search::index::DocBuilder;
+using search::index::Schema;
+using search::index::DummyFileHeaderContext;
+using search::memoryindex::Dictionary;
+using search::memoryindex::CompactDocumentWordsStore;
+using search::queryeval::Source;
+using search::SequencedTaskExecutor;
+using search::SerialNum;
+using std::set;
+using std::string;
+using vespalib::Gate;
+using vespalib::Monitor;
+using vespalib::MonitorGuard;
+using namespace proton;
+using namespace searchcorespi;
+using namespace searchcorespi::index;
+using search::TuneFileIndexing;
+using search::TuneFileIndexManager;
+using search::TuneFileAttributes;
+using vespalib::BlockingThreadStackExecutor;
+using vespalib::ThreadStackExecutor;
+using search::makeLambdaTask;
+
+namespace {
+
+class IndexManagerDummyReconfigurer : public searchcorespi::IIndexManager::Reconfigurer
+{
+    virtual bool
+    reconfigure(vespalib::Closure0<bool>::UP closure)
+    {
+        bool ret = true;
+        if (closure.get() != NULL)
+            ret = closure->call(); // Perform index manager reconfiguration now
+        return ret;
+    }
+
+};
+
+const string index_dir = "test_data";
+const string field_name = "field";
+const uint32_t docid = 1;
+
+Schema getSchema() {
+    Schema schema;
+    schema.addIndexField(Schema::IndexField(field_name, Schema::STRING));
+    return schema;
+}
+
+void removeTestData() {
+    FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str());
+}
+
+Document::UP buildDocument(DocBuilder &doc_builder, int id,
+                           const string &word) {
+    vespalib::asciistream ost;
+    ost << "doc::" << id;
+    doc_builder.startDocument(ost.str());
+    doc_builder.startIndexField(field_name).addStr(word).endField();
+    return doc_builder.endDocument();
+}
+
+std::shared_ptr<search::IDestructorCallback> emptyDestructorCallback;
+
+struct Fixture {
+    SerialNum _serial_num;
+    IndexManagerDummyReconfigurer _reconfigurer;
+    DummyFileHeaderContext _fileHeaderContext;
+    ExecutorThreadingService _writeService;
+    std::unique_ptr<IndexManager> _index_manager;
+    Schema _schema;
+    DocBuilder _builder;
+
+    Fixture()
+        : _serial_num(0),
+          _reconfigurer(),
+          _fileHeaderContext(),
+          _writeService(),
+          _index_manager(),
+          _schema(getSchema()),
+          _builder(_schema)
+    {
+        removeTestData();
+        vespalib::mkdir(index_dir, false);
+        _writeService.sync();
+        resetIndexManager();
+    }
+
+    ~Fixture() {
+        _writeService.shutdown();
+    }
+
+    template <class FunctionType>
+    inline void runAsMaster(FunctionType &&function) {
+        _writeService.master().execute(makeLambdaTask(std::move(function)));
+        _writeService.master().sync();
+    }
+    template <class FunctionType>
+    inline void runAsIndex(FunctionType &&function) {
+        _writeService.index().execute(makeLambdaTask(std::move(function)));
+        _writeService.index().sync();
+    }
+    void flushIndexManager();
+    Document::UP addDocument(uint32_t docid);
+    void resetIndexManager();
+    void removeDocument(uint32_t docId, SerialNum serialNum) {
+        runAsIndex([&]() { _index_manager->removeDocument(docId, serialNum);
+                              _index_manager->commit(serialNum,
+                                                     emptyDestructorCallback);
+                          });
+        _writeService.indexFieldWriter().sync();
+    }
+};
+
+void Fixture::flushIndexManager() {
+    vespalib::Executor::Task::UP task;
+    SerialNum serialNum = _index_manager->getCurrentSerialNum();
+    auto &maintainer = _index_manager->getMaintainer();
+    runAsMaster([&]() { task = maintainer.initFlush(serialNum, NULL); });
+    if (task.get()) {
+        task->run();
+    }
+}
+
+Document::UP Fixture::addDocument(uint32_t id) {
+    Document::UP doc = buildDocument(_builder, id, "foo");
+    SerialNum serialNum = ++_serial_num;
+    runAsIndex([&]() { _index_manager->putDocument(id, *doc, serialNum);
+                          _index_manager->commit(serialNum,
+                                                 emptyDestructorCallback); });
+    _writeService.indexFieldWriter().sync();
+    return doc;
+}
+
+void Fixture::resetIndexManager() {
+    _index_manager.reset(0);
+    _index_manager.reset(
+            new IndexManager(index_dir, 0.0, 2, 0, getSchema(), getSchema(),
+                             _reconfigurer, _writeService, _writeService.getMasterExecutor(),
+                             TuneFileIndexManager(), TuneFileAttributes(),
+                             _fileHeaderContext));
+}
+
+TEST_F("requireThatEmptyMemoryIndexIsNotFlushed", Fixture) {
+    IIndexCollection::SP sources = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(1u, sources->getSourceCount());
+
+    f.flushIndexManager();
+
+    sources = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(1u, sources->getSourceCount());
+}
+
+TEST_F("requireThatEmptyMemoryIndexIsFlushedIfSourceSelectorChanged", Fixture)
+{
+    IIndexCollection::SP sources = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(1u, sources->getSourceCount());
+
+    f.removeDocument(docid, 42);
+    f.flushIndexManager();
+
+    sources = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(2u, sources->getSourceCount());
+}
+
+set<uint32_t> readDiskIds(const string &dir, const string &type) {
+    set<uint32_t> ids;
+    FastOS_DirectoryScan dir_scan(dir.c_str());
+    while (dir_scan.ReadNext()) {
+        if (!dir_scan.IsDirectory()) {
+            continue;
+        }
+        string name = dir_scan.GetName();
+        const string flush_prefix("index." + type + ".");
+        string::size_type pos = name.find(flush_prefix);
+        if (pos != 0) {
+            continue;
+        }
+        vespalib::string idString(name.substr(flush_prefix.size()));
+        vespalib::asciistream ist(idString);
+        uint32_t id;
+        ist >> id;
+        ids.insert(id);
+    }
+    return ids;
+}
+
+TEST_F("requireThatMemoryIndexIsFlushed", Fixture) {
+    FastOS_StatInfo stat;
+    {
+        f.addDocument(docid);
+
+        IIndexCollection::SP sources =
+            f._index_manager->getMaintainer().getSourceCollection();
+        EXPECT_EQUAL(1u, sources->getSourceCount());
+        EXPECT_EQUAL(1u, sources->getSourceId(0));
+
+        IndexFlushTarget target(f._index_manager->getMaintainer());
+        EXPECT_EQUAL(0, target.getLastFlushTime().time());
+        vespalib::Executor::Task::UP flushTask;
+        f.runAsMaster([&]() { flushTask = target.initFlush(1); });
+        flushTask->run();
+        EXPECT_TRUE(FastOS_File::Stat("test_data/index.flush.1", &stat));
+        EXPECT_EQUAL(stat._modifiedTime, target.getLastFlushTime().time());
+
+        sources = f._index_manager->getMaintainer().getSourceCollection();
+        EXPECT_EQUAL(2u, sources->getSourceCount());
+        EXPECT_EQUAL(1u, sources->getSourceId(0));
+        EXPECT_EQUAL(2u, sources->getSourceId(1));
+
+        set<uint32_t> disk_ids = readDiskIds(index_dir, "flush");
+        ASSERT_TRUE(disk_ids.size() == 1);
+        EXPECT_EQUAL(1u, *disk_ids.begin());
+
+        FlushStats stats = target.getLastFlushStats();
+        EXPECT_EQUAL("test_data/index.flush.1", stats.getPath());
+        EXPECT_EQUAL(7u, stats.getPathElementsToLog());
+    }
+    { // verify last flush time when loading disk index
+        f.resetIndexManager();
+        IndexFlushTarget target(f._index_manager->getMaintainer());
+        EXPECT_EQUAL(stat._modifiedTime, target.getLastFlushTime().time());
+
+        // updated serial number & flush time when nothing to flush
+        FastOS_Thread::Sleep(8000);
+        fastos::TimeStamp now = fastos::ClockSystem::now();
+        vespalib::Executor::Task::UP task;
+        f.runAsMaster([&]() { task = target.initFlush(2); });
+        EXPECT_TRUE(task.get() == NULL);
+        EXPECT_EQUAL(2u, target.getFlushedSerialNum());
+        EXPECT_LESS(stat._modifiedTime, target.getLastFlushTime().time());
+        EXPECT_APPROX(now.time(), target.getLastFlushTime().time(), 8);
+    }
+}
+
+TEST_F("requireThatMultipleFlushesGivesMultipleIndexes", Fixture) {
+    size_t flush_count = 10;
+    for (size_t i = 0; i < flush_count; ++i) {
+        f.addDocument(docid);
+        f.flushIndexManager();
+    }
+    set<uint32_t> disk_ids = readDiskIds(index_dir, "flush");
+    EXPECT_EQUAL(flush_count, disk_ids.size());
+    uint32_t i = 1;
+    for (set<uint32_t>::iterator it = disk_ids.begin(); it != disk_ids.end();
+         ++it) {
+        EXPECT_EQUAL(i++, *it);
+    }
+}
+
+TEST_F("requireThatMaxFlushesSetsUrgent", Fixture) {
+    size_t flush_count = 20;
+    for (size_t i = 0; i < flush_count; ++i) {
+        f.addDocument(docid);
+        f.flushIndexManager();
+    }
+    IndexFusionTarget target(f._index_manager->getMaintainer());
+    EXPECT_TRUE(target.needUrgentFlush());
+}
+
+uint32_t getSource(const IIndexCollection &sources, uint32_t id) {
+    return sources.getSourceSelector().createIterator()->getSource(id);
+}
+
+TEST_F("requireThatPutDocumentUpdatesSelector", Fixture) {
+    f.addDocument(docid);
+    IIndexCollection::SP sources = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(1u, getSource(*sources, docid));
+    f.flushIndexManager();
+    f.addDocument(docid + 1);
+    sources = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(1u, getSource(*sources, docid));
+    EXPECT_EQUAL(2u, getSource(*sources, docid + 1));
+}
+
+TEST_F("requireThatRemoveDocumentUpdatesSelector", Fixture) {
+    Document::UP doc = f.addDocument(docid);
+    IIndexCollection::SP sources = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(1u, getSource(*sources, docid));
+    f.flushIndexManager();
+    f.removeDocument(docid, ++f._serial_num);
+    sources = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(2u, getSource(*sources, docid));
+}
+
+TEST_F("requireThatSourceSelectorIsFlushed", Fixture) {
+    f.addDocument(docid);
+    f.flushIndexManager();
+    FastOS_File file((index_dir + "/index.flush.1/selector.dat").c_str());
+    ASSERT_TRUE(file.OpenReadOnlyExisting());
+}
+
+TEST_F("requireThatFlushStatsAreCalculated", Fixture) {
+    Schema schema(getSchema());
+    Dictionary dict(schema);
+    SequencedTaskExecutor invertThreads(2);
+    SequencedTaskExecutor pushThreads(2);
+    search::memoryindex::DocumentInverter inverter(schema, invertThreads,
+                                                   pushThreads);
+
+    uint64_t fixed_index_size = dict.getMemoryUsage().allocatedBytes();
+    uint64_t index_size = dict.getMemoryUsage().allocatedBytes() - fixed_index_size;
+    /// Must account for both docid 0 being reserved and the extra after.
+    uint64_t selector_size = (1) * sizeof(Source);
+    EXPECT_EQUAL(index_size, f._index_manager->getMaintainer().getFlushStats().memory_before_bytes -
+                             f._index_manager->getMaintainer().getFlushStats().memory_after_bytes);
+    EXPECT_EQUAL(0u, f._index_manager->getMaintainer().getFlushStats().disk_write_bytes);
+    EXPECT_EQUAL(0u, f._index_manager->getMaintainer().getFlushStats().cpu_time_required);
+
+    Document::UP doc = f.addDocument(docid);
+    inverter.invertDocument(docid, *doc);
+    invertThreads.sync();
+    inverter.pushDocuments(dict,
+                           std::shared_ptr<search::IDestructorCallback>());
+    pushThreads.sync();
+    index_size = dict.getMemoryUsage().allocatedBytes() - fixed_index_size;
+
+    /// Must account for both docid 0 being reserved and the extra after.
+    selector_size = (docid + 1) * sizeof(Source);
+    EXPECT_EQUAL(index_size,
+                 f._index_manager->getMaintainer().getFlushStats().memory_before_bytes -
+                 f._index_manager->getMaintainer().getFlushStats().memory_after_bytes);
+    EXPECT_EQUAL(selector_size + index_size,
+                 f._index_manager->getMaintainer().getFlushStats().disk_write_bytes);
+    EXPECT_EQUAL(selector_size * (3+1) + index_size,
+                 f._index_manager->getMaintainer().getFlushStats().cpu_time_required);
+
+    doc = f.addDocument(docid + 10);
+    inverter.invertDocument(docid + 10, *doc);
+    doc = f.addDocument(docid + 100);
+    inverter.invertDocument(docid + 100, *doc);
+    invertThreads.sync();
+    inverter.pushDocuments(dict,
+                           std::shared_ptr<search::IDestructorCallback>());
+    pushThreads.sync();
+    index_size = dict.getMemoryUsage().allocatedBytes() - fixed_index_size;
+    /// Must account for both docid 0 being reserved and the extra after.
+    selector_size = (docid + 100 + 1) * sizeof(Source);
+    EXPECT_EQUAL(index_size,
+                 f._index_manager->getMaintainer().getFlushStats().memory_before_bytes -
+                 f._index_manager->getMaintainer().getFlushStats().memory_after_bytes);
+    EXPECT_EQUAL(selector_size + index_size,
+                 f._index_manager->getMaintainer().getFlushStats().disk_write_bytes);
+    EXPECT_EQUAL(selector_size * (3+1) + index_size,
+                 f._index_manager->getMaintainer().getFlushStats().cpu_time_required);
+}
+
+TEST_F("requireThatFusionStatsAreCalculated", Fixture) {
+    f.addDocument(docid);
+    EXPECT_EQUAL(0u, f._index_manager->getMaintainer().getFusionStats().diskUsage);
+    f.flushIndexManager();
+    ASSERT_TRUE(f._index_manager->getMaintainer().getFusionStats().diskUsage > 0);
+}
+
+TEST_F("requireThatPutDocumentUpdatesSerialNum", Fixture) {
+    f._serial_num = 0;
+    EXPECT_EQUAL(0u, f._index_manager->getCurrentSerialNum());
+    f.addDocument(docid);
+    EXPECT_EQUAL(1u, f._index_manager->getCurrentSerialNum());
+}
+
+TEST_F("requireThatRemoveDocumentUpdatesSerialNum", Fixture) {
+    f._serial_num = 0;
+    Document::UP doc = f.addDocument(docid);
+    EXPECT_EQUAL(1u, f._index_manager->getCurrentSerialNum());
+    f.removeDocument(docid, ++f._serial_num);
+    EXPECT_EQUAL(2u, f._index_manager->getCurrentSerialNum());
+}
+
+TEST_F("requireThatFlushUpdatesSerialNum", Fixture) {
+    f._serial_num = 0;
+    f.addDocument(docid);
+    EXPECT_EQUAL(1u, f._index_manager->getCurrentSerialNum());
+    EXPECT_EQUAL(0u, f._index_manager->getFlushedSerialNum());
+    f.flushIndexManager();
+    EXPECT_EQUAL(1u, f._index_manager->getCurrentSerialNum());
+    EXPECT_EQUAL(1u, f._index_manager->getFlushedSerialNum());
+}
+
+TEST_F("requireThatFusionUpdatesIndexes", Fixture) {
+    for (size_t i = 0; i < 10; ++i) {
+        f.addDocument(docid + i);
+        f.flushIndexManager();
+    }
+    uint32_t ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+    IIndexCollection::SP
+        source_list(f._index_manager->getMaintainer().getSourceCollection());
+    EXPECT_EQUAL(10u + 1, source_list->getSourceCount());  // disk + mem
+    EXPECT_EQUAL(ids[2], getSource(*source_list, docid + 2));
+    EXPECT_EQUAL(ids[6], getSource(*source_list, docid + 6));
+
+    FusionSpec fusion_spec;
+    fusion_spec.flush_ids.assign(ids, ids + 4);
+    f._index_manager->getMaintainer().runFusion(fusion_spec);
+
+    set<uint32_t> fusion_ids = readDiskIds(index_dir, "fusion");
+    EXPECT_EQUAL(1u, fusion_ids.size());
+    EXPECT_EQUAL(ids[3], *fusion_ids.begin());
+
+    source_list = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(10u + 1 - 4 + 1, source_list->getSourceCount());
+    EXPECT_EQUAL(0u, getSource(*source_list, docid + 2));
+    EXPECT_EQUAL(3u, getSource(*source_list, docid + 6));
+}
+
+TEST_F("requireThatFlushTriggersFusion", Fixture) {
+    const uint32_t fusion_trigger = 5;
+    f.resetIndexManager();
+
+    for (size_t i = 1; i <= fusion_trigger; ++i) {
+        f.addDocument(docid);
+        f.flushIndexManager();
+    }
+    IFlushTarget::SP target(new IndexFusionTarget(f._index_manager->getMaintainer()));
+    target->initFlush(0)->run();
+    f.addDocument(docid);
+    f.flushIndexManager();
+    set<uint32_t> fusion_ids = readDiskIds(index_dir, "fusion");
+    EXPECT_EQUAL(1u, fusion_ids.size());
+    EXPECT_EQUAL(5u, *fusion_ids.begin());
+    set<uint32_t> flush_ids = readDiskIds(index_dir, "flush");
+    EXPECT_EQUAL(1u, flush_ids.size());
+    EXPECT_EQUAL(6u, *flush_ids.begin());
+}
+
+TEST_F("requireThatFusionTargetIsSetUp", Fixture) {
+    f.addDocument(docid);
+    f.flushIndexManager();
+    f.addDocument(docid);
+    f.flushIndexManager();
+    IFlushTarget::List lst(f._index_manager->getFlushTargets());
+    EXPECT_EQUAL(2u, lst.size());
+    IFlushTarget::SP target(lst.at(1));
+    EXPECT_EQUAL("memoryindex.fusion", target->getName());
+    EXPECT_FALSE(target->needUrgentFlush());
+    f.addDocument(docid);
+    f.flushIndexManager();
+    lst = f._index_manager->getFlushTargets();
+    EXPECT_EQUAL(2u, lst.size());
+    target = lst.at(1);
+    EXPECT_EQUAL("memoryindex.fusion", target->getName());
+    EXPECT_TRUE(target->needUrgentFlush());
+}
+
+TEST_F("requireThatFusionCleansUpOldIndexes", Fixture) {
+    f.addDocument(docid);
+    f.flushIndexManager();
+    // hold reference to index.flush.1
+    IIndexCollection::SP fsc = f._index_manager->getMaintainer().getSourceCollection();
+
+    f.addDocument(docid + 1);
+    f.flushIndexManager();
+
+    set<uint32_t> flush_ids = readDiskIds(index_dir, "flush");
+    EXPECT_EQUAL(2u, flush_ids.size());
+
+    FusionSpec fusion_spec;
+    fusion_spec.flush_ids.push_back(1);
+    fusion_spec.flush_ids.push_back(2);
+    f._index_manager->getMaintainer().runFusion(fusion_spec);
+
+    flush_ids = readDiskIds(index_dir, "flush");
+    EXPECT_EQUAL(1u, flush_ids.size());
+    EXPECT_EQUAL(1u, *flush_ids.begin());
+
+    fsc.reset();
+    f._index_manager->getMaintainer().removeOldDiskIndexes();
+    flush_ids = readDiskIds(index_dir, "flush");
+    EXPECT_EQUAL(0u, flush_ids.size());
+}
+
+bool contains(const IIndexCollection &fsc, uint32_t id) {
+    set<uint32_t> ids;
+    for (size_t i = 0; i < fsc.getSourceCount(); ++i) {
+        ids.insert(fsc.getSourceId(i));
+    }
+    return ids.find(id) != ids.end();
+}
+
+bool indexExists(const string &type, uint32_t id) {
+    set<uint32_t> disk_ids = readDiskIds(index_dir, type);
+    return disk_ids.find(id) != disk_ids.end();
+}
+
+TEST_F("requireThatDiskIndexesAreLoadedOnStartup", Fixture) {
+    f.addDocument(docid);
+    f.flushIndexManager();
+    f._index_manager.reset(0);
+
+    ASSERT_TRUE(indexExists("flush", 1));
+    f.resetIndexManager();
+
+    IIndexCollection::SP fsc = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(2u, fsc->getSourceCount());
+    EXPECT_TRUE(contains(*fsc, 1u));
+    EXPECT_TRUE(contains(*fsc, 2u));
+    EXPECT_EQUAL(1u, getSource(*fsc, docid));
+    fsc.reset();
+
+
+    f.addDocument(docid + 1);
+    f.flushIndexManager();
+    ASSERT_TRUE(indexExists("flush", 2));
+    FusionSpec fusion_spec;
+    fusion_spec.flush_ids.push_back(1);
+    fusion_spec.flush_ids.push_back(2);
+    f._index_manager->getMaintainer().runFusion(fusion_spec);
+    f._index_manager.reset(0);
+
+    ASSERT_TRUE(!indexExists("flush", 1));
+    ASSERT_TRUE(!indexExists("flush", 2));
+    ASSERT_TRUE(indexExists("fusion", 2));
+    f.resetIndexManager();
+
+    fsc = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(2u, fsc->getSourceCount());
+    EXPECT_TRUE(contains(*fsc, 0u));
+    EXPECT_TRUE(contains(*fsc, 1u));
+    EXPECT_EQUAL(0u, getSource(*fsc, docid));
+    EXPECT_EQUAL(0u, getSource(*fsc, docid + 1));
+    /// Must account for both docid 0 being reserved and the extra after.
+    EXPECT_EQUAL(docid + 2, fsc->getSourceSelector().getDocIdLimit());
+    fsc.reset();
+
+
+    f.addDocument(docid + 2);
+    f.flushIndexManager();
+    f._index_manager.reset(0);
+
+    ASSERT_TRUE(indexExists("fusion", 2));
+    ASSERT_TRUE(indexExists("flush", 3));
+    f.resetIndexManager();
+
+    fsc = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(3u, fsc->getSourceCount());
+    EXPECT_TRUE(contains(*fsc, 0u));
+    EXPECT_TRUE(contains(*fsc, 1u));
+    EXPECT_TRUE(contains(*fsc, 2u));
+    EXPECT_EQUAL(0u, getSource(*fsc, docid));
+    EXPECT_EQUAL(0u, getSource(*fsc, docid + 1));
+    EXPECT_EQUAL(1u, getSource(*fsc, docid + 2));
+    fsc.reset();
+}
+
+TEST_F("requireThatExistingIndexesAreToBeFusionedOnStartup", Fixture) {
+    f.addDocument(docid);
+    f.flushIndexManager();
+    f.addDocument(docid + 1);
+    f.flushIndexManager();
+    f.resetIndexManager();
+
+    IFlushTarget::SP target(new IndexFusionTarget(f._index_manager->getMaintainer()));
+    target->initFlush(0)->run();
+    f.addDocument(docid);
+    f.flushIndexManager();
+
+    set<uint32_t> fusion_ids = readDiskIds(index_dir, "fusion");
+    EXPECT_EQUAL(1u, fusion_ids.size());
+    EXPECT_EQUAL(2u, *fusion_ids.begin());
+}
+
+TEST_F("requireThatSerialNumberIsWrittenOnFlush", Fixture) {
+    f.addDocument(docid);
+    f.flushIndexManager();
+    FastOS_File file((index_dir + "/index.flush.1/serial.dat").c_str());
+    EXPECT_TRUE(file.OpenReadOnly());
+}
+
+TEST_F("requireThatSerialNumberIsCopiedOnFusion", Fixture) {
+    f.addDocument(docid);
+    f.flushIndexManager();
+    f.addDocument(docid);
+    f.flushIndexManager();
+    FusionSpec fusion_spec;
+    fusion_spec.flush_ids.push_back(1);
+    fusion_spec.flush_ids.push_back(2);
+    f._index_manager->getMaintainer().runFusion(fusion_spec);
+    FastOS_File file((index_dir + "/index.fusion.2/serial.dat").c_str());
+    EXPECT_TRUE(file.OpenReadOnly());
+}
+
+TEST_F("requireThatSerialNumberIsReadOnLoad", Fixture) {
+    f.addDocument(docid);
+    f.flushIndexManager();
+    EXPECT_EQUAL(f._serial_num, f._index_manager->getFlushedSerialNum());
+    f.resetIndexManager();
+    EXPECT_EQUAL(f._serial_num, f._index_manager->getFlushedSerialNum());
+
+    f.addDocument(docid);
+    f.flushIndexManager();
+    f.addDocument(docid);
+    f.flushIndexManager();
+    search::SerialNum serial = f._serial_num;
+    f.addDocument(docid);
+    f.resetIndexManager();
+    EXPECT_EQUAL(serial, f._index_manager->getFlushedSerialNum());
+}
+
+void crippleFusion(uint32_t fusionId) {
+    vespalib::asciistream ost;
+    ost << index_dir << "/index.flush." << fusionId << "/serial.dat";
+    FastOS_File(ost.str().c_str()).Delete();
+}
+
+TEST_F("requireThatFailedFusionIsRetried", Fixture) {
+    f.resetIndexManager();
+
+    f.addDocument(docid);
+    f.flushIndexManager();
+    f.addDocument(docid);
+    f.flushIndexManager();
+
+    crippleFusion(2);
+
+    IndexFusionTarget target(f._index_manager->getMaintainer());
+    vespalib::Executor::Task::UP fusionTask = target.initFlush(1);
+    fusionTask->run();
+
+    FusionSpec spec = f._index_manager->getMaintainer().getFusionSpec();
+    set<uint32_t> fusion_ids = readDiskIds(index_dir, "fusion");
+    EXPECT_TRUE(fusion_ids.empty());
+    EXPECT_EQUAL(0u, spec.last_fusion_id);
+    EXPECT_EQUAL(2u, spec.flush_ids.size());
+    EXPECT_EQUAL(1u, spec.flush_ids[0]);
+    EXPECT_EQUAL(2u, spec.flush_ids[1]);
+}
+
+TEST_F("require that wipeHistory updates schema on disk", Fixture) {
+    Schema empty_schema;
+    f.addDocument(docid);
+    f.flushIndexManager();
+    f.runAsMaster([&]() { f._index_manager->setSchema(empty_schema,
+                                                      empty_schema); });
+    f.addDocument(docid);
+    f.flushIndexManager();
+
+    Schema s;
+    s.loadFromFile("test_data/index.flush.1/schema.txt");
+    EXPECT_EQUAL(1u, s.getNumIndexFields());
+
+    f.runAsMaster([&]() { f._index_manager->wipeHistory(f._serial_num,
+                                                        empty_schema); });
+
+    s.loadFromFile("test_data/index.flush.1/schema.txt");
+    EXPECT_EQUAL(0u, s.getNumIndexFields());
+}
+
+
+}  // namespace
+
+TEST_MAIN() {
+    TEST_DO(removeTestData());
+    DummyFileHeaderContext::setCreator("indexmanager_test");
+    TEST_RUN_ALL();
+    TEST_DO(removeTestData());
+}
diff --git a/searchcore/src/tests/proton/initializer/.gitignore b/searchcore/src/tests/proton/initializer/.gitignore
new file mode 100644
index 00000000000..486dbb0b11e
--- /dev/null
+++ b/searchcore/src/tests/proton/initializer/.gitignore
@@ -0,0 +1 @@
+searchcore_task_runner_test_app
diff --git a/searchcore/src/tests/proton/initializer/CMakeLists.txt b/searchcore/src/tests/proton/initializer/CMakeLists.txt
new file mode 100644
index 00000000000..0f20324cae3
--- /dev/null
+++ b/searchcore/src/tests/proton/initializer/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_task_runner_test_app
+    SOURCES
+    task_runner_test.cpp
+    DEPENDS
+    searchcore_initializer
+)
+vespa_add_test(NAME searchcore_task_runner_test_app COMMAND searchcore_task_runner_test_app)
diff --git a/searchcore/src/tests/proton/initializer/DESC b/searchcore/src/tests/proton/initializer/DESC
new file mode 100644
index 00000000000..be2743a181e
--- /dev/null
+++ b/searchcore/src/tests/proton/initializer/DESC
@@ -0,0 +1 @@
+TaskRunner test. Take a look at task_runner_test.cpp for details.
diff --git a/searchcore/src/tests/proton/initializer/FILES b/searchcore/src/tests/proton/initializer/FILES
new file mode 100644
index 00000000000..bbbbe1c2d86
--- /dev/null
+++ b/searchcore/src/tests/proton/initializer/FILES
@@ -0,0 +1 @@
+task_runner_test.cpp
diff --git a/searchcore/src/tests/proton/initializer/task_runner_test.cpp b/searchcore/src/tests/proton/initializer/task_runner_test.cpp
new file mode 100644
index 00000000000..afa807fd0e6
--- /dev/null
+++ b/searchcore/src/tests/proton/initializer/task_runner_test.cpp
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("task_runner_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+#include <vespa/searchcore/proton/initializer/task_runner.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <mutex>
+
+using proton::initializer::InitializerTask;
+using proton::initializer::TaskRunner;
+
+struct TestLog
+{
+    std::mutex _lock;
+    vespalib::string _log;
+    using UP = std::unique_ptr<TestLog>;
+
+    TestLog()
+        : _lock(),
+          _log()
+    {
+    }
+
+    void append(vespalib::string str) {
+        std::lock_guard<std::mutex> guard(_lock);
+        _log += str;
+    }
+
+    vespalib::string result() const { return _log; }
+};
+
+class NamedTask : public InitializerTask
+{
+protected:
+    vespalib::string  _name;
+    TestLog          &_log;
+public:
+    NamedTask(const vespalib::string &name, TestLog &log)
+        : _name(name),
+          _log(log)
+    {
+    }
+
+    virtual void run() { _log.append(_name); }
+};
+
+
+struct TestJob {
+    TestLog::UP _log;
+    InitializerTask::SP _root;
+
+    TestJob(TestLog::UP log, InitializerTask::SP root)
+        : _log(std::move(log)),
+          _root(std::move(root))
+    {
+    }
+
+    static TestJob setupCDependsOnAandB()
+    {
+        TestLog::UP log = std::make_unique<TestLog>();
+        InitializerTask::SP A(std::make_shared<NamedTask>("A", *log));
+        InitializerTask::SP B(std::make_shared<NamedTask>("B", *log));
+        InitializerTask::SP C(std::make_shared<NamedTask>("C", *log));
+        C->addDependency(A);
+        C->addDependency(B);
+        return TestJob(std::move(log), std::move(C));
+    }
+
+    static TestJob setupDiamond()
+    {
+        TestLog::UP log = std::make_unique<TestLog>();
+        InitializerTask::SP A(std::make_shared<NamedTask>("A", *log));
+        InitializerTask::SP B(std::make_shared<NamedTask>("B", *log));
+        InitializerTask::SP C(std::make_shared<NamedTask>("C", *log));
+        InitializerTask::SP D(std::make_shared<NamedTask>("D", *log));
+        C->addDependency(A);
+        C->addDependency(B);
+        A->addDependency(D);
+        B->addDependency(D);
+        return TestJob(std::move(log), std::move(C));
+    }
+};
+
+
+struct Fixture
+{
+    vespalib::ThreadStackExecutor _executor;
+    TaskRunner _taskRunner;
+
+    Fixture(uint32_t numThreads = 1)
+        : _executor(numThreads, 128 * 1024),
+          _taskRunner(_executor)
+    {
+    }
+
+    void run(const InitializerTask::SP &task) { _taskRunner.runTask(task); }
+};
+
+
+TEST_F("1 thread, 2 dependees, 1 depender", Fixture(1))
+{
+    TestJob job = TestJob::setupCDependsOnAandB();
+    f.run(job._root);
+    EXPECT_EQUAL("ABC", job._log->result());
+}
+
+TEST_F("1 thread, dag graph", Fixture(1))
+{
+    for (int iter = 0; iter < 1000; ++iter) {
+        TestJob job = TestJob::setupDiamond();
+        f.run(job._root);
+        EXPECT_EQUAL("DABC", job._log->result());
+    }
+}
+
+TEST_F("multiple threads, dag graph", Fixture(10))
+{
+    int dabc_count = 0;
+    int dbac_count = 0;
+    for (int iter = 0; iter < 1000; ++iter) {
+        TestJob job = TestJob::setupDiamond();
+        f.run(job._root);
+        vespalib::string result = job._log->result();
+        EXPECT_TRUE("DABC" == result || "DBAC" == result);
+        if ("DABC" == result) {
+            ++dabc_count;
+        }
+        if ("DBAC" == result) {
+            ++dbac_count;
+        }
+    }
+    LOG(info, "dabc=%d, dbac=%d", dabc_count, dbac_count);
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/matchengine/.gitignore b/searchcore/src/tests/proton/matchengine/.gitignore
new file mode 100644
index 00000000000..4e2a3a6df8d
--- /dev/null
+++ b/searchcore/src/tests/proton/matchengine/.gitignore
@@ -0,0 +1,6 @@
+.depend
+Makefile
+matchengine_test
+query_test
+queryenvbuilder_test
+searchcore_matchengine_test_app
diff --git a/searchcore/src/tests/proton/matchengine/CMakeLists.txt b/searchcore/src/tests/proton/matchengine/CMakeLists.txt
new file mode 100644
index 00000000000..32c0b47ae4b
--- /dev/null
+++ b/searchcore/src/tests/proton/matchengine/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_matchengine_test_app
+    SOURCES
+    matchengine.cpp
+    DEPENDS
+    searchcore_matchengine
+    searchcore_matching
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_matchengine_test_app COMMAND searchcore_matchengine_test_app)
diff --git a/searchcore/src/tests/proton/matchengine/DESC b/searchcore/src/tests/proton/matchengine/DESC
new file mode 100644
index 00000000000..1530502e2de
--- /dev/null
+++ b/searchcore/src/tests/proton/matchengine/DESC
@@ -0,0 +1 @@
+matchengine test. Take a look at matchengine.cpp for details.
diff --git a/searchcore/src/tests/proton/matchengine/FILES b/searchcore/src/tests/proton/matchengine/FILES
new file mode 100644
index 00000000000..91961877ee2
--- /dev/null
+++ b/searchcore/src/tests/proton/matchengine/FILES
@@ -0,0 +1 @@
+matchengine.cpp
diff --git a/searchcore/src/tests/proton/matchengine/matchengine.cpp b/searchcore/src/tests/proton/matchengine/matchengine.cpp
new file mode 100644
index 00000000000..617c2f81b74
--- /dev/null
+++ b/searchcore/src/tests/proton/matchengine/matchengine.cpp
@@ -0,0 +1,214 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("matchengine_test");
+
+#include <vespa/searchcore/proton/matchengine/matchengine.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+
+using namespace proton;
+using namespace search::engine;
+using namespace vespalib::slime;
+using vespalib::Slime;
+
+class MySearchHandler : public ISearchHandler {
+    size_t _numHits;
+    std::string _name;
+    std::string _reply;
+public:
+    MySearchHandler(size_t numHits = 0,
+                    const std::string & name = "my",
+                    const std::string & reply = "myreply") :
+        _numHits(numHits), _name(name), _reply(reply) {}
+    virtual DocsumReply::UP getDocsums(const DocsumRequest &) {
+        return DocsumReply::UP(new DocsumReply);
+    }
+
+    virtual search::engine::SearchReply::UP match(
+            const ISearchHandler::SP &,
+            const search::engine::SearchRequest &,
+            vespalib::ThreadBundle &) const {
+        SearchReply::UP retval(new SearchReply);
+        for (size_t i = 0; i < _numHits; ++i) {
+            retval->hits.push_back(SearchReply::Hit());
+        }
+        return retval;
+    }
+};
+
+class LocalSearchClient : public SearchClient {
+private:
+    vespalib::Monitor _monitor;
+    SearchReply::UP   _reply;
+
+public:
+    void searchDone(SearchReply::UP reply) {
+        vespalib::MonitorGuard guard(_monitor);
+        _reply = std::move(reply);
+        guard.broadcast();
+    }
+
+    SearchReply::UP getReply(uint32_t millis) {
+        vespalib::MonitorGuard guard(_monitor);
+        vespalib::TimedWaiter waiter(guard, millis);
+        while (_reply.get() == NULL && waiter.hasTime()) {
+            waiter.wait();
+        }
+        return std::move(_reply);
+    }
+};
+
+TEST("requireThatSearchesExecute")
+{
+    int numMatcherThreads = 16;
+    MatchEngine engine(numMatcherThreads, 1, 7);
+    engine.setOnline();
+    engine.setNodeUp(true);
+
+    MySearchHandler::SP handler(new MySearchHandler);
+    DocTypeName dtnvfoo("foo");
+    engine.putSearchHandler(dtnvfoo, handler);
+
+    LocalSearchClient client;
+    SearchRequest::Source request(new SearchRequest());
+    SearchReply::UP reply = engine.search(std::move(request), client);
+    EXPECT_TRUE(reply.get() == NULL);
+
+    reply = client.getReply(10000);
+    EXPECT_TRUE(reply.get() != NULL);
+}
+
+bool
+assertSearchReply(MatchEngine & engine, const std::string & searchDocType, size_t expHits)
+{
+    SearchRequest *request = new SearchRequest();
+    request->propertiesMap.lookupCreate(search::MapNames::MATCH).add("documentdb.searchdoctype", searchDocType);
+    LocalSearchClient client;
+    engine.search(SearchRequest::Source(request), client);
+    SearchReply::UP reply = client.getReply(10000);
+    return EXPECT_EQUAL(expHits, reply->hits.size());
+}
+
+TEST("requireThatCorrectHandlerIsUsed")
+{
+    MatchEngine engine(1, 1, 7);
+    engine.setOnline();
+    engine.setNodeUp(true);
+    ISearchHandler::SP h1(new MySearchHandler(2));
+    ISearchHandler::SP h2(new MySearchHandler(4));
+    ISearchHandler::SP h3(new MySearchHandler(6));
+    DocTypeName dtnvfoo("foo");
+    DocTypeName dtnvbar("bar");
+    DocTypeName dtnvbaz("baz");
+    engine.putSearchHandler(dtnvfoo, h1);
+    engine.putSearchHandler(dtnvbar, h2);
+    engine.putSearchHandler(dtnvbaz, h3);
+
+    EXPECT_TRUE(assertSearchReply(engine, "foo", 2));
+    EXPECT_TRUE(assertSearchReply(engine, "bar", 4));
+    EXPECT_TRUE(assertSearchReply(engine, "baz", 6));
+    EXPECT_TRUE(assertSearchReply(engine, "not", 4)); // uses the first (sorted on name)
+}
+
+struct ObserveBundleMatchHandler : MySearchHandler {
+    typedef std::shared_ptr<ObserveBundleMatchHandler> SP;
+    mutable size_t bundleSize;
+    ObserveBundleMatchHandler() : bundleSize(0) {}
+
+    virtual search::engine::SearchReply::UP match(
+            const ISearchHandler::SP &,
+            const search::engine::SearchRequest &,
+            vespalib::ThreadBundle &threadBundle) const
+    {
+        bundleSize = threadBundle.size();
+        return SearchReply::UP(new SearchReply);
+    }
+};
+
+TEST("requireThatBundlesAreUsed")
+{
+    MatchEngine engine(15, 5, 7);
+    engine.setOnline();
+    engine.setNodeUp(true);
+
+    ObserveBundleMatchHandler::SP handler(new ObserveBundleMatchHandler());
+    DocTypeName dtnvfoo("foo");
+    engine.putSearchHandler(dtnvfoo, handler);
+
+    LocalSearchClient client;
+    SearchRequest::Source request(new SearchRequest());
+    engine.search(std::move(request), client);
+    SearchReply::UP reply = client.getReply(10000);
+    EXPECT_EQUAL(7u, reply->getDistributionKey());
+    EXPECT_EQUAL(5u, handler->bundleSize);
+}
+
+TEST("requireThatHandlersCanBeRemoved")
+{
+    MatchEngine engine(1, 1, 7);
+    engine.setOnline();
+    engine.setNodeUp(true);
+    ISearchHandler::SP h(new MySearchHandler(1));
+    DocTypeName docType("foo");
+    engine.putSearchHandler(docType, h);
+
+    ISearchHandler::SP r = engine.getSearchHandler(docType);
+    EXPECT_TRUE(r.get() != NULL);
+    EXPECT_TRUE(h.get() == r.get());
+
+    r = engine.removeSearchHandler(docType);
+    EXPECT_TRUE(r.get() != NULL);
+    EXPECT_TRUE(h.get() == r.get());
+
+    r = engine.getSearchHandler(docType);
+    EXPECT_TRUE(r.get() == NULL);
+}
+
+TEST("requireThatEngineCanBeSetOffline")
+{
+    MatchEngine engine(1, 1, 7);
+    engine.setNodeUp(true);
+    engine.setOnline();
+    engine.setInService();
+    ASSERT_TRUE(engine.isOnline());
+    engine.setOffline();
+    ASSERT_FALSE(engine.isOnline());
+    engine.setOnline();
+    ASSERT_TRUE(engine.isOnline());
+    engine.setOutOfService();
+    ASSERT_FALSE(engine.isOnline());
+}
+
+TEST("requireThatEmptySearchReplyIsReturnedWhenEngineIsClosed")
+{
+    MatchEngine engine(1, 1, 7);
+    engine.setOnline();
+    engine.setNodeUp(true);
+    engine.close();
+    LocalSearchClient client;
+    SearchRequest::Source request(new SearchRequest());
+    SearchReply::UP reply = engine.search(std::move(request), client);
+    EXPECT_TRUE(reply.get() != NULL);
+    EXPECT_EQUAL(0u, reply->hits.size());
+    EXPECT_EQUAL(7u, reply->getDistributionKey());
+}
+
+TEST("requireThatStateIsReported")
+{
+    MatchEngine engine(1, 1, 7);
+
+    Slime slime;
+    SlimeInserter inserter(slime);
+    engine.get_state(inserter, false);
+    EXPECT_EQUAL(
+            "{\n"
+            "    \"status\": {\n"
+            "        \"state\": \"OFFLINE\",\n"
+            "        \"message\": \"Search interface is offline\"\n"
+            "    }\n"
+            "}\n",
+            slime.toString());
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/.cvsignore b/searchcore/src/tests/proton/matching/.cvsignore
new file mode 100644
index 00000000000..75b0a127c8f
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/.cvsignore
@@ -0,0 +1,3 @@
+.depend
+Makefile
+matching_test
diff --git a/searchcore/src/tests/proton/matching/.gitignore b/searchcore/src/tests/proton/matching/.gitignore
new file mode 100644
index 00000000000..c9789272a35
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/.gitignore
@@ -0,0 +1,14 @@
+.depend
+Makefile
+matching_test
+query_test
+querynodes_test
+resolveviewvisitor_test
+termdataextractor_test
+searchcore_matching_stats_test_app
+searchcore_matching_test_app
+searchcore_query_test_app
+searchcore_querynodes_test_app
+searchcore_resolveviewvisitor_test_app
+searchcore_sessionmanager_test_app
+searchcore_termdataextractor_test_app
diff --git a/searchcore/src/tests/proton/matching/CMakeLists.txt b/searchcore/src/tests/proton/matching/CMakeLists.txt
new file mode 100644
index 00000000000..8007ff0344d
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/CMakeLists.txt
@@ -0,0 +1,60 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_matching_test_app
+    SOURCES
+    matching_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_fconfig
+    searchcore_matching
+    searchcore_feedoperation
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_grouping
+    searchcore_util
+)
+vespa_add_test(NAME searchcore_matching_test_app COMMAND searchcore_matching_test_app)
+vespa_add_executable(searchcore_sessionmanager_test_app
+    SOURCES
+    sessionmanager_test.cpp
+    DEPENDS
+    searchcore_matching
+    searchcore_grouping
+)
+vespa_add_test(NAME searchcore_sessionmanager_test_app COMMAND searchcore_sessionmanager_test_app)
+vespa_add_executable(searchcore_matching_stats_test_app
+    SOURCES
+    matching_stats_test.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_matching_stats_test_app COMMAND searchcore_matching_stats_test_app)
+vespa_add_executable(searchcore_query_test_app
+    SOURCES
+    query_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_query_test_app COMMAND searchcore_query_test_app)
+vespa_add_executable(searchcore_termdataextractor_test_app
+    SOURCES
+    termdataextractor_test.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_termdataextractor_test_app COMMAND searchcore_termdataextractor_test_app)
+vespa_add_executable(searchcore_resolveviewvisitor_test_app
+    SOURCES
+    resolveviewvisitor_test.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_resolveviewvisitor_test_app COMMAND searchcore_resolveviewvisitor_test_app)
+vespa_add_executable(searchcore_querynodes_test_app
+    SOURCES
+    querynodes_test.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_querynodes_test_app COMMAND searchcore_querynodes_test_app)
diff --git a/searchcore/src/tests/proton/matching/DESC b/searchcore/src/tests/proton/matching/DESC
new file mode 100644
index 00000000000..435b17f333e
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/DESC
@@ -0,0 +1 @@
+matching test. Take a look at matching.cpp for details.
diff --git a/searchcore/src/tests/proton/matching/FILES b/searchcore/src/tests/proton/matching/FILES
new file mode 100644
index 00000000000..0213f77d899
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/FILES
@@ -0,0 +1 @@
+matching.cpp
diff --git a/searchcore/src/tests/proton/matching/docid_range_scheduler/.gitignore b/searchcore/src/tests/proton/matching/docid_range_scheduler/.gitignore
new file mode 100644
index 00000000000..8de390797da
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/docid_range_scheduler/.gitignore
@@ -0,0 +1,3 @@
+/docid_range_scheduler_bench
+searchcore_docid_range_scheduler_test_app
+searchcore_docid_range_scheduler_bench_app
diff --git a/searchcore/src/tests/proton/matching/docid_range_scheduler/CMakeLists.txt b/searchcore/src/tests/proton/matching/docid_range_scheduler/CMakeLists.txt
new file mode 100644
index 00000000000..3892ac41b92
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/docid_range_scheduler/CMakeLists.txt
@@ -0,0 +1,15 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_docid_range_scheduler_test_app
+    SOURCES
+    docid_range_scheduler_test.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_docid_range_scheduler_test_app COMMAND searchcore_docid_range_scheduler_test_app)
+vespa_add_executable(searchcore_docid_range_scheduler_bench_app
+    SOURCES
+    docid_range_scheduler_bench.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_docid_range_scheduler_bench_app COMMAND searchcore_docid_range_scheduler_bench_app BENCHMARK)
diff --git a/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_bench.cpp b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_bench.cpp
new file mode 100644
index 00000000000..848743e0f23
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_bench.cpp
@@ -0,0 +1,226 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchcore/proton/matching/docid_range_scheduler.h>
+#include <vespa/vespalib/util/benchmark_timer.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using namespace proton::matching;
+using namespace vespalib;
+
+//-----------------------------------------------------------------------------
+
+size_t do_work(size_t cost) __attribute__((noinline));
+size_t do_work(size_t cost) {
+    size_t result = 0;
+    size_t loop_cnt = 42;
+    for (size_t n = 0; n < cost; ++n) {
+        result += (cost * n);
+        for (size_t i = 0; i < loop_cnt; ++i) {
+            result += (cost * n * i);
+            for (size_t j = 0; j < loop_cnt; ++j) {
+                result += (cost * n * i * j);
+                for (size_t k = 0; k < loop_cnt; ++k) {
+                    result += (cost * n * i * j * k);
+                }
+            }
+        }
+    }
+    return result;
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("measure do_work overhead for different cost inputs") {
+    for (size_t cost: {0, 1, 10, 100, 1000}) {
+        BenchmarkTimer timer(1.0);
+        while (timer.has_budget()) {
+            timer.before();
+            (void) do_work(cost);
+            timer.after();
+        }
+        double min_time_s = timer.min_time();
+        fprintf(stderr, "const %zu: %g us\n", cost, min_time_s * 1000.0 * 1000.0);
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+struct Work {
+    typedef std::unique_ptr<Work> UP;
+    virtual vespalib::string desc() const = 0;
+    virtual void perform(uint32_t docid) const = 0;
+    virtual ~Work() {}
+};
+
+struct UniformWork : public Work {
+    size_t cost;
+    UniformWork(size_t cost_in) : cost(cost_in) {}
+    vespalib::string desc() const override { return make_string("uniform(%zu)", cost); }
+    void perform(uint32_t) const override { (void) do_work(cost); }
+};
+
+struct TriangleWork : public Work {
+    size_t div;
+    TriangleWork(size_t div_in) : div(div_in) {}
+    vespalib::string desc() const override { return make_string("triangle(docid/%zu)", div); }
+    void perform(uint32_t docid) const override { (void) do_work(docid/div); }
+};
+
+struct SpikeWork : public Work {
+    uint32_t begin;
+    uint32_t end;
+    size_t cost;
+    SpikeWork(uint32_t begin_in, uint32_t end_in, size_t cost_in)
+        : begin(begin_in), end(end_in), cost(cost_in) {}
+    vespalib::string desc() const override { return make_string("spike(%u,%u,%zu)", begin, end, cost); }
+    void perform(uint32_t docid) const override {
+        if ((docid >= begin) && (docid < end)) {
+            (void) do_work(cost);
+        }
+    }
+};
+
+struct WorkList {
+    std::vector<Work::UP> work_list;
+    WorkList() : work_list() {
+        work_list.push_back(std::make_unique<UniformWork>(10));
+        work_list.push_back(std::make_unique<TriangleWork>(4878));
+        work_list.push_back(std::make_unique<SpikeWork>(1, 10001, 100));
+        work_list.push_back(std::make_unique<SpikeWork>(1, 1001, 1000));
+        work_list.push_back(std::make_unique<SpikeWork>(1, 101, 10000));
+        work_list.push_back(std::make_unique<SpikeWork>(1, 11, 100000));
+        work_list.push_back(std::make_unique<SpikeWork>(90001, 100001, 100));
+        work_list.push_back(std::make_unique<SpikeWork>(99001, 100001, 1000));
+        work_list.push_back(std::make_unique<SpikeWork>(99901, 100001, 10000));
+        work_list.push_back(std::make_unique<SpikeWork>(99991, 100001, 100000));
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+struct SchedulerFactory {
+    typedef std::unique_ptr<SchedulerFactory> UP;
+    virtual vespalib::string desc() const = 0;    
+    virtual DocidRangeScheduler::UP create(uint32_t docid_limit) const = 0;
+    virtual ~SchedulerFactory() {}
+};
+
+struct PartitionSchedulerFactory : public SchedulerFactory {
+    size_t num_threads;
+    PartitionSchedulerFactory(size_t num_threads_in) : num_threads(num_threads_in) {}
+    vespalib::string desc() const override { return make_string("partition(threads:%zu)", num_threads); }
+    DocidRangeScheduler::UP create(uint32_t docid_limit) const override {
+        return std::make_unique<PartitionDocidRangeScheduler>(num_threads, docid_limit);
+    }
+};
+
+struct TaskSchedulerFactory : public SchedulerFactory {
+    size_t num_threads;
+    size_t num_tasks;
+    TaskSchedulerFactory(size_t num_threads_in, size_t num_tasks_in)
+        : num_threads(num_threads_in), num_tasks(num_tasks_in) {}
+    vespalib::string desc() const override { return make_string("task(threads:%zu,num_tasks:%zu)", num_threads, num_tasks); }
+    DocidRangeScheduler::UP create(uint32_t docid_limit) const override {
+        return std::make_unique<TaskDocidRangeScheduler>(num_threads, num_tasks, docid_limit);
+    }
+};
+
+struct AdaptiveSchedulerFactory : public SchedulerFactory {
+    size_t num_threads;
+    size_t min_task;
+    AdaptiveSchedulerFactory(size_t num_threads_in, size_t min_task_in)
+        : num_threads(num_threads_in), min_task(min_task_in) {}
+    vespalib::string desc() const override { return make_string("adaptive(threads:%zu,min_task:%zu)", num_threads, min_task); }
+    DocidRangeScheduler::UP create(uint32_t docid_limit) const override {
+        return std::make_unique<AdaptiveDocidRangeScheduler>(num_threads, min_task, docid_limit);
+    }
+};
+
+struct SchedulerList {
+    std::vector<SchedulerFactory::UP> factory_list;
+    SchedulerList(size_t num_threads) : factory_list() {
+        factory_list.push_back(std::make_unique<PartitionSchedulerFactory>(num_threads));
+        factory_list.push_back(std::make_unique<TaskSchedulerFactory>(num_threads, num_threads));
+        factory_list.push_back(std::make_unique<TaskSchedulerFactory>(num_threads, 64));
+        factory_list.push_back(std::make_unique<TaskSchedulerFactory>(num_threads, 256));
+        factory_list.push_back(std::make_unique<TaskSchedulerFactory>(num_threads, 1024));
+        factory_list.push_back(std::make_unique<TaskSchedulerFactory>(num_threads, 4096));
+        factory_list.push_back(std::make_unique<AdaptiveSchedulerFactory>(num_threads, 1000));
+        factory_list.push_back(std::make_unique<AdaptiveSchedulerFactory>(num_threads, 100));
+        factory_list.push_back(std::make_unique<AdaptiveSchedulerFactory>(num_threads, 10));
+        factory_list.push_back(std::make_unique<AdaptiveSchedulerFactory>(num_threads, 1));
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+void worker(DocidRangeScheduler &scheduler, const Work &work, size_t thread_id) {
+    IdleObserver observer = scheduler.make_idle_observer();
+    if (observer.is_always_zero()) {
+        for (DocidRange range = scheduler.first_range(thread_id);
+             !range.empty();
+             range = scheduler.next_range(thread_id))
+        {
+            do_work(10); // represents init-range cost
+            for (uint32_t docid = range.begin; docid < range.end; ++docid) {
+                work.perform(docid);
+            }
+        }
+    } else {
+        for (DocidRange range = scheduler.first_range(thread_id);
+             !range.empty();
+             range = scheduler.next_range(thread_id))
+        {
+            do_work(10); // represents init-range cost
+            for (uint32_t docid = range.begin; docid < range.end; ++docid) {
+                work.perform(docid);
+                if (observer.get() > 0) {
+                    range = scheduler.share_range(thread_id, DocidRange(docid, range.end));
+                }
+            }
+        }
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_MT_FFF("benchmark different combinations of schedulers and work loads", 8,
+            DocidRangeScheduler::UP(), SchedulerList(num_threads), WorkList())
+{
+    if (thread_id == 0) {
+        fprintf(stderr, "Benchmarking with %zu threads:\n", num_threads);
+    }
+    for (size_t scheduler = 0; scheduler < f2.factory_list.size(); ++scheduler) {
+        for (size_t work = 0; work < f3.work_list.size(); ++work) {
+            if (thread_id == 0) {
+                fprintf(stderr, "  scheduler: %s, work load: %s ",
+                        f2.factory_list[scheduler]->desc().c_str(),
+                        f3.work_list[work]->desc().c_str());
+            }
+            BenchmarkTimer timer(1.0);
+            for (size_t i = 0; i < 5; ++i) {
+                TEST_BARRIER();
+                if (thread_id == 0) {
+                    f1 = f2.factory_list[scheduler]->create(100001);
+                }
+                TEST_BARRIER();
+                timer.before();
+                worker(*f1, *f3.work_list[work], thread_id);
+                TEST_BARRIER();
+                timer.after();
+                if (thread_id == 0) {
+                    fprintf(stderr, ".");
+                }
+            }
+            if (thread_id == 0) {
+                fprintf(stderr, " real time: %g ms\n", timer.min_time() * 1000.0);
+            }
+        }
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp
new file mode 100644
index 00000000000..6716e945a0d
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp
@@ -0,0 +1,286 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchcore/proton/matching/docid_range_scheduler.h>
+#include <chrono>
+#include <thread>
+
+using namespace proton::matching;
+
+void verify_range(DocidRange a, DocidRange b) {
+    EXPECT_EQUAL(a.begin, b.begin);
+    EXPECT_EQUAL(a.end, b.end);
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that default docid range constructor creates and empty range") {
+    EXPECT_TRUE(DocidRange().empty());
+    EXPECT_EQUAL(DocidRange().size(), 0u);
+}
+
+TEST("require that docid range ensures end is not less than begin") {
+    EXPECT_EQUAL(DocidRange(10, 20).size(), 10u);
+    EXPECT_TRUE(!DocidRange(10, 20).empty());
+    EXPECT_EQUAL(DocidRange(10, 20).begin, 10u);
+    EXPECT_EQUAL(DocidRange(10, 20).end, 20u);
+    EXPECT_EQUAL(DocidRange(20, 10).size(), 0u);
+    EXPECT_TRUE(DocidRange(20, 10).empty());
+    EXPECT_EQUAL(DocidRange(20, 10).begin, 20u);
+    EXPECT_EQUAL(DocidRange(20, 10).end, 20u);
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that default constructed IdleObserver is always zero") {
+    IdleObserver observer;
+    EXPECT_TRUE(observer.is_always_zero());
+    EXPECT_EQUAL(0u, observer.get());
+}
+
+TEST("require that IdleObserver can observe an atomic size_t value") {
+    std::atomic<size_t> idle(0);
+    IdleObserver observer(idle);
+    EXPECT_TRUE(!observer.is_always_zero());
+    EXPECT_EQUAL(0u, observer.get());
+    idle = 10;
+    EXPECT_EQUAL(10u, observer.get());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that the docid range splitter can split a docid range") {
+    DocidRangeSplitter splitter(DocidRange(1, 16), 4);
+    TEST_DO(verify_range(splitter.get(0), DocidRange(1, 5)));
+    TEST_DO(verify_range(splitter.get(1), DocidRange(5, 9)));
+    TEST_DO(verify_range(splitter.get(2), DocidRange(9, 13)));
+    TEST_DO(verify_range(splitter.get(3), DocidRange(13, 16)));
+}
+
+TEST("require that the docid range splitter can split an empty range") {
+    DocidRangeSplitter splitter(DocidRange(5, 5), 2);
+    TEST_DO(verify_range(splitter.get(0), DocidRange(5, 5)));
+    TEST_DO(verify_range(splitter.get(1), DocidRange(5, 5)));
+}
+
+TEST("require that the docid range splitter can split a range into more parts than values") {
+    DocidRangeSplitter splitter(DocidRange(1, 4), 4);
+    TEST_DO(verify_range(splitter.get(0), DocidRange(1, 2)));
+    TEST_DO(verify_range(splitter.get(1), DocidRange(2, 3)));
+    TEST_DO(verify_range(splitter.get(2), DocidRange(3, 4)));
+    TEST_DO(verify_range(splitter.get(3), DocidRange(4, 4)));
+}
+
+TEST("require that the docid range splitter gives empty ranges if accessed with too high index") {
+    DocidRangeSplitter splitter(DocidRange(1, 4), 3);
+    TEST_DO(verify_range(splitter.get(0), DocidRange(1, 2)));
+    TEST_DO(verify_range(splitter.get(1), DocidRange(2, 3)));
+    TEST_DO(verify_range(splitter.get(2), DocidRange(3, 4)));
+    TEST_DO(verify_range(splitter.get(3), DocidRange(4, 4)));
+    TEST_DO(verify_range(splitter.get(100), DocidRange(4, 4)));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that the partition scheduler acts as expected") {
+    PartitionDocidRangeScheduler scheduler(4, 16);
+    TEST_DO(verify_range(scheduler.total_span(0), DocidRange(1, 5)));
+    TEST_DO(verify_range(scheduler.total_span(1), DocidRange(5, 9)));
+    TEST_DO(verify_range(scheduler.total_span(2), DocidRange(9, 13)));
+    TEST_DO(verify_range(scheduler.total_span(3), DocidRange(13, 16)));
+    EXPECT_EQUAL(scheduler.total_size(0), 4u);
+    EXPECT_EQUAL(scheduler.total_size(1), 4u);
+    EXPECT_EQUAL(scheduler.total_size(2), 4u);
+    EXPECT_EQUAL(scheduler.total_size(3), 3u);
+    EXPECT_EQUAL(scheduler.unassigned_size(), 0u);
+    TEST_DO(verify_range(scheduler.first_range(0), DocidRange(1, 5)));
+    TEST_DO(verify_range(scheduler.first_range(1), DocidRange(5, 9)));
+    TEST_DO(verify_range(scheduler.first_range(2), DocidRange(9, 13)));
+    TEST_DO(verify_range(scheduler.first_range(3), DocidRange(13, 16)));
+    TEST_DO(verify_range(scheduler.next_range(0), DocidRange()));
+    TEST_DO(verify_range(scheduler.next_range(1), DocidRange()));
+    TEST_DO(verify_range(scheduler.next_range(2), DocidRange()));
+    TEST_DO(verify_range(scheduler.next_range(3), DocidRange()));
+}
+
+TEST("require that the partition scheduler protects against documents underflow") {
+    PartitionDocidRangeScheduler scheduler(2, 0);
+    TEST_DO(verify_range(scheduler.total_span(0), DocidRange(1,1)));
+    TEST_DO(verify_range(scheduler.total_span(1), DocidRange(1,1)));
+    EXPECT_EQUAL(scheduler.total_size(0), 0u);
+    EXPECT_EQUAL(scheduler.total_size(1), 0u);
+    EXPECT_EQUAL(scheduler.unassigned_size(), 0u);
+    TEST_DO(verify_range(scheduler.first_range(0), DocidRange(1,1)));
+    TEST_DO(verify_range(scheduler.first_range(1), DocidRange(1,1)));
+    TEST_DO(verify_range(scheduler.next_range(0), DocidRange()));
+    TEST_DO(verify_range(scheduler.next_range(1), DocidRange()));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that the task scheduler acts as expected") {
+    TaskDocidRangeScheduler scheduler(2, 5, 20);
+    EXPECT_EQUAL(scheduler.unassigned_size(), 19u);
+    TEST_DO(verify_range(scheduler.total_span(0), DocidRange(1, 20)));
+    TEST_DO(verify_range(scheduler.total_span(1), DocidRange(1, 20)));
+    EXPECT_EQUAL(scheduler.total_size(0), 0u);
+    EXPECT_EQUAL(scheduler.total_size(1), 0u);
+    TEST_DO(verify_range(scheduler.first_range(1), DocidRange(1, 5)));
+    TEST_DO(verify_range(scheduler.first_range(0), DocidRange(5, 9)));
+    TEST_DO(verify_range(scheduler.next_range(0), DocidRange(9, 13)));
+    EXPECT_EQUAL(scheduler.unassigned_size(), 7u);
+    TEST_DO(verify_range(scheduler.next_range(1), DocidRange(13, 17)));
+    TEST_DO(verify_range(scheduler.next_range(0), DocidRange(17, 20)));
+    TEST_DO(verify_range(scheduler.next_range(0), DocidRange(20, 20)));
+    TEST_DO(verify_range(scheduler.next_range(1), DocidRange(20, 20)));
+    EXPECT_EQUAL(scheduler.total_size(0), 11u);
+    EXPECT_EQUAL(scheduler.total_size(1), 8u);
+    EXPECT_EQUAL(scheduler.unassigned_size(), 0u);
+}
+
+TEST("require that the task scheduler protects against documents underflow") {
+    TaskDocidRangeScheduler scheduler(2, 4, 0);
+    TEST_DO(verify_range(scheduler.total_span(0), DocidRange(1,1)));
+    TEST_DO(verify_range(scheduler.total_span(1), DocidRange(1,1)));
+    EXPECT_EQUAL(scheduler.total_size(0), 0u);
+    EXPECT_EQUAL(scheduler.total_size(1), 0u);
+    EXPECT_EQUAL(scheduler.unassigned_size(), 0u);
+    TEST_DO(verify_range(scheduler.first_range(0), DocidRange(1,1)));
+    TEST_DO(verify_range(scheduler.first_range(1), DocidRange(1,1)));
+    TEST_DO(verify_range(scheduler.next_range(0), DocidRange(1,1)));
+    TEST_DO(verify_range(scheduler.next_range(1), DocidRange(1,1)));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that the adaptive scheduler starts by dividing the docid space equally") {
+    AdaptiveDocidRangeScheduler scheduler(4, 1, 16);
+    EXPECT_EQUAL(scheduler.total_size(0), 4u);
+    EXPECT_EQUAL(scheduler.total_size(1), 4u);
+    EXPECT_EQUAL(scheduler.total_size(2), 4u);
+    EXPECT_EQUAL(scheduler.total_size(3), 3u);
+    EXPECT_EQUAL(scheduler.unassigned_size(), 0u);    
+    TEST_DO(verify_range(scheduler.first_range(0), DocidRange(1, 5)));
+    TEST_DO(verify_range(scheduler.first_range(1), DocidRange(5, 9)));
+    TEST_DO(verify_range(scheduler.first_range(2), DocidRange(9, 13)));
+    TEST_DO(verify_range(scheduler.first_range(3), DocidRange(13, 16)));
+}
+
+TEST("require that the adaptive scheduler reports the full span to all threads") {
+    AdaptiveDocidRangeScheduler scheduler(3, 1, 16);
+    TEST_DO(verify_range(scheduler.total_span(0), DocidRange(1,16)));
+    TEST_DO(verify_range(scheduler.total_span(1), DocidRange(1,16)));
+    TEST_DO(verify_range(scheduler.total_span(2), DocidRange(1,16)));
+}
+
+TEST_MT_F("require that the adaptive scheduler terminates when all workers request more work",
+          4, AdaptiveDocidRangeScheduler(num_threads, 1, 16))
+{
+    (void) f1.first_range(thread_id);
+    DocidRange range = f1.next_range(thread_id);
+    EXPECT_TRUE(range.empty());
+}
+
+void wait_idle(const DocidRangeScheduler &scheduler, size_t wanted) {
+    IdleObserver observer = scheduler.make_idle_observer();
+    while (observer.get() != wanted) {
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+}
+
+TEST_MT_F("require that the adaptive scheduler enables threads to share work",
+          3, AdaptiveDocidRangeScheduler(num_threads, 1, 28))
+{
+    DocidRange range = f1.first_range(thread_id);
+    if (thread_id == 0) {
+        TEST_DO(verify_range(range, DocidRange(1,10)));
+    } else if (thread_id == 1) {
+        TEST_DO(verify_range(range, DocidRange(10,19)));
+    } else {
+        TEST_DO(verify_range(range, DocidRange(19,28)));
+    }
+    EXPECT_EQUAL(f1.total_size(thread_id), 9u);
+    TEST_DO(verify_range(f1.share_range(thread_id, range), range));
+    TEST_BARRIER();
+    if (thread_id == 0) {
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange(25,28)));
+    } else if (thread_id == 1) {
+        wait_idle(f1, 1);
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange(22,25)));
+    } else {
+        wait_idle(f1, 2);
+        verify_range(f1.share_range(thread_id, range), DocidRange(19,22));
+    }
+    TEST_DO(verify_range(f1.next_range(thread_id), DocidRange()));
+    EXPECT_EQUAL(f1.total_size(0), 12u);
+    EXPECT_EQUAL(f1.total_size(1), 12u);
+    EXPECT_EQUAL(f1.total_size(2), 3u);
+}
+
+TEST("require that the adaptive scheduler protects against documents underflow") {
+    AdaptiveDocidRangeScheduler scheduler(2, 1, 0);
+    TEST_DO(verify_range(scheduler.first_range(0), DocidRange(1,1)));
+    TEST_DO(verify_range(scheduler.first_range(1), DocidRange(1,1)));
+    EXPECT_EQUAL(scheduler.total_size(0), 0u);
+    EXPECT_EQUAL(scheduler.total_size(1), 0u);
+    EXPECT_EQUAL(scheduler.unassigned_size(), 0u);
+}
+
+TEST_MT_F("require that the adaptive scheduler respects the minimal task size",
+          2, AdaptiveDocidRangeScheduler(num_threads, 3, 21))
+{
+    EXPECT_EQUAL(f1.first_range(thread_id).size(), 10u);
+    if (thread_id == 0) {
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange(18,21)));
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange()));
+    } else {
+        wait_idle(f1, 1);
+        // a range with size 5 will not be split
+        TEST_DO(verify_range(f1.share_range(thread_id, DocidRange(16,21)), DocidRange(16,21)));
+        // a range with size 6 will be split
+        TEST_DO(verify_range(f1.share_range(thread_id, DocidRange(15,21)), DocidRange(15,18)));
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange()));
+    }
+}
+
+TEST_MT_F("require that the adaptive scheduler will never split a task with size 1",
+          2, AdaptiveDocidRangeScheduler(num_threads, 0, 21))
+{
+    EXPECT_EQUAL(f1.first_range(thread_id).size(), 10u);
+    if (thread_id == 0) {
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange()));
+    } else {
+        IdleObserver observer = f1.make_idle_observer();
+        while (observer.get() == 0) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+        DocidRange small_range = DocidRange(20,21);
+        verify_range(f1.share_range(thread_id, small_range), small_range);
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange()));
+    }
+}
+
+TEST_MT_F("require that the adaptive scheduler can leave idle workers alone due to minimal task size",
+          3, AdaptiveDocidRangeScheduler(num_threads, 3, 28))
+{
+    EXPECT_EQUAL(f1.first_range(thread_id).size(), 9u);
+    if (thread_id == 0) {
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange()));
+    } else if (thread_id == 1) {
+        wait_idle(f1, 1);
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange(24,28)));
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange()));
+    } else {
+        wait_idle(f1, 2);
+        verify_range(f1.share_range(thread_id, DocidRange(20,28)), DocidRange(20,24));
+        TEST_DO(verify_range(f1.next_range(thread_id), DocidRange()));
+    }
+    EXPECT_EQUAL(f1.total_size(0), 9u);
+    EXPECT_EQUAL(f1.total_size(1), 13u);
+    EXPECT_EQUAL(f1.total_size(2), 5u);
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/.gitignore b/searchcore/src/tests/proton/matching/match_loop_communicator/.gitignore
new file mode 100644
index 00000000000..c3797981bab
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/match_loop_communicator/.gitignore
@@ -0,0 +1 @@
+searchcore_match_loop_communicator_test_app
diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt b/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt
new file mode 100644
index 00000000000..513e002f064
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_match_loop_communicator_test_app
+    SOURCES
+    match_loop_communicator_test.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_match_loop_communicator_test_app COMMAND searchcore_match_loop_communicator_test_app)
diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/FILES b/searchcore/src/tests/proton/matching/match_loop_communicator/FILES
new file mode 100644
index 00000000000..d2f1096aaa3
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/match_loop_communicator/FILES
@@ -0,0 +1 @@
+match_loop_communicator_test.cpp
diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp
new file mode 100644
index 00000000000..92139a1c027
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchcore/proton/matching/match_loop_communicator.h>
+#include <vespa/vespalib/util/box.h>
+
+using namespace proton::matching;
+
+using vespalib::Box;
+using vespalib::make_box;
+
+typedef MatchLoopCommunicator::Range Range;
+typedef MatchLoopCommunicator::RangePair RangePair;
+typedef MatchLoopCommunicator::feature_t feature_t;
+typedef MatchLoopCommunicator::Matches Matches;
+
+std::vector<feature_t> makeScores(size_t id) {
+    switch (id) {
+    case 0: return make_box<feature_t>(5.4, 4.4, 3.4, 2.4, 1.4);
+    case 1: return make_box<feature_t>(5.3, 4.3, 3.3, 2.3, 1.3);
+    case 2: return make_box<feature_t>(5.2, 4.2, 3.2, 2.2, 1.2);
+    case 3: return make_box<feature_t>(5.1, 4.1, 3.1, 2.1, 1.1);
+    case 4: return make_box<feature_t>(5.0, 4.0, 3.0, 2.0, 1.0);
+    }
+    return Box<feature_t>();
+}
+
+RangePair makeRanges(size_t id) {
+    switch (id) {
+    case 0: return std::make_pair(Range(5, 5), Range(7, 7));
+    case 1: return std::make_pair(Range(2, 2), Range(8, 8));
+    case 2: return std::make_pair(Range(3, 3), Range(6, 6));
+    case 3: return std::make_pair(Range(1, 1), Range(5, 5));
+    case 4: return std::make_pair(Range(4, 4), Range(9, 9));
+    }
+    return std::make_pair(Range(-50, -60), Range(60, 50));
+}
+
+TEST_F("require that selectBest gives appropriate results for single thread", MatchLoopCommunicator(num_threads, 3)) {
+    EXPECT_EQUAL(2u, f1.selectBest(make_box<feature_t>(5, 4)));
+    EXPECT_EQUAL(3u, f1.selectBest(make_box<feature_t>(5, 4, 3)));
+    EXPECT_EQUAL(3u, f1.selectBest(make_box<feature_t>(5, 4, 3, 2)));
+}
+
+TEST_MT_F("require that selectBest works with no hits", 10, MatchLoopCommunicator(num_threads, 10)) {
+    EXPECT_EQUAL(0u, f1.selectBest(Box<feature_t>()));
+}
+
+TEST_MT_F("require that selectBest works with too many hits from all threads", 5, MatchLoopCommunicator(num_threads, 13)) {
+    if (thread_id < 3) {
+        EXPECT_EQUAL(3u, f1.selectBest(makeScores(thread_id)));
+    } else {
+        EXPECT_EQUAL(2u, f1.selectBest(makeScores(thread_id)));
+    }
+}
+
+TEST_MT_F("require that selectBest works with some exhausted threads", 5, MatchLoopCommunicator(num_threads, 22)) {
+    if (thread_id < 2) {
+        EXPECT_EQUAL(5u, f1.selectBest(makeScores(thread_id)));
+    } else {
+        EXPECT_EQUAL(4u, f1.selectBest(makeScores(thread_id)));
+    }
+}
+
+TEST_MT_F("require that selectBest can select all hits from all threads", 5, MatchLoopCommunicator(num_threads, 100)) {
+    EXPECT_EQUAL(5u, f1.selectBest(makeScores(thread_id)));
+}
+
+TEST_MT_F("require that selectBest works with some empty threads", 10, MatchLoopCommunicator(num_threads, 7)) {
+    if (thread_id < 2) {
+        EXPECT_EQUAL(2u, f1.selectBest(makeScores(thread_id)));
+    } else if (thread_id < 5) {
+        EXPECT_EQUAL(1u, f1.selectBest(makeScores(thread_id)));
+    } else {
+        EXPECT_EQUAL(0u, f1.selectBest(makeScores(thread_id)));
+    }
+}
+
+TEST_F("require that rangeCover is identity function for single thread", MatchLoopCommunicator(num_threads, 5)) {
+    RangePair res = f1.rangeCover(std::make_pair(Range(2, 4), Range(3, 5)));
+    EXPECT_EQUAL(2, res.first.low);
+    EXPECT_EQUAL(4, res.first.high);
+    EXPECT_EQUAL(3, res.second.low);
+    EXPECT_EQUAL(5, res.second.high);
+}
+
+TEST_MT_F("require that rangeCover can mix ranges from multiple threads", 5, MatchLoopCommunicator(num_threads, 5)) {
+    RangePair res = f1.rangeCover(makeRanges(thread_id));
+    EXPECT_EQUAL(1, res.first.low);
+    EXPECT_EQUAL(5, res.first.high);
+    EXPECT_EQUAL(5, res.second.low);
+    EXPECT_EQUAL(9, res.second.high);
+}
+
+TEST_MT_F("require that invalid ranges are ignored", 10, MatchLoopCommunicator(num_threads, 5)) {
+    RangePair res = f1.rangeCover(makeRanges(thread_id));
+    EXPECT_EQUAL(1, res.first.low);
+    EXPECT_EQUAL(5, res.first.high);
+    EXPECT_EQUAL(5, res.second.low);
+    EXPECT_EQUAL(9, res.second.high);
+}
+
+TEST_MT_F("require that only invalid ranges produce default invalid range", 3, MatchLoopCommunicator(num_threads, 5)) {
+    RangePair res = f1.rangeCover(makeRanges(10));
+    Range expect;
+    EXPECT_FALSE(expect.isValid());
+    EXPECT_EQUAL(expect.low, res.first.low);
+    EXPECT_EQUAL(expect.high, res.first.high);
+    EXPECT_EQUAL(expect.low, res.second.low);
+    EXPECT_EQUAL(expect.high, res.second.high);
+}
+
+TEST_MT_F("require that count_matches will count hits and docs across threads", 4, MatchLoopCommunicator(num_threads, 5)) {
+    double freq = (0.0/10.0 + 1.0/11.0 + 2.0/12.0 + 3.0/13.0) / 4.0;
+    EXPECT_APPROX(freq, f1.estimate_match_frequency(Matches(thread_id, thread_id + 10)), 0.00001);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/match_phase_limiter/.gitignore b/searchcore/src/tests/proton/matching/match_phase_limiter/.gitignore
new file mode 100644
index 00000000000..69806654ee0
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/match_phase_limiter/.gitignore
@@ -0,0 +1 @@
+searchcore_match_phase_limiter_test_app
diff --git a/searchcore/src/tests/proton/matching/match_phase_limiter/CMakeLists.txt b/searchcore/src/tests/proton/matching/match_phase_limiter/CMakeLists.txt
new file mode 100644
index 00000000000..78c16d1435d
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/match_phase_limiter/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_match_phase_limiter_test_app
+    SOURCES
+    match_phase_limiter_test.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_match_phase_limiter_test_app COMMAND searchcore_match_phase_limiter_test_app)
diff --git a/searchcore/src/tests/proton/matching/match_phase_limiter/FILES b/searchcore/src/tests/proton/matching/match_phase_limiter/FILES
new file mode 100644
index 00000000000..776925a0d69
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/match_phase_limiter/FILES
@@ -0,0 +1 @@
+match_phase_limiter_test.cpp
diff --git a/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp b/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp
new file mode 100644
index 00000000000..35757cb43c7
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp
@@ -0,0 +1,361 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchcore/proton/matching/match_phase_limiter.h>
+#include <vespa/searchlib/queryeval/termasstring.h>
+#include <vespa/searchlib/queryeval/andsearchstrict.h>
+#include <vespa/searchlib/queryeval/fake_requestcontext.h>
+
+using namespace proton::matching;
+using search::queryeval::SearchIterator;
+using search::queryeval::Searchable;
+using search::queryeval::Blueprint;
+using search::queryeval::SimpleLeafBlueprint;
+using search::queryeval::FieldSpec;
+using search::queryeval::FieldSpecBaseList;
+using search::queryeval::AndSearchStrict;
+using search::queryeval::termAsString;
+using search::queryeval::FakeRequestContext;
+using search::fef::TermFieldMatchDataArray;
+
+//-----------------------------------------------------------------------------
+
+SearchIterator::UP prepare(SearchIterator * search)
+{
+    search->initFullRange();
+    return SearchIterator::UP(search);
+}
+
+struct MockSearch : SearchIterator {
+    FieldSpec spec;
+    vespalib::string term;
+    vespalib::Trinary _strict;
+    TermFieldMatchDataArray tfmda;
+    bool postings_fetched;
+    uint32_t last_seek = beginId();
+    uint32_t last_unpack = beginId();
+    MockSearch(const vespalib::string &term_in)
+        : spec(0, 0, 0), term(term_in), _strict(vespalib::Trinary::True), tfmda(), postings_fetched(false) {}
+    MockSearch(const FieldSpec &spec_in, const vespalib::string &term_in, bool strict_in,
+               const TermFieldMatchDataArray &tfmda_in, bool postings_fetched_in)
+        : spec(spec_in), term(term_in),
+          _strict(strict_in ? vespalib::Trinary::True : vespalib::Trinary::False),
+          tfmda(tfmda_in),
+          postings_fetched(postings_fetched_in) {}
+    void doSeek(uint32_t docid) override { last_seek = docid; setDocId(docid); }
+    void doUnpack(uint32_t docid) override { last_unpack = docid; }
+    vespalib::Trinary is_strict() const override { return _strict; } 
+    bool strict() const { return (is_strict() == vespalib::Trinary::True); }
+};
+
+struct MockBlueprint : SimpleLeafBlueprint {
+    FieldSpec spec;
+    vespalib::string term;
+    bool postings_fetched = false;
+    bool postings_strict = false;
+    MockBlueprint(const FieldSpec &spec_in, const vespalib::string &term_in)
+        : SimpleLeafBlueprint(FieldSpecBaseList().add(spec_in)), spec(spec_in), term(term_in)
+    {
+        setEstimate(HitEstimate(756, false));
+    }    
+    virtual SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda,
+                                                bool strict) const override
+    {
+        if (postings_fetched) {
+            EXPECT_EQUAL(postings_strict, strict);
+        }
+        return SearchIterator::UP(new MockSearch(spec, term, strict, tfmda,
+                                                 postings_fetched));
+    }
+    virtual void fetchPostings(bool strict) override {
+        postings_strict = strict;
+        postings_fetched = true;
+    }
+};
+
+struct MockSearchable : Searchable {
+    size_t create_cnt = 0;
+    virtual Blueprint::UP createBlueprint(const search::queryeval::IRequestContext & requestContext,
+                                          const FieldSpec &field,
+                                          const search::query::Node &term) override
+    {
+        (void) requestContext;
+        ++create_cnt;
+        return Blueprint::UP(new MockBlueprint(field, termAsString(term)));
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+TEST("require that match phase limit calculator gives expert values") {
+    MatchPhaseLimitCalculator calc(5000, 1, 0.2);
+    EXPECT_EQUAL(1000u, calc.sample_hits_per_thread(1));
+    EXPECT_EQUAL(100u, calc.sample_hits_per_thread(10));
+    EXPECT_EQUAL(10000u, calc.wanted_num_docs(0.5));
+    EXPECT_EQUAL(50000u, calc.wanted_num_docs(0.1));
+}
+
+TEST("require that match phase limit calculator can estimate hits") {
+    MatchPhaseLimitCalculator calc(0, 1, 0.2); // max hits not used
+    EXPECT_EQUAL(0u, calc.estimated_hits(0.0, 0));
+    EXPECT_EQUAL(0u, calc.estimated_hits(0.0, 1));
+    EXPECT_EQUAL(0u, calc.estimated_hits(0.0, 1000));
+    EXPECT_EQUAL(1u, calc.estimated_hits(1.0, 1));
+    EXPECT_EQUAL(10u, calc.estimated_hits(1.0, 10));
+    EXPECT_EQUAL(5u, calc.estimated_hits(0.5, 10));
+    EXPECT_EQUAL(500u, calc.estimated_hits(0.5, 1000));
+}
+
+TEST("require that match phase limit calculator has lower bound on global sample hits") {
+    MatchPhaseLimitCalculator calc(100, 1, 0.2);
+    EXPECT_EQUAL(128u, calc.sample_hits_per_thread(1));
+    EXPECT_EQUAL(4u, calc.sample_hits_per_thread(32));
+}
+
+TEST("require that match phase limit calculator has lower bound on thread sample hits") {
+    MatchPhaseLimitCalculator calc(5000, 1, 0.2);
+    EXPECT_EQUAL(1u, calc.sample_hits_per_thread(10000));
+}
+
+TEST("require that match phase limit calculator has lower bound on wanted hits") {
+    MatchPhaseLimitCalculator calc(100, 1, 0.2);
+    EXPECT_EQUAL(128u, calc.wanted_num_docs(1.0));
+}
+
+TEST("require that match phase limit calculator has upper bound on wanted hits") {
+    MatchPhaseLimitCalculator calc(100000000, 1, 0.2);
+    EXPECT_EQUAL(0x7fffFFFFu, calc.wanted_num_docs(0.0000001));
+}
+
+TEST("require that match phase limit calculator gives sane values with no hits") {
+    MatchPhaseLimitCalculator calc(100, 1, 0.2);
+    EXPECT_EQUAL(128u, calc.wanted_num_docs(1.0));
+    EXPECT_EQUAL(0x7fffFFFFu, calc.wanted_num_docs(0.000000001));
+    EXPECT_EQUAL(0x7fffFFFFu, calc.wanted_num_docs(0.000000001));
+}
+
+TEST("verify numbers used in matching test") {
+    MatchPhaseLimitCalculator calc(150, 1, 0.2);
+    EXPECT_EQUAL(1u, calc.sample_hits_per_thread(75));
+    EXPECT_EQUAL(176u, calc.wanted_num_docs(74.0 / 87.0));
+}
+
+TEST("require that max group size is calculated correctly") {
+    for (size_t min_groups: std::vector<size_t>({0, 1, 2, 3, 4, 10, 500})) {
+        for (size_t wanted_hits: std::vector<size_t>({0, 3, 321, 921})) {
+            MatchPhaseLimitCalculator calc(100, min_groups, 0.2);
+            if (min_groups == 0) {
+                EXPECT_EQUAL(wanted_hits, calc.max_group_size(wanted_hits));
+            } else {
+                EXPECT_EQUAL((wanted_hits / min_groups), calc.max_group_size(wanted_hits));
+            }
+        }
+    }
+}
+
+TEST("require that the attribute limiter works correctly") {
+    FakeRequestContext requestContext;
+    for (int i = 0; i <= 7; ++i) {
+        bool descending = (i & 1) != 0;
+        bool strict     = (i & 2) != 0;
+        bool diverse    = (i & 4) != 0;
+        MockSearchable searchable;
+        AttributeLimiter limiter(searchable, requestContext, "limiter_attribute", descending, "category", 10.0, AttributeLimiter::LOOSE);
+        EXPECT_EQUAL(0u, searchable.create_cnt);
+        EXPECT_FALSE(limiter.was_used());
+        SearchIterator::UP s1 = limiter.create_search(42, diverse ? 3 : 42, strict);
+        EXPECT_TRUE(limiter.was_used());
+        EXPECT_EQUAL(1u, searchable.create_cnt);
+        SearchIterator::UP s2 = limiter.create_search(42, diverse ? 3 : 42, strict);
+        EXPECT_EQUAL(1u, searchable.create_cnt);
+        MockSearch *ms = dynamic_cast<MockSearch*>(s1.get());
+        ASSERT_TRUE(ms != nullptr);
+        EXPECT_EQUAL("limiter_attribute", ms->spec.getName());
+        EXPECT_EQUAL(0u, ms->spec.getFieldId());
+        EXPECT_EQUAL(0u, ms->spec.getHandle());
+        EXPECT_EQUAL(strict, ms->strict());
+        EXPECT_TRUE(ms->postings_fetched);
+        if (descending) {
+            if (diverse) {
+                EXPECT_EQUAL("[;;-42;category;3;140;loose]", ms->term);
+            } else {
+                EXPECT_EQUAL("[;;-42]", ms->term);
+            }
+        } else {
+            if (diverse) {
+                EXPECT_EQUAL("[;;42;category;3;140;loose]", ms->term);
+            } else {
+                EXPECT_EQUAL("[;;42]", ms->term);
+            }
+        }
+        ASSERT_EQUAL(1u, ms->tfmda.size());
+        EXPECT_EQUAL(0u, ms->tfmda[0]->getFieldId());
+    }
+}
+
+TEST("require that no limiter has no behavior") {
+    NoMatchPhaseLimiter no_limiter;
+    MaybeMatchPhaseLimiter &limiter = no_limiter;
+    EXPECT_FALSE(limiter.is_enabled());
+    EXPECT_EQUAL(0u, limiter.sample_hits_per_thread(1));
+    SearchIterator::UP search = limiter.maybe_limit(prepare(new MockSearch("search")), 1.0, 100000000);
+    limiter.updateDocIdSpaceEstimate(1000, 9000);
+    EXPECT_EQUAL(std::numeric_limits<size_t>::max(), limiter.getDocIdSpaceEstimate());
+    MockSearch *ms = dynamic_cast<MockSearch*>(search.get());
+    ASSERT_TRUE(ms != nullptr);
+    EXPECT_EQUAL("search", ms->term);
+    EXPECT_FALSE(limiter.was_limited());
+}
+
+TEST("require that the match phase limiter may chose not to limit the query") {
+    FakeRequestContext requestContext;
+    MockSearchable searchable;
+    MatchPhaseLimiter yes_limiter(10000, searchable, requestContext, "limiter_attribute", 1000, true, 1.0, 0.2, 1.0, "", 1, 10.0, AttributeLimiter::LOOSE);
+    MaybeMatchPhaseLimiter &limiter = yes_limiter;
+    EXPECT_TRUE(limiter.is_enabled());
+    EXPECT_EQUAL(20u, limiter.sample_hits_per_thread(10));
+    SearchIterator::UP search = limiter.maybe_limit(prepare(new MockSearch("search")),
+                                                    0.005, 100000);
+    limiter.updateDocIdSpaceEstimate(1000, 9000);
+    EXPECT_EQUAL(10000u, limiter.getDocIdSpaceEstimate());
+    MockSearch *ms = dynamic_cast<MockSearch*>(search.get());
+    ASSERT_TRUE(ms != nullptr);
+    EXPECT_EQUAL("search", ms->term);
+    EXPECT_FALSE(limiter.was_limited());
+}
+
+struct MaxFilterCoverageLimiterFixture {
+
+    FakeRequestContext requestContext;
+    MockSearchable searchable;
+
+    MatchPhaseLimiter::UP getMaxFilterCoverageLimiter() {
+        MatchPhaseLimiter::UP yes_limiter(new MatchPhaseLimiter(10000, searchable, requestContext, "limiter_attribute", 10000, true, 0.05, 1.0, 1.0, "", 1, 10.0, AttributeLimiter::LOOSE));
+        MaybeMatchPhaseLimiter &limiter = *yes_limiter;
+        EXPECT_TRUE(limiter.is_enabled());
+        EXPECT_EQUAL(1000u, limiter.sample_hits_per_thread(10));
+        return yes_limiter;
+    }
+};
+
+TEST_F("require that the match phase limiter may chose not to limit the query when considering max-filter-coverage", MaxFilterCoverageLimiterFixture) {
+    MatchPhaseLimiter::UP limiterUP = f.getMaxFilterCoverageLimiter();
+    MaybeMatchPhaseLimiter & limiter = *limiterUP;
+    SearchIterator::UP search = limiter.maybe_limit(prepare(new MockSearch("search")), 0.10, 1900000);
+    limiter.updateDocIdSpaceEstimate(1000, 1899000);
+    EXPECT_EQUAL(1900000u, limiter.getDocIdSpaceEstimate());
+    MockSearch *ms = dynamic_cast<MockSearch *>(search.get());
+    ASSERT_TRUE(ms != nullptr);
+    EXPECT_EQUAL("search", ms->term);
+    EXPECT_FALSE(limiter.was_limited());
+}
+
+TEST_F("require that the match phase limiter may chose to limit the query even when considering max-filter-coverage", MaxFilterCoverageLimiterFixture) {
+    MatchPhaseLimiter::UP limiterUP = f.getMaxFilterCoverageLimiter();
+    MaybeMatchPhaseLimiter & limiter = *limiterUP;
+    SearchIterator::UP search = limiter.maybe_limit(prepare(new MockSearch("search")), 0.10, 2100000);
+    limiter.updateDocIdSpaceEstimate(1000, 2099000);
+    EXPECT_EQUAL(159684u, limiter.getDocIdSpaceEstimate());
+    LimitedSearch *strict_and = dynamic_cast<LimitedSearch*>(search.get());
+    ASSERT_TRUE(strict_and != nullptr);
+    const MockSearch *ms1 = dynamic_cast<const MockSearch*>(&strict_and->getFirst());
+    ASSERT_TRUE(ms1 != nullptr);
+    const MockSearch *ms2 = dynamic_cast<const MockSearch*>(&strict_and->getSecond());
+    ASSERT_TRUE(ms2 != nullptr);
+    EXPECT_EQUAL("[;;-100000]", ms1->term);
+    EXPECT_EQUAL("search", ms2->term);
+    EXPECT_TRUE(ms1->strict());
+    EXPECT_TRUE(ms2->strict());
+    EXPECT_TRUE(limiter.was_limited());
+}
+
+TEST("require that the match phase limiter is able to pre-limit the query") {
+    FakeRequestContext requestContext;
+    MockSearchable searchable;
+    MatchPhaseLimiter yes_limiter(10000, searchable, requestContext, "limiter_attribute", 500, true, 1.0, 0.2, 1.0, "", 1, 10.0, AttributeLimiter::LOOSE);
+    MaybeMatchPhaseLimiter &limiter = yes_limiter;
+    EXPECT_TRUE(limiter.is_enabled());
+    EXPECT_EQUAL(12u, limiter.sample_hits_per_thread(10));
+    SearchIterator::UP search = limiter.maybe_limit(prepare(new MockSearch("search")),
+                                                    0.1, 100000);
+    limiter.updateDocIdSpaceEstimate(1000, 9000);
+    EXPECT_EQUAL(1680u, limiter.getDocIdSpaceEstimate());
+    LimitedSearch *strict_and = dynamic_cast<LimitedSearch*>(search.get());
+    ASSERT_TRUE(strict_and != nullptr);
+    const MockSearch *ms1 = dynamic_cast<const MockSearch*>(&strict_and->getFirst());
+    ASSERT_TRUE(ms1 != nullptr);
+    const MockSearch *ms2 = dynamic_cast<const MockSearch*>(&strict_and->getSecond());
+    ASSERT_TRUE(ms2 != nullptr);
+    EXPECT_EQUAL("[;;-5000]", ms1->term);
+    EXPECT_EQUAL("search", ms2->term);
+    EXPECT_TRUE(ms1->strict());
+    EXPECT_TRUE(ms2->strict());
+    search->seek(100);
+    EXPECT_EQUAL(100u, ms1->last_seek);
+    EXPECT_EQUAL(100u, ms2->last_seek);
+    search->unpack(100);
+    EXPECT_EQUAL(0u, ms1->last_unpack); // will not unpack limiting term
+    EXPECT_EQUAL(100u, ms2->last_unpack);
+    EXPECT_TRUE(limiter.was_limited());
+}
+
+TEST("require that the match phase limiter is able to post-limit the query") {
+    MockSearchable searchable;
+    FakeRequestContext requestContext;
+    MatchPhaseLimiter yes_limiter(10000, searchable, requestContext,"limiter_attribute", 1500, true, 1.0, 0.2, 1.0, "", 1, 10.0, AttributeLimiter::LOOSE);
+    MaybeMatchPhaseLimiter &limiter = yes_limiter;
+    EXPECT_TRUE(limiter.is_enabled());
+    EXPECT_EQUAL(30u, limiter.sample_hits_per_thread(10));
+    SearchIterator::UP search = limiter.maybe_limit(prepare(new MockSearch("search")), 0.1, 100000);
+    limiter.updateDocIdSpaceEstimate(1000, 9000);
+    EXPECT_EQUAL(1680u, limiter.getDocIdSpaceEstimate());
+    LimitedSearch *strict_and = dynamic_cast<LimitedSearch*>(search.get());
+    ASSERT_TRUE(strict_and != nullptr);
+    const MockSearch *ms1 = dynamic_cast<const MockSearch*>(&strict_and->getFirst());
+    ASSERT_TRUE(ms1 != nullptr);
+    const MockSearch *ms2 = dynamic_cast<const MockSearch*>(&strict_and->getSecond());
+    ASSERT_TRUE(ms2 != nullptr);
+    EXPECT_EQUAL("search", ms1->term);
+    EXPECT_EQUAL("[;;-15000]", ms2->term);
+    EXPECT_TRUE(ms1->strict());
+    EXPECT_FALSE(ms2->strict());
+    search->seek(100);
+    EXPECT_EQUAL(100u, ms1->last_seek);
+    EXPECT_EQUAL(100u, ms2->last_seek);
+    search->unpack(100);
+    EXPECT_EQUAL(100u, ms1->last_unpack);
+    EXPECT_EQUAL(0u, ms2->last_unpack); // will not unpack limiting term
+    EXPECT_TRUE(limiter.was_limited());
+}
+
+void verifyDiversity(AttributeLimiter::DiversityCutoffStrategy strategy)
+{
+    MockSearchable searchable;
+    FakeRequestContext requestContext;
+    MatchPhaseLimiter yes_limiter(10000, searchable, requestContext,"limiter_attribute", 500, true, 1.0, 0.2, 1.0, "category", 10, 13.1, strategy);
+    MaybeMatchPhaseLimiter &limiter = yes_limiter;
+    SearchIterator::UP search = limiter.maybe_limit(prepare(new MockSearch("search")), 0.1, 100000);
+    limiter.updateDocIdSpaceEstimate(1000, 9000);
+    EXPECT_EQUAL(1680u, limiter.getDocIdSpaceEstimate());
+    LimitedSearch *strict_and = dynamic_cast<LimitedSearch*>(search.get());
+    ASSERT_TRUE(strict_and != nullptr);
+    const MockSearch *ms1 = dynamic_cast<const MockSearch*>(&strict_and->getFirst());
+    ASSERT_TRUE(ms1 != nullptr);
+    if (strategy == AttributeLimiter::LOOSE) {
+        EXPECT_EQUAL("[;;-5000;category;500;131;loose]", ms1->term);
+    } else if (strategy == AttributeLimiter::STRICT) {
+        EXPECT_EQUAL("[;;-5000;category;500;131;strict]", ms1->term);
+    } else {
+        ASSERT_TRUE(false);
+    }
+}
+
+TEST("require that the match phase limiter can use loose diversity") {
+    verifyDiversity(AttributeLimiter::LOOSE);
+}
+
+TEST("require that the match phase limiter can use strict diversity") {
+    verifyDiversity(AttributeLimiter::STRICT);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp
new file mode 100644
index 00000000000..237f283f042
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("matching_stats_test");
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchcore/proton/matching/matching_stats.h>
+
+using namespace proton::matching;
+
+TEST("requireThatDocCountsAddUp") {
+    MatchingStats stats;
+    EXPECT_EQUAL(0u, stats.docsMatched());
+    EXPECT_EQUAL(0u, stats.docsRanked());
+    EXPECT_EQUAL(0u, stats.docsReRanked());
+    EXPECT_EQUAL(0u, stats.queries());
+    EXPECT_EQUAL(0u, stats.limited_queries());
+    {
+        MatchingStats rhs;
+        EXPECT_EQUAL(&rhs.docsMatched(1000), &rhs);
+        EXPECT_EQUAL(&rhs.docsRanked(100), &rhs);
+        EXPECT_EQUAL(&rhs.docsReRanked(10), &rhs);
+        EXPECT_EQUAL(&rhs.queries(2), &rhs);
+        EXPECT_EQUAL(&rhs.limited_queries(1), &rhs);
+        EXPECT_EQUAL(&stats.add(rhs), &stats);
+    }
+    EXPECT_EQUAL(1000u, stats.docsMatched());
+    EXPECT_EQUAL(100u, stats.docsRanked());
+    EXPECT_EQUAL(10u, stats.docsReRanked());
+    EXPECT_EQUAL(2u, stats.queries());
+    EXPECT_EQUAL(1u, stats.limited_queries());
+    EXPECT_EQUAL(&stats.add(MatchingStats().docsMatched(1000).docsRanked(100)
+                            .docsReRanked(10).queries(2).limited_queries(1)), &stats);
+    EXPECT_EQUAL(2000u, stats.docsMatched());
+    EXPECT_EQUAL(200u, stats.docsRanked());
+    EXPECT_EQUAL(20u, stats.docsReRanked());
+    EXPECT_EQUAL(4u, stats.queries());
+    EXPECT_EQUAL(2u, stats.limited_queries());
+}
+
+TEST("requireThatAverageTimesAreRecorded") {
+    MatchingStats stats;
+    EXPECT_APPROX(0.0, stats.matchTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.0, stats.groupingTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.0, stats.rerankTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.0, stats.queryCollateralTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.0, stats.queryLatencyAvg(), 0.00001);
+    EXPECT_EQUAL(0u, stats.matchTimeCount());
+    EXPECT_EQUAL(0u, stats.groupingTimeCount());
+    EXPECT_EQUAL(0u, stats.rerankTimeCount());
+    EXPECT_EQUAL(0u, stats.queryCollateralTimeCount());
+    EXPECT_EQUAL(0u, stats.queryLatencyCount());
+    stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).queryLatency(1.0);
+    EXPECT_APPROX(0.01, stats.matchTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.1, stats.groupingTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.5, stats.rerankTimeAvg(), 0.00001);
+    EXPECT_APPROX(2.0, stats.queryCollateralTimeAvg(), 0.00001);
+    EXPECT_APPROX(1.0, stats.queryLatencyAvg(), 0.00001);
+    stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).queryLatency(3.0));
+    EXPECT_APPROX(0.02, stats.matchTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.2, stats.groupingTimeAvg(), 0.00001);
+    EXPECT_APPROX(1.0, stats.rerankTimeAvg(), 0.00001);
+    EXPECT_APPROX(4.0, stats.queryCollateralTimeAvg(), 0.00001);
+    EXPECT_APPROX(2.0, stats.queryLatencyAvg(), 0.00001);
+    stats.add(MatchingStats().matchTime(0.05)
+              .groupingTime(0.5)
+              .rerankTime(2.5)
+              .queryCollateralTime(10.0)
+              .queryLatency(5.0));
+    stats.add(MatchingStats().matchTime(0.05).matchTime(0.03)
+              .groupingTime(0.5).groupingTime(0.3)
+              .rerankTime(2.5).rerankTime(1.5)
+              .queryCollateralTime(10.0).queryCollateralTime(6.0)
+              .queryLatency(5.0).queryLatency(3.0));
+    EXPECT_APPROX(0.03, stats.matchTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.3, stats.groupingTimeAvg(), 0.00001);
+    EXPECT_APPROX(1.5, stats.rerankTimeAvg(), 0.00001);
+    EXPECT_APPROX(6.0, stats.queryCollateralTimeAvg(), 0.00001);
+    EXPECT_APPROX(3.0, stats.queryLatencyAvg(), 0.00001);
+    EXPECT_EQUAL(4u, stats.matchTimeCount());
+    EXPECT_EQUAL(4u, stats.groupingTimeCount());
+    EXPECT_EQUAL(4u, stats.rerankTimeCount());
+    EXPECT_EQUAL(4u, stats.queryCollateralTimeCount());
+    EXPECT_EQUAL(4u, stats.queryLatencyCount());
+}
+
+TEST("requireThatPartitionsAreAddedCorrectly") {
+    MatchingStats all1;
+    EXPECT_EQUAL(0u, all1.docsMatched());
+    EXPECT_EQUAL(0u, all1.getNumPartitions());
+
+    MatchingStats::Partition subPart;
+    subPart.docsMatched(3).docsRanked(2).docsReRanked(1)
+        .active_time(1.0).wait_time(0.5);
+    EXPECT_EQUAL(3u, subPart.docsMatched());
+    EXPECT_EQUAL(2u, subPart.docsRanked());
+    EXPECT_EQUAL(1u, subPart.docsReRanked());
+    EXPECT_EQUAL(1.0, subPart.active_time_avg());
+    EXPECT_EQUAL(0.5, subPart.wait_time_avg());
+    EXPECT_EQUAL(1u, subPart.active_time_count());
+    EXPECT_EQUAL(1u, subPart.wait_time_count());
+
+    all1.merge_partition(subPart, 0);
+    EXPECT_EQUAL(3u, all1.docsMatched());
+    EXPECT_EQUAL(2u, all1.docsRanked());
+    EXPECT_EQUAL(1u, all1.docsReRanked());
+    EXPECT_EQUAL(1u, all1.getNumPartitions());
+    EXPECT_EQUAL(3u, all1.getPartition(0).docsMatched());
+    EXPECT_EQUAL(2u, all1.getPartition(0).docsRanked());
+    EXPECT_EQUAL(1u, all1.getPartition(0).docsReRanked());
+    EXPECT_EQUAL(1.0, all1.getPartition(0).active_time_avg());
+    EXPECT_EQUAL(0.5, all1.getPartition(0).wait_time_avg());
+    EXPECT_EQUAL(1u, all1.getPartition(0).active_time_count());
+    EXPECT_EQUAL(1u, all1.getPartition(0).wait_time_count());
+    
+    all1.merge_partition(subPart, 1);
+    EXPECT_EQUAL(6u, all1.docsMatched());
+    EXPECT_EQUAL(4u, all1.docsRanked());
+    EXPECT_EQUAL(2u, all1.docsReRanked());
+    EXPECT_EQUAL(2u, all1.getNumPartitions());
+    EXPECT_EQUAL(3u, all1.getPartition(1).docsMatched());
+    EXPECT_EQUAL(2u, all1.getPartition(1).docsRanked());
+    EXPECT_EQUAL(1u, all1.getPartition(1).docsReRanked());
+    EXPECT_EQUAL(1.0, all1.getPartition(1).active_time_avg());
+    EXPECT_EQUAL(0.5, all1.getPartition(1).wait_time_avg());
+    EXPECT_EQUAL(1u, all1.getPartition(1).active_time_count());
+    EXPECT_EQUAL(1u, all1.getPartition(1).wait_time_count());
+
+    all1.add(all1);
+    EXPECT_EQUAL(12u, all1.docsMatched());
+    EXPECT_EQUAL(8u, all1.docsRanked());
+    EXPECT_EQUAL(4u, all1.docsReRanked());
+    EXPECT_EQUAL(2u, all1.getNumPartitions());
+    EXPECT_EQUAL(6u, all1.getPartition(0).docsMatched());
+    EXPECT_EQUAL(4u, all1.getPartition(0).docsRanked());
+    EXPECT_EQUAL(2u, all1.getPartition(0).docsReRanked());
+    EXPECT_EQUAL(1.0, all1.getPartition(0).active_time_avg());
+    EXPECT_EQUAL(0.5, all1.getPartition(0).wait_time_avg());
+    EXPECT_EQUAL(2u, all1.getPartition(0).active_time_count());
+    EXPECT_EQUAL(2u, all1.getPartition(0).wait_time_count());
+    EXPECT_EQUAL(6u, all1.getPartition(1).docsMatched());
+    EXPECT_EQUAL(4u, all1.getPartition(1).docsRanked());
+    EXPECT_EQUAL(2u, all1.getPartition(1).docsReRanked());
+    EXPECT_EQUAL(1.0, all1.getPartition(1).active_time_avg());
+    EXPECT_EQUAL(0.5, all1.getPartition(1).wait_time_avg());
+    EXPECT_EQUAL(2u, all1.getPartition(1).active_time_count());
+    EXPECT_EQUAL(2u, all1.getPartition(1).wait_time_count());
+}
+
+TEST_MAIN() {
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp
new file mode 100644
index 00000000000..b650c983be0
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/matching_test.cpp
@@ -0,0 +1,775 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("matching_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/document/base/globalid.h>
+#include <initializer_list>
+#include <vespa/searchcommon/attribute/iattributecontext.h>
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+#include <vespa/searchcore/proton/matching/fakesearchcontext.h>
+#include <vespa/searchcore/proton/matching/isearchcontext.h>
+#include <vespa/searchcore/proton/matching/matcher.h>
+#include <vespa/searchcore/proton/matching/querynodes.h>
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
+#include <vespa/searchcore/proton/matching/viewresolver.h>
+#include <vespa/searchlib/aggregation/aggregation.h>
+#include <vespa/searchlib/aggregation/grouping.h>
+#include <vespa/searchlib/aggregation/perdocexpression.h>
+#include <vespa/searchlib/attribute/extendableattributes.h>
+#include <vespa/searchlib/common/featureset.h>
+#include <vespa/searchlib/engine/docsumrequest.h>
+#include <vespa/searchlib/fef/properties.h>
+#include <vespa/searchlib/query/tree/querybuilder.h>
+#include <vespa/searchlib/query/tree/stackdumpcreator.h>
+#include <vespa/searchlib/queryeval/isourceselector.h>
+#include <vespa/vespalib/util/simple_thread_bundle.h>
+#include <vespa/searchcore/proton/matching/match_params.h>
+
+using namespace proton::matching;
+using namespace proton;
+using namespace search::aggregation;
+using namespace search::attribute;
+using namespace search::engine;
+using namespace search::expression;
+using namespace search::fef;
+using namespace search::grouping;
+using namespace search::index;
+using namespace search::query;
+using namespace search::queryeval;
+using namespace search;
+using storage::spi::Timestamp;
+
+void inject_match_phase_limiting(Properties &setup, const vespalib::string &attribute, size_t max_hits, bool descending)
+{
+    Properties cfg;
+    cfg.add(indexproperties::matchphase::DegradationAttribute::NAME, attribute);
+    cfg.add(indexproperties::matchphase::DegradationAscendingOrder::NAME, descending ? "false" : "true");
+    cfg.add(indexproperties::matchphase::DegradationMaxHits::NAME, vespalib::make_string("%zu", max_hits));
+    setup.import(cfg);
+}
+
+//-----------------------------------------------------------------------------
+
+const uint32_t NUM_DOCS = 1000;
+
+//-----------------------------------------------------------------------------
+
+class MyAttributeContext : public IAttributeContext
+{
+private:
+    typedef std::map<string, IAttributeVector *> Map;
+    Map _vectors;
+
+public:
+    const IAttributeVector *get(const string &name) const {
+        if (_vectors.find(name) == _vectors.end()) {
+            return 0;
+        }
+        return _vectors.find(name)->second;
+    }
+    virtual const IAttributeVector *
+    getAttribute(const string &name) const {
+        return get(name);
+    }
+    virtual const IAttributeVector *
+    getAttributeStableEnum(const string &name) const {
+        return get(name);
+    }
+    virtual void
+    getAttributeList(std::vector<const IAttributeVector *> & list) const {
+        Map::const_iterator pos = _vectors.begin();
+        Map::const_iterator end = _vectors.end();
+        for (; pos != end; ++pos) {
+            list.push_back(pos->second);
+        }
+    }
+    ~MyAttributeContext() {
+        Map::iterator pos = _vectors.begin();
+        Map::iterator end = _vectors.end();
+        for (; pos != end; ++pos) {
+            delete pos->second;
+        }
+    }
+
+    //-------------------------------------------------------------------------
+
+    void add(IAttributeVector *attr) {
+        _vectors[attr->getName()] = attr;
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+struct MyWorld {
+    Schema                  schema;
+    Properties              config;
+    FakeSearchContext       searchContext;
+    MyAttributeContext      attributeContext;
+    SessionManager::SP      sessionManager;
+    DocumentMetaStore       metaStore;
+    MatchingStats           matchingStats;
+    vespalib::Clock         clock;
+    QueryLimiter            queryLimiter;
+
+    MyWorld()
+        : schema(),
+          config(),
+          searchContext(),
+          attributeContext(),
+          sessionManager(),
+          metaStore(std::make_shared<BucketDBOwner>()),
+          matchingStats(),
+          clock(),
+          queryLimiter()
+    {
+    }
+
+    void basicSetup(size_t heapSize=10, size_t arraySize=100) {
+        // schema
+        schema.addIndexField(Schema::IndexField("f1", Schema::STRING));
+        schema.addIndexField(Schema::IndexField("f2", Schema::STRING));
+        schema.addIndexField(Schema::IndexField("tensor_field", Schema::TENSOR));
+        schema.addAttributeField(Schema::AttributeField("a1", Schema::INT32));
+        schema.addAttributeField(Schema::AttributeField("a2", Schema::INT32));
+        schema.addAttributeField(Schema::AttributeField("predicate_field", Schema::BOOLEANTREE));
+
+        // config
+        config.add(indexproperties::rank::FirstPhase::NAME, "attribute(a1)");
+        config.add(indexproperties::hitcollector::HeapSize::NAME, (vespalib::asciistream() << heapSize).str());
+        config.add(indexproperties::hitcollector::ArraySize::NAME, (vespalib::asciistream() << arraySize).str());
+        config.add(indexproperties::summary::Feature::NAME, "attribute(a1)");
+        config.add(indexproperties::summary::Feature::NAME, "value(100)");
+        config.add(indexproperties::dump::IgnoreDefaultFeatures::NAME, "true");
+        config.add(indexproperties::dump::Feature::NAME, "attribute(a2)");
+
+        // search context
+        searchContext.setLimit(NUM_DOCS);
+        searchContext.addIdx(0).addIdx(1);
+        for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+            searchContext.selector().setSource(i, i % 2); // even -> 0
+                                                    // odd  -> 1
+        }
+
+        // attribute context
+        {
+            SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("a1");
+            AttributeVector::DocId docid;
+            for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+                attr->addDoc(docid);
+                attr->add(i, docid); // value = docid
+            }
+            assert(docid + 1 == NUM_DOCS);
+            attributeContext.add(attr);
+        }
+        {
+            SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("a2");
+            AttributeVector::DocId docid;
+            for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+                attr->addDoc(docid);
+                attr->add(i * 2, docid); // value = docid * 2
+            }
+            assert(docid + 1 == NUM_DOCS);
+            attributeContext.add(attr);
+        }
+
+        // grouping
+        sessionManager = SessionManager::SP(new SessionManager(100));
+
+        // metaStore
+        for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+            document::DocumentId docId(vespalib::make_string("doc::%u", i));
+            const document::GlobalId &gid = docId.getGlobalId();
+            typedef DocumentMetaStore::Result PutRes;
+            document::BucketId bucketId(BucketFactory::getBucketId(docId));
+            PutRes putRes(metaStore.put(gid,
+                                        bucketId,
+                                        Timestamp(0u),
+                                        i));
+            metaStore.setBucketState(bucketId, true);
+        }
+    }
+
+    void set_property(const vespalib::string &name, const vespalib::string &value) {
+        Properties cfg;
+        cfg.add(name, value);
+        config.import(cfg);
+    }
+
+    void setup_match_phase_limiting(const vespalib::string &attribute, size_t max_hits, bool descending)
+    {
+        inject_match_phase_limiting(config, attribute, max_hits, descending);
+    }
+
+    void add_match_phase_limiting_result(const vespalib::string &attribute, size_t want_docs,
+                                         bool descending, std::initializer_list<uint32_t> docs)
+    {
+        vespalib::string term = vespalib::make_string("[;;%s%zu]", descending ? "-" : "", want_docs);
+        FakeResult result;
+        for (uint32_t doc: docs) {
+            result.doc(doc);
+        }
+        searchContext.attr().addResult(attribute, term, result);
+    }
+
+    void setupSecondPhaseRanking() {
+        Properties cfg;
+        cfg.add(indexproperties::rank::SecondPhase::NAME, "attribute(a2)");
+        cfg.add(indexproperties::hitcollector::HeapSize::NAME, "3");
+        config.import(cfg);
+    }
+
+    void verbose_a1_result(const vespalib::string &term) {
+        FakeResult result;
+        for (uint32_t i = 15; i < NUM_DOCS; ++i) {
+            result.doc(i);
+        }
+        searchContext.attr().addResult("a1", term, result);
+    }
+
+    void basicResults() {
+        searchContext.idx(0).getFake().addResult("f1", "foo",
+                                                 FakeResult()
+                                                 .doc(10).doc(20).doc(30));
+        searchContext.idx(0).getFake().addResult(
+                "f1", "spread",
+                FakeResult()
+                .doc(100).doc(200).doc(300).doc(400).doc(500)
+                .doc(600).doc(700).doc(800).doc(900));
+    }
+
+    void setStackDump(Request &request, const vespalib::string &field,
+                      const vespalib::string &term) {
+        QueryBuilder<ProtonNodeTypes> builder;
+        builder.addStringTerm(term, field, 1, search::query::Weight(1));
+        vespalib::string stack_dump =
+            StackDumpCreator::create(*builder.build());
+        request.stackDump.assign(stack_dump.data(),
+                                 stack_dump.data() + stack_dump.size());
+    }
+
+    SearchRequest::SP createSimpleRequest(const vespalib::string &field,
+            const vespalib::string &term)
+    {
+        SearchRequest::SP request(new SearchRequest);
+        request->setTimeout(60 * fastos::TimeStamp::SEC);
+        setStackDump(*request, field, term);
+        request->maxhits = 10;
+        return request;
+    }
+
+    struct MySearchHandler : ISearchHandler {
+        Matcher::SP _matcher;
+
+        MySearchHandler(Matcher::SP matcher) : _matcher(matcher) {}
+
+        virtual DocsumReply::UP getDocsums(const DocsumRequest &)
+        { return DocsumReply::UP(); }
+        virtual SearchReply::UP match(const ISearchHandler::SP &,
+                                      const SearchRequest &,
+                                      vespalib::ThreadBundle &) const
+        { return SearchReply::UP(); }
+    };
+
+    double get_first_phase_termwise_limit() {
+        Matcher matcher(schema, config, clock, queryLimiter, 0);
+        SearchRequest::SP request = createSimpleRequest("f1", "spread");
+        search::fef::Properties overrides;
+        MatchToolsFactory::UP match_tools_factory = matcher.create_match_tools_factory(
+                *request, searchContext, attributeContext, metaStore, overrides);
+        MatchTools::UP match_tools = match_tools_factory->createMatchTools();
+        RankProgram::UP rank_program = match_tools->first_phase_program();
+        return rank_program->match_data().get_termwise_limit();
+    }
+
+    SearchReply::UP performSearch(SearchRequest::SP req, size_t threads) {
+        Matcher::SP matcher(new Matcher(schema, config, clock, queryLimiter, 0));
+        SearchSession::OwnershipBundle owned_objects;
+        owned_objects.search_handler.reset(new MySearchHandler(matcher));
+        owned_objects.context.reset(new MatchContext(
+                        IAttributeContext::UP(new MyAttributeContext),
+                        ISearchContext::UP(new FakeSearchContext)));
+        vespalib::SimpleThreadBundle threadBundle(threads);
+        SearchReply::UP reply =
+            matcher->match(*req, threadBundle, searchContext, attributeContext,
+                           *sessionManager, metaStore,
+                           std::move(owned_objects));
+        matchingStats.add(matcher->getStats());
+        return reply;
+    }
+
+    DocsumRequest::SP createSimpleDocsumRequest(const vespalib::string & field,
+                                                const vespalib::string & term)
+    {
+        DocsumRequest::SP request(new DocsumRequest);
+        setStackDump(*request, field, term);
+
+        // match a subset of basic result + request for a non-hit (not
+        // sorted on docid)
+        request->hits.push_back(DocsumRequest::Hit());
+        request->hits.back().docid = 30;
+        request->hits.push_back(DocsumRequest::Hit());
+        request->hits.back().docid = 10;
+        request->hits.push_back(DocsumRequest::Hit());
+        request->hits.back().docid = 15;
+        return request;
+    }
+
+    std::unique_ptr<FieldInfo> get_field_info(const vespalib::string &field_name) {
+        Matcher::SP matcher(new Matcher(schema, config, clock, queryLimiter, 0));
+        const FieldInfo *field = matcher->get_index_env().getFieldByName(field_name);
+        if (field == nullptr) {
+            return std::unique_ptr<FieldInfo>(nullptr);
+        }
+        return std::make_unique<FieldInfo>(*field);
+    }
+
+    FeatureSet::SP getSummaryFeatures(DocsumRequest::SP req) {
+        Matcher matcher(schema, config, clock, queryLimiter, 0);
+        return matcher.getSummaryFeatures(*req, searchContext,
+                                          attributeContext, *sessionManager);
+    }
+
+    FeatureSet::SP getRankFeatures(DocsumRequest::SP req) {
+        Matcher matcher(schema, config, clock, queryLimiter, 0);
+        return matcher.getRankFeatures(*req, searchContext, attributeContext,
+                                       *sessionManager);
+    }
+
+};
+
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+
+void verifyViewResolver(const ViewResolver &resolver) {
+    {
+        std::vector<vespalib::string> fields;
+        EXPECT_TRUE(resolver.resolve("foo", fields));
+        ASSERT_TRUE(fields.size() == 2u);
+        EXPECT_EQUAL("x", fields[0]);
+        EXPECT_EQUAL("y", fields[1]);
+    }
+    {
+        std::vector<vespalib::string> fields;
+        EXPECT_TRUE(resolver.resolve("bar", fields));
+        ASSERT_TRUE(fields.size() == 1u);
+        EXPECT_EQUAL("z", fields[0]);
+    }
+    {
+        std::vector<vespalib::string> fields;
+        EXPECT_TRUE(!resolver.resolve("baz", fields));
+        ASSERT_TRUE(fields.size() == 1u);
+        EXPECT_EQUAL("baz", fields[0]);
+    }
+}
+
+TEST("require that view resolver can be set up directly") {
+    ViewResolver resolver;
+    resolver.add("foo", "x").add("foo", "y").add("bar", "z");
+    TEST_DO(verifyViewResolver(resolver));
+}
+
+TEST("require that view resolver can be set up from schema") {
+    Schema schema;
+    Schema::FieldSet foo("foo");
+    foo.addField("x").addField("y");
+    Schema::FieldSet bar("bar");
+    bar.addField("z");
+    schema.addFieldSet(foo);
+    schema.addFieldSet(bar);
+    ViewResolver resolver = ViewResolver::createFromSchema(schema);
+    TEST_DO(verifyViewResolver(resolver));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("require that matching is performed (multi-threaded)") {
+    for (size_t threads = 1; threads <= 16; ++threads) {
+        MyWorld world;
+        world.basicSetup();
+        world.basicResults();
+        SearchRequest::SP request = world.createSimpleRequest("f1", "spread");
+        SearchReply::UP reply = world.performSearch(request, threads);
+        EXPECT_EQUAL(9u, world.matchingStats.docsMatched());
+        EXPECT_EQUAL(9u, reply->hits.size());
+        EXPECT_GREATER(world.matchingStats.matchTimeAvg(), 0.0000001);
+    }
+}
+
+TEST("require that matching also returns hits when only bitvector is used (multi-threaded)") {
+    for (size_t threads = 1; threads <= 16; ++threads) {
+        MyWorld world;
+        world.basicSetup(0, 0);
+        world.verbose_a1_result("all");
+        SearchRequest::SP request = world.createSimpleRequest("a1", "all");
+        SearchReply::UP reply = world.performSearch(request, threads);
+        EXPECT_EQUAL(985u, world.matchingStats.docsMatched());
+        EXPECT_EQUAL(10u, reply->hits.size());
+        EXPECT_GREATER(world.matchingStats.matchTimeAvg(), 0.0000001);
+    }
+}
+
+TEST("require that ranking is performed (multi-threaded)") {
+    for (size_t threads = 1; threads <= 16; ++threads) {
+        MyWorld world;
+        world.basicSetup();
+        world.basicResults();
+        SearchRequest::SP request = world.createSimpleRequest("f1", "spread");
+        SearchReply::UP reply = world.performSearch(request, threads);
+        EXPECT_EQUAL(9u, world.matchingStats.docsMatched());
+        EXPECT_EQUAL(9u, world.matchingStats.docsRanked());
+        EXPECT_EQUAL(0u, world.matchingStats.docsReRanked());
+        ASSERT_TRUE(reply->hits.size() == 9u);
+        EXPECT_EQUAL(document::DocumentId("doc::900").getGlobalId(),  reply->hits[0].gid);
+        EXPECT_EQUAL(900.0, reply->hits[0].metric);
+        EXPECT_EQUAL(document::DocumentId("doc::800").getGlobalId(),  reply->hits[1].gid);
+        EXPECT_EQUAL(800.0, reply->hits[1].metric);
+        EXPECT_EQUAL(document::DocumentId("doc::700").getGlobalId(),  reply->hits[2].gid);
+        EXPECT_EQUAL(700.0, reply->hits[2].metric);
+        EXPECT_GREATER(world.matchingStats.matchTimeAvg(), 0.0000001);
+        EXPECT_EQUAL(0.0, world.matchingStats.rerankTimeAvg());
+    }
+}
+
+TEST("require that re-ranking is performed (multi-threaded)") {
+    for (size_t threads = 1; threads <= 16; ++threads) {
+        MyWorld world;
+        world.basicSetup();
+        world.setupSecondPhaseRanking();
+        world.basicResults();
+        SearchRequest::SP request = world.createSimpleRequest("f1", "spread");
+        SearchReply::UP reply = world.performSearch(request, threads);
+        EXPECT_EQUAL(9u, world.matchingStats.docsMatched());
+        EXPECT_EQUAL(9u, world.matchingStats.docsRanked());
+        EXPECT_EQUAL(3u, world.matchingStats.docsReRanked());
+        ASSERT_TRUE(reply->hits.size() == 9u);
+        EXPECT_EQUAL(document::DocumentId("doc::900").getGlobalId(),  reply->hits[0].gid);
+        EXPECT_EQUAL(1800.0, reply->hits[0].metric);
+        EXPECT_EQUAL(document::DocumentId("doc::800").getGlobalId(),  reply->hits[1].gid);
+        EXPECT_EQUAL(1600.0, reply->hits[1].metric);
+        EXPECT_EQUAL(document::DocumentId("doc::700").getGlobalId(),  reply->hits[2].gid);
+        EXPECT_EQUAL(1400.0, reply->hits[2].metric);
+        EXPECT_EQUAL(document::DocumentId("doc::600").getGlobalId(),  reply->hits[3].gid);
+        EXPECT_EQUAL(600.0, reply->hits[3].metric);
+        EXPECT_EQUAL(document::DocumentId("doc::500").getGlobalId(),  reply->hits[4].gid);
+        EXPECT_EQUAL(500.0, reply->hits[4].metric);
+        EXPECT_GREATER(world.matchingStats.matchTimeAvg(), 0.0000001);
+        EXPECT_GREATER(world.matchingStats.rerankTimeAvg(), 0.0000001);
+    }
+}
+
+TEST("require that sortspec can be used (multi-threaded)") {
+    for (size_t threads = 1; threads <= 16; ++threads) {
+        MyWorld world;
+        world.basicSetup();
+        world.basicResults();
+        SearchRequest::SP request = world.createSimpleRequest("f1", "spread");
+        request->sortSpec = "+a1";
+        SearchReply::UP reply = world.performSearch(request, threads);
+        ASSERT_EQUAL(9u, reply->hits.size());
+        EXPECT_EQUAL(document::DocumentId("doc::100").getGlobalId(),  reply->hits[0].gid);
+        EXPECT_EQUAL(0.0, reply->hits[0].metric);
+        EXPECT_EQUAL(document::DocumentId("doc::200").getGlobalId(),  reply->hits[1].gid);
+        EXPECT_EQUAL(0.0, reply->hits[1].metric);
+        EXPECT_EQUAL(document::DocumentId("doc::300").getGlobalId(),  reply->hits[2].gid);
+        EXPECT_EQUAL(0.0, reply->hits[2].metric);
+    }
+}
+
+TEST("require that grouping is performed (multi-threaded)") {
+    for (size_t threads = 1; threads <= 16; ++threads) {
+        MyWorld world;
+        world.basicSetup();
+        world.basicResults();
+        SearchRequest::SP request = world.createSimpleRequest("f1", "spread");
+        {
+            vespalib::nbostream buf;
+            vespalib::NBOSerializer os(buf);
+            uint32_t n = 1;
+            os << n;
+            Grouping grequest =
+                Grouping()
+                .setRoot(Group()
+                         .addResult(SumAggregationResult()
+                                    .setExpression(AttributeNode("a1"))));
+            grequest.serialize(os);
+            request->groupSpec.assign(buf.c_str(), buf.c_str() + buf.size());
+        }
+        SearchReply::UP reply = world.performSearch(request, threads);
+        {
+            vespalib::nbostream buf(&reply->groupResult[0],
+                                    reply->groupResult.size());
+            vespalib::NBOSerializer is(buf);
+            uint32_t n;
+            is >> n;
+            EXPECT_EQUAL(1u, n);
+            Grouping gresult;
+            gresult.deserialize(is);
+            Grouping gexpect = Grouping()
+                               .setRoot(Group()
+                                        .addResult(SumAggregationResult()
+                                                   .setExpression(AttributeNode("a1"))
+                                                   .setResult(Int64ResultNode(4500))));
+            EXPECT_EQUAL(gexpect.root().asString(), gresult.root().asString());
+        }
+        EXPECT_GREATER(world.matchingStats.groupingTimeAvg(), 0.0000001);
+    }
+}
+
+TEST("require that summary features are filled") {
+    MyWorld world;
+    world.basicSetup();
+    world.basicResults();
+    DocsumRequest::SP req = world.createSimpleDocsumRequest("f1", "foo");
+    FeatureSet::SP fs = world.getSummaryFeatures(req);
+    const feature_t * f = NULL;
+    EXPECT_EQUAL(2u, fs->numFeatures());
+    EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]);
+    EXPECT_EQUAL("value(100)", fs->getNames()[1]);
+    EXPECT_EQUAL(2u, fs->numDocs());
+    f = fs->getFeaturesByDocId(10);
+    EXPECT_TRUE(f != NULL);
+    EXPECT_EQUAL(10, f[0]);
+    EXPECT_EQUAL(100, f[1]);
+    f = fs->getFeaturesByDocId(15);
+    EXPECT_TRUE(f == NULL);
+    f = fs->getFeaturesByDocId(30);
+    EXPECT_TRUE(f != NULL);
+    EXPECT_EQUAL(30, f[0]);
+    EXPECT_EQUAL(100, f[1]);
+}
+
+TEST("require that rank features are filled") {
+    MyWorld world;
+    world.basicSetup();
+    world.basicResults();
+    DocsumRequest::SP req = world.createSimpleDocsumRequest("f1", "foo");
+    FeatureSet::SP fs = world.getRankFeatures(req);
+    const feature_t * f = NULL;
+    EXPECT_EQUAL(1u, fs->numFeatures());
+    EXPECT_EQUAL("attribute(a2)", fs->getNames()[0]);
+    EXPECT_EQUAL(2u, fs->numDocs());
+    f = fs->getFeaturesByDocId(10);
+    EXPECT_TRUE(f != NULL);
+    EXPECT_EQUAL(20, f[0]);
+    f = fs->getFeaturesByDocId(15);
+    EXPECT_TRUE(f == NULL);
+    f = fs->getFeaturesByDocId(30);
+    EXPECT_TRUE(f != NULL);
+    EXPECT_EQUAL(60, f[0]);
+}
+
+TEST("require that search session can be cached") {
+    MyWorld world;
+    world.basicSetup();
+    world.basicResults();
+    SearchRequest::SP request = world.createSimpleRequest("f1", "foo");
+    request->propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true");
+    request->sessionId.push_back('a');
+    EXPECT_EQUAL(0u, world.sessionManager->getSearchStats().numInsert);
+    SearchReply::UP reply = world.performSearch(request, 1);
+    EXPECT_EQUAL(1u, world.sessionManager->getSearchStats().numInsert);
+    SearchSession::SP session = world.sessionManager->pickSearch("a");
+    ASSERT_TRUE(session.get());
+    EXPECT_EQUAL(request->getTimeOfDoom(), session->getTimeOfDoom());
+    EXPECT_EQUAL("a", session->getSessionId());
+}
+
+TEST("require that getSummaryFeatures can use cached query setup") {
+    MyWorld world;
+    world.basicSetup();
+    world.basicResults();
+    SearchRequest::SP request = world.createSimpleRequest("f1", "foo");
+    request->propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true");
+    request->sessionId.push_back('a');
+    world.performSearch(request, 1);
+
+    DocsumRequest::SP docsum_request(new DocsumRequest);  // no stack dump
+    docsum_request->sessionId = request->sessionId;
+    docsum_request->
+        propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true");
+    docsum_request->hits.push_back(DocsumRequest::Hit());
+    docsum_request->hits.back().docid = 30;
+
+    FeatureSet::SP fs = world.getSummaryFeatures(docsum_request);
+    ASSERT_EQUAL(2u, fs->numFeatures());
+    EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]);
+    EXPECT_EQUAL("value(100)", fs->getNames()[1]);
+    ASSERT_EQUAL(1u, fs->numDocs());
+    const feature_t *f = fs->getFeaturesByDocId(30);
+    ASSERT_TRUE(f);
+    EXPECT_EQUAL(30, f[0]);
+    EXPECT_EQUAL(100, f[1]);
+
+    // getSummaryFeatures can be called multiple times.
+    fs = world.getSummaryFeatures(docsum_request);
+    ASSERT_EQUAL(2u, fs->numFeatures());
+    EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]);
+    EXPECT_EQUAL("value(100)", fs->getNames()[1]);
+    ASSERT_EQUAL(1u, fs->numDocs());
+    f = fs->getFeaturesByDocId(30);
+    ASSERT_TRUE(f);
+    EXPECT_EQUAL(30, f[0]);
+    EXPECT_EQUAL(100, f[1]);
+}
+
+TEST("require that getSummaryFeatures prefers cached query setup") {
+    MyWorld world;
+    world.basicSetup();
+    world.basicResults();
+    SearchRequest::SP request = world.createSimpleRequest("f1", "spread");
+    request->propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true");
+    request->sessionId.push_back('a');
+    world.performSearch(request, 1);
+
+    DocsumRequest::SP req = world.createSimpleDocsumRequest("f1", "foo");
+    req->sessionId = request->sessionId;
+    req->propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true");
+    FeatureSet::SP fs = world.getSummaryFeatures(req);
+    EXPECT_EQUAL(2u, fs->numFeatures());
+    ASSERT_EQUAL(0u, fs->numDocs());  // "spread" has no hits
+
+    // Empty cache
+    auto pruneTime = fastos::ClockSystem::now() +
+                     fastos::TimeStamp::MINUTE * 10;
+    world.sessionManager->pruneTimedOutSessions(pruneTime);
+
+    fs = world.getSummaryFeatures(req);
+    EXPECT_EQUAL(2u, fs->numFeatures());
+    ASSERT_EQUAL(2u, fs->numDocs());  // "foo" has two hits
+}
+
+TEST("require that match params are set up straight with ranking on") {
+    MatchParams p(1, 2, 4, 0.7, 0, 1, true, true);
+    ASSERT_EQUAL(1u, p.numDocs);
+    ASSERT_EQUAL(2u, p.heapSize);
+    ASSERT_EQUAL(4u, p.arraySize);
+    ASSERT_EQUAL(0.7, p.rankDropLimit);
+    ASSERT_EQUAL(0u, p.offset);
+    ASSERT_EQUAL(1u, p.hits);
+}
+
+TEST("require that match params are set up straight with ranking on arraySize is atleast the size of heapSize") {
+    MatchParams p(1, 6, 4, 0.7, 1, 1, true, true);
+    ASSERT_EQUAL(1u, p.numDocs);
+    ASSERT_EQUAL(6u, p.heapSize);
+    ASSERT_EQUAL(6u, p.arraySize);
+    ASSERT_EQUAL(0.7, p.rankDropLimit);
+    ASSERT_EQUAL(1u, p.offset);
+    ASSERT_EQUAL(1u, p.hits);
+}
+
+TEST("require that match params are set up straight with ranking on arraySize is atleast the size of hits+offset") {
+    MatchParams p(1, 6, 4, 0.7, 4, 4, true, true);
+    ASSERT_EQUAL(1u, p.numDocs);
+    ASSERT_EQUAL(6u, p.heapSize);
+    ASSERT_EQUAL(8u, p.arraySize);
+    ASSERT_EQUAL(0.7, p.rankDropLimit);
+    ASSERT_EQUAL(4u, p.offset);
+    ASSERT_EQUAL(4u, p.hits);
+}
+
+TEST("require that match params are set up straight with ranking off array and heap size is 0") {
+    MatchParams p(1, 6, 4, 0.7, 4, 4, true, false);
+    ASSERT_EQUAL(1u, p.numDocs);
+    ASSERT_EQUAL(0u, p.heapSize);
+    ASSERT_EQUAL(0u, p.arraySize);
+    ASSERT_EQUAL(0.7, p.rankDropLimit);
+    ASSERT_EQUAL(4u, p.offset);
+    ASSERT_EQUAL(4u, p.hits);
+}
+
+TEST("require that match phase limiting works") {
+    for (int s = 0; s <= 1; ++s) {
+        for (int i = 0; i <= 6; ++i) {
+            bool enable = (i != 0);
+            bool index_time = (i == 1) || (i == 2) || (i == 5) || (i == 6);
+            bool query_time = (i == 3) || (i == 4) || (i == 5) || (i == 6);
+            bool descending = (i == 2) || (i == 4) || (i == 6);
+            bool use_sorting = (s == 1);
+            size_t want_threads = 75;
+            MyWorld world;
+            world.basicSetup();
+            world.verbose_a1_result("all");
+            if (enable) {
+                if (index_time) {
+                    if (query_time) {
+                        // inject bogus setup to be overridden by query
+                        world.setup_match_phase_limiting("limiter", 10, true);
+                    } else {
+                        world.setup_match_phase_limiting("limiter", 150, descending);
+                    }
+                }
+                world.add_match_phase_limiting_result("limiter", 152, descending, {948, 951, 963, 987, 991, 994, 997});
+            }
+            SearchRequest::SP request = world.createSimpleRequest("a1", "all");
+            if (query_time) {
+                inject_match_phase_limiting(request->propertiesMap.lookupCreate(search::MapNames::RANK), "limiter", 150, descending);
+            }
+            if (use_sorting) {
+                request->sortSpec = "-a1";
+            }
+            SearchReply::UP reply = world.performSearch(request, want_threads);
+            ASSERT_EQUAL(10u, reply->hits.size());
+            if (enable) {
+                EXPECT_EQUAL(79u, reply->totalHitCount);
+                if (!use_sorting) {
+                    EXPECT_EQUAL(997.0, reply->hits[0].metric);
+                    EXPECT_EQUAL(994.0, reply->hits[1].metric);
+                    EXPECT_EQUAL(991.0, reply->hits[2].metric);
+                    EXPECT_EQUAL(987.0, reply->hits[3].metric);
+                    EXPECT_EQUAL(974.0, reply->hits[4].metric);
+                    EXPECT_EQUAL(963.0, reply->hits[5].metric);
+                    EXPECT_EQUAL(961.0, reply->hits[6].metric);
+                    EXPECT_EQUAL(951.0, reply->hits[7].metric);
+                    EXPECT_EQUAL(948.0, reply->hits[8].metric);
+                    EXPECT_EQUAL(935.0, reply->hits[9].metric);
+                }
+            } else {
+                EXPECT_EQUAL(985u, reply->totalHitCount);
+                if (!use_sorting) {
+                    EXPECT_EQUAL(999.0, reply->hits[0].metric);
+                    EXPECT_EQUAL(998.0, reply->hits[1].metric);
+                    EXPECT_EQUAL(997.0, reply->hits[2].metric);
+                    EXPECT_EQUAL(996.0, reply->hits[3].metric);
+                }
+            }
+        }
+    }
+}
+
+TEST("require that arithmetic used for rank drop limit works") {
+    double small = -HUGE_VAL;
+    double limit = -std::numeric_limits<feature_t>::quiet_NaN();
+    EXPECT_TRUE(!(small <= limit));
+}
+
+TEST("require that termwise limit is set correctly for first phase ranking program") {
+    MyWorld world;
+    world.basicSetup();
+    world.basicResults();
+    EXPECT_EQUAL(1.0, world.get_first_phase_termwise_limit());
+    world.set_property(indexproperties::matching::TermwiseLimit::NAME, "0.02");
+    EXPECT_EQUAL(0.02, world.get_first_phase_termwise_limit());
+}
+
+TEST("require that fields are tagged with data type") {
+    MyWorld world;
+    world.basicSetup();
+    auto int32_field = world.get_field_info("a1");
+    auto string_field = world.get_field_info("f1");
+    auto tensor_field = world.get_field_info("tensor_field");
+    auto predicate_field = world.get_field_info("predicate_field");
+    ASSERT_TRUE(bool(int32_field));
+    ASSERT_TRUE(bool(string_field));
+    ASSERT_TRUE(bool(tensor_field));
+    ASSERT_TRUE(bool(predicate_field));
+    EXPECT_EQUAL(int32_field->get_data_type(), FieldInfo::DataType::INT32);
+    EXPECT_EQUAL(string_field->get_data_type(), FieldInfo::DataType::STRING);
+    EXPECT_EQUAL(tensor_field->get_data_type(), FieldInfo::DataType::TENSOR);
+    EXPECT_EQUAL(predicate_field->get_data_type(), FieldInfo::DataType::BOOLEANTREE);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/partial_result/.gitignore b/searchcore/src/tests/proton/matching/partial_result/.gitignore
new file mode 100644
index 00000000000..0284be2ead8
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/partial_result/.gitignore
@@ -0,0 +1 @@
+searchcore_partial_result_test_app
diff --git a/searchcore/src/tests/proton/matching/partial_result/CMakeLists.txt b/searchcore/src/tests/proton/matching/partial_result/CMakeLists.txt
new file mode 100644
index 00000000000..39c1679fc27
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/partial_result/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_partial_result_test_app
+    SOURCES
+    partial_result_test.cpp
+    DEPENDS
+    searchcore_matching
+)
+vespa_add_test(NAME searchcore_partial_result_test_app COMMAND searchcore_partial_result_test_app)
diff --git a/searchcore/src/tests/proton/matching/partial_result/FILES b/searchcore/src/tests/proton/matching/partial_result/FILES
new file mode 100644
index 00000000000..cb7cdbd3bb6
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/partial_result/FILES
@@ -0,0 +1 @@
+partial_result_test.cpp
diff --git a/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp b/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp
new file mode 100644
index 00000000000..48b92c5ae46
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchcore/proton/matching/partial_result.h>
+#include <vespa/vespalib/util/box.h>
+
+using proton::matching::PartialResult;
+using namespace vespalib;
+
+void checkMerge(const std::vector<double> &a, const std::vector<double> &b,
+                size_t maxHits, const std::vector<double> &expect)
+{
+    PartialResult res_a(maxHits, false);
+    PartialResult res_b(maxHits, false);
+    for (size_t i = 0; i < a.size(); ++i) {
+        res_a.add(search::RankedHit(i, a[i]));
+    }
+    res_a.totalHits(a.size());
+    for (size_t i = 0; i < b.size(); ++i) {
+        res_b.add(search::RankedHit(i, b[i]));
+    }
+    res_b.totalHits(b.size());
+    res_a.merge(res_b);
+    EXPECT_EQUAL(a.size() + b.size(), res_a.totalHits());
+    ASSERT_EQUAL(expect.size(), res_a.size());
+    for (size_t i = 0; i < expect.size(); ++i) {
+        EXPECT_EQUAL(expect[i], res_a.hit(i)._rankValue);
+    }
+}
+
+void checkMerge(const std::vector<std::string> &a, const std::vector<std::string> &b,
+                size_t maxHits, const std::vector<std::string> &expect)
+{
+    size_t len = 0;
+    PartialResult res_a(maxHits, true);
+    PartialResult res_b(maxHits, true);
+    len = 0;
+    for (size_t i = 0; i < a.size(); ++i) {
+        len += a[i].size();
+        res_a.add(search::RankedHit(i, 0.0), PartialResult::SortRef(a[i].data(), a[i].size()));
+    }
+    res_a.totalHits(a.size());
+    EXPECT_EQUAL(len, res_a.sortDataSize());
+    len = 0;
+    for (size_t i = 0; i < b.size(); ++i) {
+        len += b[i].size();
+        res_b.add(search::RankedHit(i, 0.0), PartialResult::SortRef(b[i].data(), b[i].size()));
+    }
+    res_b.totalHits(b.size());
+    EXPECT_EQUAL(len, res_b.sortDataSize());
+    res_a.merge(res_b);
+    EXPECT_EQUAL(a.size() + b.size(), res_a.totalHits());
+    ASSERT_EQUAL(expect.size(), res_a.size());
+    len = 0;
+    for (size_t i = 0; i < expect.size(); ++i) {
+        len += expect[i].size();
+        EXPECT_EQUAL(expect[i], std::string(res_a.sortData(i).first, res_a.sortData(i).second));
+    }
+    EXPECT_EQUAL(len, res_a.sortDataSize());
+}
+
+TEST("require that partial results can be created without sort data") {
+    PartialResult res(100, false);
+    EXPECT_EQUAL(0u, res.size());
+    EXPECT_EQUAL(100u, res.maxSize());
+    EXPECT_EQUAL(0u, res.totalHits());
+    EXPECT_FALSE(res.hasSortData());
+    EXPECT_EQUAL(0u, res.sortDataSize());
+    res.add(search::RankedHit(1, 10.0));
+    res.add(search::RankedHit(2, 5.0));
+    res.totalHits(1000);
+    EXPECT_EQUAL(1000u, res.totalHits());
+    ASSERT_EQUAL(2u, res.size());
+    EXPECT_EQUAL(1u, res.hit(0)._docId);
+    EXPECT_EQUAL(10.0, res.hit(0)._rankValue);
+    EXPECT_EQUAL(2u, res.hit(1)._docId);
+    EXPECT_EQUAL(5.0, res.hit(1)._rankValue);
+}
+
+TEST("require that partial results can be created with sort data") {
+    std::string str1("aaa");
+    std::string str2("bbb");
+    PartialResult res(100, true);
+    EXPECT_EQUAL(0u, res.size());
+    EXPECT_EQUAL(100u, res.maxSize());
+    EXPECT_EQUAL(0u, res.totalHits());
+    EXPECT_TRUE(res.hasSortData());
+    EXPECT_EQUAL(0u, res.sortDataSize());
+    res.add(search::RankedHit(1, 10.0), PartialResult::SortRef(str1.data(), str1.size()));
+    res.add(search::RankedHit(2, 5.0), PartialResult::SortRef(str2.data(), str2.size()));
+    res.totalHits(1000);
+    EXPECT_EQUAL(1000u, res.totalHits());
+    ASSERT_EQUAL(2u, res.size());
+    EXPECT_EQUAL(1u, res.hit(0)._docId);
+    EXPECT_EQUAL(10.0, res.hit(0)._rankValue);
+    EXPECT_EQUAL(str1.data(), res.sortData(0).first);
+    EXPECT_EQUAL(str1.size(), res.sortData(0).second);
+    EXPECT_EQUAL(2u, res.hit(1)._docId);
+    EXPECT_EQUAL(5.0, res.hit(1)._rankValue);
+    EXPECT_EQUAL(str2.data(), res.sortData(1).first);
+    EXPECT_EQUAL(str2.size(), res.sortData(1).second);
+}
+
+TEST("require that partial results without sort data are merged correctly") {
+    TEST_DO(checkMerge(make_box(5.0, 4.0, 3.0), make_box(4.5, 3.5), 3, make_box(5.0, 4.5, 4.0)));
+    TEST_DO(checkMerge(make_box(4.5, 3.5), make_box(5.0, 4.0, 3.0), 3, make_box(5.0, 4.5, 4.0)));
+    TEST_DO(checkMerge(make_box(1.0), make_box(2.0), 10, make_box(2.0, 1.0)));
+    TEST_DO(checkMerge(make_box(2.0), make_box(1.0), 10, make_box(2.0, 1.0)));
+    TEST_DO(checkMerge(std::vector<double>(), make_box(1.0), 10, make_box(1.0)));
+    TEST_DO(checkMerge(make_box(1.0), std::vector<double>(), 10, make_box(1.0)));
+    TEST_DO(checkMerge(std::vector<double>(), make_box(1.0), 0, std::vector<double>()));
+    TEST_DO(checkMerge(make_box(1.0), std::vector<double>(), 0, std::vector<double>()));
+    TEST_DO(checkMerge(std::vector<double>(), std::vector<double>(), 10, std::vector<double>()));
+}
+
+TEST("require that partial results with sort data are merged correctly") {
+    TEST_DO(checkMerge(make_box<std::string>("a", "c", "e"), make_box<std::string>("b", "d"), 3, make_box<std::string>("a", "b", "c")));
+    TEST_DO(checkMerge(make_box<std::string>("b", "d"), make_box<std::string>("a", "c", "e"), 3, make_box<std::string>("a", "b", "c")));
+    TEST_DO(checkMerge(make_box<std::string>("a"), make_box<std::string>("aa"), 10, make_box<std::string>("a", "aa")));
+    TEST_DO(checkMerge(make_box<std::string>("aa"), make_box<std::string>("a"), 10, make_box<std::string>("a", "aa")));
+    TEST_DO(checkMerge(std::vector<std::string>(), make_box<std::string>("a"), 10, make_box<std::string>("a")));
+    TEST_DO(checkMerge(make_box<std::string>("a"), std::vector<std::string>(), 10, make_box<std::string>("a")));
+    TEST_DO(checkMerge(std::vector<std::string>(), make_box<std::string>("a"), 0, std::vector<std::string>()));
+    TEST_DO(checkMerge(make_box<std::string>("a"), std::vector<std::string>(), 0, std::vector<std::string>()));
+    TEST_DO(checkMerge(std::vector<std::string>(), std::vector<std::string>(), 10, std::vector<std::string>()));
+}
+
+TEST("require that lower docid is preferred when sorting on rank") {
+    PartialResult res_a(1, false);
+    PartialResult res_b(1, false);
+    PartialResult res_c(1, false);
+    res_a.add(search::RankedHit(2, 1.0));
+    res_b.add(search::RankedHit(3, 1.0));
+    res_c.add(search::RankedHit(1, 1.0));
+    res_a.merge(res_b);
+    ASSERT_EQUAL(1u, res_a.size());
+    EXPECT_EQUAL(2u, res_a.hit(0)._docId);
+    res_a.merge(res_c);
+    ASSERT_EQUAL(1u, res_a.size());
+    EXPECT_EQUAL(1u, res_a.hit(0)._docId);
+}
+
+TEST("require that lower docid is preferred when using sortspec") {
+    std::string foo("foo");
+    PartialResult res_a(1, true);
+    PartialResult res_b(1, true);
+    PartialResult res_c(1, true);
+    res_a.add(search::RankedHit(2, 1.0), PartialResult::SortRef(foo.data(), foo.size()));
+    res_b.add(search::RankedHit(3, 1.0), PartialResult::SortRef(foo.data(), foo.size()));
+    res_c.add(search::RankedHit(1, 1.0), PartialResult::SortRef(foo.data(), foo.size()));
+    res_a.merge(res_b);
+    ASSERT_EQUAL(1u, res_a.size());
+    EXPECT_EQUAL(2u, res_a.hit(0)._docId);
+    res_a.merge(res_c);
+    ASSERT_EQUAL(1u, res_a.size());
+    EXPECT_EQUAL(1u, res_a.hit(0)._docId);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp
new file mode 100644
index 00000000000..caf52a5fca4
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/query_test.cpp
@@ -0,0 +1,900 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for query.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("query_test");
+
+#include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/searchcore/proton/matching/fakesearchcontext.h>
+#include <vespa/searchcore/proton/matching/matchdatareservevisitor.h>
+#include <vespa/searchcore/proton/matching/blueprintbuilder.h>
+#include <vespa/searchcore/proton/matching/query.h>
+#include <vespa/searchcore/proton/matching/querynodes.h>
+#include <vespa/searchcore/proton/matching/resolveviewvisitor.h>
+#include <vespa/searchcore/proton/matching/termdataextractor.h>
+#include <vespa/searchcore/proton/matching/viewresolver.h>
+#include <vespa/searchlib/features/utils.h>
+#include <vespa/searchlib/fef/itermfielddata.h>
+#include <vespa/searchlib/fef/matchdata.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/searchlib/fef/test/indexenvironment.h>
+#include <vespa/searchlib/query/tree/customtypetermvisitor.h>
+#include <vespa/searchlib/query/tree/querybuilder.h>
+#include <vespa/searchlib/query/tree/stackdumpcreator.h>
+#include <vespa/searchlib/query/weight.h>
+#include <vespa/searchlib/queryeval/intermediate_blueprints.h>
+#include <vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h>
+#include <vespa/searchlib/queryeval/leaf_blueprints.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/simpleresult.h>
+#include <vespa/searchlib/queryeval/fake_requestcontext.h>
+#include <vespa/searchlib/queryeval/termasstring.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vector>
+
+using document::PositionDataType;
+using search::fef::CollectionType;
+using search::fef::FieldInfo;
+using search::fef::FieldType;
+using search::fef::ITermData;
+using search::fef::ITermFieldData;
+using search::fef::IllegalHandle;
+using search::fef::MatchData;
+using search::fef::MatchDataLayout;
+using search::fef::TermFieldMatchData;
+using search::fef::TermFieldHandle;
+using search::query::CustomTypeTermVisitor;
+using search::query::Node;
+using search::query::QueryBuilder;
+using search::query::Range;
+using search::query::StackDumpCreator;
+using search::query::Weight;
+using search::queryeval::termAsString;
+using search::queryeval::Blueprint;
+using search::queryeval::FakeResult;
+using search::queryeval::FakeSearchable;
+using search::queryeval::FakeRequestContext;
+using search::queryeval::FieldSpec;
+using search::queryeval::FieldSpecList;
+using search::queryeval::Searchable;
+using search::queryeval::SearchIterator;
+using search::queryeval::SimpleBlueprint;
+using search::queryeval::SimpleResult;
+using search::queryeval::ParallelWeakAndBlueprint;
+using std::string;
+using std::vector;
+namespace fef_test = search::fef::test;
+
+namespace proton {
+namespace matching {
+namespace {
+
+class Test : public vespalib::TestApp {
+    MatchData::UP _match_data;
+    Blueprint::UP _blueprint;
+    FakeRequestContext _requestContext;
+
+    void setUp();
+    void tearDown();
+
+    void requireThatMatchDataIsReserved();
+    void requireThatMatchDataIsReservedForEachFieldInAView();
+    void requireThatTermsAreLookedUp();
+    void requireThatTermsAreLookedUpInMultipleFieldsFromAView();
+    void requireThatAttributeTermsAreLookedUpInAttributeSource();
+    void requireThatAttributeTermDataHandlesAreAllocated();
+    void requireThatTermDataIsFilledIn();
+
+    SearchIterator::UP getIterator(Node &node, ISearchContext &context);
+
+    void requireThatSingleIndexCanUseBlendingAsBlacklisting();
+    void requireThatIteratorsAreBuiltWithBlending();
+    void requireThatIteratorsAreBuiltForAllTermNodes();
+    void requireThatNearIteratorsCanBeBuilt();
+    void requireThatONearIteratorsCanBeBuilt();
+    void requireThatPhraseIteratorsCanBeBuilt();
+
+    void requireThatUnknownFieldActsEmpty();
+    void requireThatIllegalFieldsAreIgnored();
+    void requireThatQueryGluesEverythingTogether();
+    void requireThatQueryAddsLocation();
+    void requireThatQueryAddsLocationCutoff();
+    void requireThatFakeFieldSearchDumpsDiffer();
+    void requireThatNoDocsGiveZeroDocFrequency();
+    void requireThatWeakAndBlueprintsAreCreatedCorrectly();
+    void requireThatParallelWandBlueprintsAreCreatedCorrectly();
+    void requireThatBlackListBlueprintCanBeUsed();
+
+public:
+    int Main();
+};
+
+#define TEST_CALL(func) \
+    TEST_DO(setUp()); \
+    TEST_DO(func()); \
+    TEST_DO(tearDown())
+
+void Test::setUp() {
+    _match_data.reset(0);
+    _blueprint.reset(0);
+}
+
+void Test::tearDown() {
+    _match_data.reset(0);
+    _blueprint.reset(0);
+}
+
+const string field = "field";
+const string resolved_field1 = "resolved1";
+const string resolved_field2 = "resolved2";
+const string unknown_field = "unknown_field";
+const string float_term = "3.14";
+const string int_term = "42";
+const string prefix_term = "foo";
+const string string_term = "bar";
+const uint32_t string_id = 4;
+const Weight string_weight(4);
+const string substring_term = "baz";
+const string suffix_term = "qux";
+const string phrase_term = "quux";
+const Range range_term = Range(32, 47);
+const int doc_count = 100;
+const int field_id = 154;
+const uint32_t term_index = 23;
+const uint32_t term_count = 8;
+
+fef_test::IndexEnvironment plain_index_env;
+fef_test::IndexEnvironment resolved_index_env;
+fef_test::IndexEnvironment attribute_index_env;
+
+void setupIndexEnvironments()
+{
+    FieldInfo field_info(FieldType::INDEX, CollectionType::SINGLE, field, field_id);
+    plain_index_env.getFields().push_back(field_info);
+
+    FieldInfo field_info1(FieldType::INDEX, CollectionType::SINGLE, resolved_field1, field_id);
+    resolved_index_env.getFields().push_back(field_info1);
+    FieldInfo field_info2(FieldType::INDEX, CollectionType::SINGLE, resolved_field2, field_id + 1);
+    resolved_index_env.getFields().push_back(field_info2);
+
+    FieldInfo attr_info(FieldType::ATTRIBUTE, CollectionType::SINGLE, field, 0);
+    attribute_index_env.getFields().push_back(attr_info);
+}
+
+Node::UP buildQueryTree(const ViewResolver &resolver,
+                        const search::fef::IIndexEnvironment &idxEnv)
+{
+    QueryBuilder<ProtonNodeTypes> query_builder;
+    query_builder.addOr(term_count);
+    query_builder.addNumberTerm(float_term, field, 0, Weight(0));
+    query_builder.addNumberTerm(int_term, field, 1, Weight(0));
+    query_builder.addPrefixTerm(prefix_term, field, 2, Weight(0));
+    query_builder.addRangeTerm(range_term, field, 3, Weight(0));
+    query_builder.addStringTerm(string_term, field, string_id, string_weight)
+        .setTermIndex(term_index);
+    query_builder.addSubstringTerm(substring_term, field, 5, Weight(0));
+    query_builder.addSuffixTerm(suffix_term, field, 6, Weight(0));
+    query_builder.addPhrase(2, field, 7, Weight(0));
+    query_builder.addStringTerm(phrase_term, field, 8, Weight(0));
+    query_builder.addStringTerm(phrase_term, field, 9, Weight(0));
+    Node::UP node = query_builder.build();
+
+    ResolveViewVisitor visitor(resolver, idxEnv);
+    node->accept(visitor);
+    return node;
+}
+
+void Test::requireThatMatchDataIsReserved() {
+    Node::UP node = buildQueryTree(ViewResolver(), plain_index_env);
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor visitor(mdl);
+    node->accept(visitor);
+    MatchData::UP match_data = mdl.createMatchData();
+
+    EXPECT_EQUAL(term_count, match_data->getNumTermFields());
+}
+
+ViewResolver getViewResolver() {
+    ViewResolver resolver;
+    resolver.add(field, resolved_field1);
+    resolver.add(field, resolved_field2);
+    return resolver;
+}
+
+void Test::requireThatMatchDataIsReservedForEachFieldInAView() {
+    Node::UP node = buildQueryTree(getViewResolver(), resolved_index_env);
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor visitor(mdl);
+    node->accept(visitor);
+    MatchData::UP match_data = mdl.createMatchData();
+
+    EXPECT_EQUAL(term_count * 2, match_data->getNumTermFields());
+}
+
+class LookupTestCheckerVisitor : public CustomTypeTermVisitor<ProtonNodeTypes>
+{
+    int Main() { return 0; }
+
+public:
+    template <class TermType>
+    void checkNode(const TermType &n, int estimatedHitCount, bool empty) {
+        EXPECT_EQUAL(empty, (estimatedHitCount == 0));
+        EXPECT_EQUAL((double)estimatedHitCount / doc_count, n.field(0).getDocFreq());
+    }
+
+    virtual void visit(ProtonNumberTerm &n) { checkNode(n, 1, false); }
+    virtual void visit(ProtonLocationTerm &n) { checkNode(n, 0, true); }
+    virtual void visit(ProtonPrefixTerm &n) { checkNode(n, 1, false); }
+    virtual void visit(ProtonRangeTerm &n) { checkNode(n, 2, false); }
+    virtual void visit(ProtonStringTerm &n) { checkNode(n, 2, false); }
+    virtual void visit(ProtonSubstringTerm &n) { checkNode(n, 0, true); }
+    virtual void visit(ProtonSuffixTerm &n) { checkNode(n, 2, false); }
+    virtual void visit(ProtonPhrase &n) { checkNode(n, 0, true); }
+    virtual void visit(ProtonWeightedSetTerm &) {}
+    virtual void visit(ProtonDotProduct &) {}
+    virtual void visit(ProtonWandTerm &) {}
+    virtual void visit(ProtonPredicateQuery &) {}
+    virtual void visit(ProtonRegExpTerm &) {}
+};
+
+void Test::requireThatTermsAreLookedUp() {
+    FakeRequestContext requestContext;
+    Node::UP node = buildQueryTree(ViewResolver(), plain_index_env);
+
+    FakeSearchContext context;
+    context.addIdx(1).addIdx(2);
+    context.idx(0).getFake()
+        .addResult(field, prefix_term, FakeResult().doc(1).pos(2))
+        .addResult(field, string_term,
+                   FakeResult().doc(2).pos(3).doc(3).pos(4))
+        .addResult(field, termAsString(int_term),
+                   FakeResult().doc(4).pos(5));
+    context.idx(1).getFake()
+        .addResult(field, string_term, FakeResult().doc(6).pos(7))
+        .addResult(field, suffix_term,
+                   FakeResult().doc(7).pos(8).doc(8).pos(9))
+        .addResult(field, termAsString(float_term),
+                   FakeResult().doc(9).pos(10))
+        .addResult(field, termAsString(int_term),
+                   FakeResult().doc(10).pos(11))
+        .addResult(field, termAsString(range_term),
+                   FakeResult().doc(12).pos(13).doc(13).pos(14));
+    context.setLimit(doc_count + 1);
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor visitor(mdl);
+    node->accept(visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, *node, context);
+
+    LookupTestCheckerVisitor checker;
+    TEST_DO(node->accept(checker));
+}
+
+void Test::requireThatTermsAreLookedUpInMultipleFieldsFromAView() {
+    Node::UP node = buildQueryTree(getViewResolver(), resolved_index_env);
+
+    FakeRequestContext requestContext;
+    FakeSearchContext context;
+    context.addIdx(1).addIdx(2);
+    context.idx(0).getFake()
+        .addResult(resolved_field1, prefix_term,
+                   FakeResult().doc(1).pos(2))
+        .addResult(resolved_field2, string_term,
+                   FakeResult().doc(2).pos(3).doc(3).pos(4))
+        .addResult(resolved_field1, termAsString(int_term),
+                   FakeResult().doc(4).pos(5));
+    context.idx(1).getFake()
+        .addResult(resolved_field1, string_term,
+                   FakeResult().doc(6).pos(7))
+        .addResult(resolved_field2, suffix_term,
+                   FakeResult().doc(7).pos(8).doc(8).pos(9))
+        .addResult(resolved_field1, termAsString(float_term),
+                   FakeResult().doc(9).pos(10))
+        .addResult(resolved_field2, termAsString(int_term),
+                   FakeResult().doc(10).pos(11))
+        .addResult(resolved_field1, termAsString(range_term),
+                   FakeResult().doc(12).pos(13).doc(13).pos(14));
+    context.setLimit(doc_count + 1);
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor visitor(mdl);
+    node->accept(visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, *node, context);
+
+    LookupTestCheckerVisitor checker;
+    TEST_DO(node->accept(checker));
+}
+
+void Test::requireThatAttributeTermsAreLookedUpInAttributeSource() {
+    const string term = "bar";
+    ProtonStringTerm node(term, field, 1, Weight(2));
+    node.resolve(ViewResolver(), attribute_index_env);
+
+    FakeRequestContext requestContext;
+    FakeSearchContext context;
+    context.addIdx(1);
+    context.attr().addResult(field, term, FakeResult().doc(1).pos(2));
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor visitor(mdl);
+    node.accept(visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, node, context);
+
+    EXPECT_TRUE(!blueprint->getState().estimate().empty);
+    EXPECT_EQUAL(1u, blueprint->getState().estimate().estHits);
+}
+
+void Test::requireThatAttributeTermDataHandlesAreAllocated() {
+    const string term = "bar";
+    ProtonStringTerm node(term, field, 1, Weight(2));
+    node.resolve(ViewResolver(), attribute_index_env);
+
+    FakeSearchContext context;
+    FakeRequestContext requestContext;
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor reserve_visitor(mdl);
+    node.accept(reserve_visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, node, context);
+
+    MatchData::UP match_data = mdl.createMatchData();
+
+    EXPECT_EQUAL(1u, match_data->getNumTermFields());
+    EXPECT_TRUE(node.field(0).attribute_field);
+}
+
+
+class SetUpTermDataTestCheckerVisitor
+    : public CustomTypeTermVisitor<ProtonNodeTypes>
+{
+    int Main() { return 0; }
+
+public:
+    virtual void visit(ProtonNumberTerm &) {}
+    virtual void visit(ProtonLocationTerm &) {}
+    virtual void visit(ProtonPrefixTerm &) {}
+    virtual void visit(ProtonRangeTerm &) {}
+
+    virtual void visit(ProtonStringTerm &n) {
+        const ITermData &term_data = n;
+        EXPECT_EQUAL(string_weight.percent(),
+                   term_data.getWeight().percent());
+        EXPECT_EQUAL(1u, term_data.getPhraseLength());
+        EXPECT_EQUAL(-1u, term_data.getTermIndex());
+        EXPECT_EQUAL(string_id, term_data.getUniqueId());
+        EXPECT_EQUAL(term_data.numFields(), n.numFields());
+        for (size_t i = 0; i < term_data.numFields(); ++i) {
+            const ITermFieldData &term_field_data = term_data.field(i);
+            EXPECT_APPROX(2.0 / doc_count, term_field_data.getDocFreq(), 1.0e-6);
+            EXPECT_TRUE(!n.field(i).attribute_field);
+            EXPECT_EQUAL(field_id + i, term_field_data.getFieldId());
+        }
+    }
+
+    virtual void visit(ProtonSubstringTerm &) {}
+    virtual void visit(ProtonSuffixTerm &) {}
+    virtual void visit(ProtonPhrase &n) {
+        const ITermData &term_data = n;
+        EXPECT_EQUAL(2u, term_data.getPhraseLength());
+    }
+    virtual void visit(ProtonWeightedSetTerm &) {}
+    virtual void visit(ProtonDotProduct &) {}
+    virtual void visit(ProtonWandTerm &) {}
+    virtual void visit(ProtonPredicateQuery &) {}
+    virtual void visit(ProtonRegExpTerm &) {}
+};
+
+void Test::requireThatTermDataIsFilledIn() {
+    Node::UP node = buildQueryTree(getViewResolver(), resolved_index_env);
+
+    FakeRequestContext requestContext;
+    FakeSearchContext context;
+    context.addIdx(1);
+    context.idx(0).getFake().addResult(resolved_field1, string_term,
+                                       FakeResult().doc(1).pos(2).doc(5).pos(3));
+    context.setLimit(doc_count + 1);
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor reserve_visitor(mdl);
+    node->accept(reserve_visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, *node, context);
+
+    TEST_DO(
+            SetUpTermDataTestCheckerVisitor checker;
+            node->accept(checker);
+    );
+}
+
+SearchIterator::UP Test::getIterator(Node &node, ISearchContext &context) {
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor mdr_visitor(mdl);
+    node.accept(mdr_visitor);
+    _match_data = mdl.createMatchData();
+
+    _blueprint = BlueprintBuilder::build(_requestContext, node, context);
+
+    _blueprint->fetchPostings(true);
+    SearchIterator::UP search(_blueprint->createSearch(*_match_data, true));
+    search->initFullRange();
+    return search;
+}
+
+FakeIndexSearchable getFakeSearchable(const string &term, int doc1, int doc2) {
+    FakeIndexSearchable source;
+    source.getFake().addResult(field, term,
+                               FakeResult().doc(doc1).pos(2).doc(doc2).pos(3));
+    return source;
+}
+
+void Test::requireThatSingleIndexCanUseBlendingAsBlacklisting() {
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addStringTerm(string_term, field, 1, Weight(2))
+        .resolve(ViewResolver(), plain_index_env);
+    Node::UP node = builder.build();
+    ASSERT_TRUE(node.get());
+
+    FakeSearchContext context;
+    context.addIdx(1).idx(0) = getFakeSearchable(string_term, 2, 5);
+    context.selector().setSource(5, 1);
+
+    SearchIterator::UP iterator = getIterator(*node, context);
+    ASSERT_TRUE(iterator.get());
+    EXPECT_TRUE(!iterator->seek(1));
+    EXPECT_TRUE(!iterator->seek(2));
+    EXPECT_TRUE(iterator->seek(5));
+    iterator->unpack(5);
+}
+
+void Test::requireThatIteratorsAreBuiltWithBlending() {
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addStringTerm(string_term, field, 1, Weight(2))
+        .resolve(ViewResolver(), plain_index_env);
+    Node::UP node = builder.build();
+    ASSERT_TRUE(node.get());
+
+    FakeSearchContext context;
+    context.addIdx(1).idx(0) = getFakeSearchable(string_term, 3, 7);
+    context.addIdx(0).idx(1) = getFakeSearchable(string_term, 2, 6);
+    context.selector().setSource(3, 1);
+    context.selector().setSource(7, 1);
+
+    SearchIterator::UP iterator = getIterator(*node, context);
+    ASSERT_TRUE(iterator.get());
+
+    EXPECT_TRUE(!iterator->seek(1));
+    EXPECT_TRUE(iterator->seek(2));
+    EXPECT_TRUE(iterator->seek(3));
+    EXPECT_TRUE(iterator->seek(6));
+    EXPECT_TRUE(iterator->seek(7));
+}
+
+void Test::requireThatIteratorsAreBuiltForAllTermNodes() {
+    Node::UP node = buildQueryTree(ViewResolver(), plain_index_env);
+    ASSERT_TRUE(node.get());
+
+    FakeSearchContext context(42);
+    context.addIdx(0).idx(0).getFake()
+        .addResult(field, termAsString(float_term),
+                   FakeResult().doc(2).pos(2))
+        .addResult(field, termAsString(int_term),
+                   FakeResult().doc(4).pos(2))
+        .addResult(field, prefix_term, FakeResult().doc(8).pos(2))
+        .addResult(field, termAsString(range_term),
+                   FakeResult().doc(15).pos(2))
+        .addResult(field, string_term, FakeResult().doc(16).pos(2))
+        .addResult(field, substring_term, FakeResult().doc(23).pos(2))
+        .addResult(field, suffix_term, FakeResult().doc(42).pos(2));
+
+    SearchIterator::UP iterator = getIterator(*node, context);
+    ASSERT_TRUE(iterator.get());
+
+    EXPECT_TRUE(!iterator->seek(1));
+    EXPECT_TRUE(iterator->seek(2));
+    EXPECT_TRUE(iterator->seek(4));
+    EXPECT_TRUE(iterator->seek(8));
+    EXPECT_TRUE(iterator->seek(15));
+    EXPECT_TRUE(iterator->seek(16));
+    EXPECT_TRUE(iterator->seek(23));
+    EXPECT_TRUE(iterator->seek(42));
+}
+
+void Test::requireThatNearIteratorsCanBeBuilt() {
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addNear(2, 4);
+    builder.addStringTerm(string_term, field, 1, Weight(2));
+    builder.addStringTerm(prefix_term, field, 1, Weight(2));
+    Node::UP node = builder.build();
+    ResolveViewVisitor resolver(ViewResolver(), plain_index_env);
+    node->accept(resolver);
+    ASSERT_TRUE(node.get());
+
+    FakeSearchContext context(8);
+    context.addIdx(0).idx(0).getFake()
+        .addResult(field, prefix_term, FakeResult()
+                   .doc(4).pos(2).len(50).doc(8).pos(2).len(50))
+        .addResult(field, string_term, FakeResult()
+                   .doc(4).pos(40).len(50).doc(8).pos(5).len(50));
+
+    SearchIterator::UP iterator = getIterator(*node, context);
+    ASSERT_TRUE(iterator.get());
+    EXPECT_TRUE(!iterator->seek(4));
+    EXPECT_TRUE(iterator->seek(8));
+}
+
+void Test::requireThatONearIteratorsCanBeBuilt() {
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addONear(2, 4);
+    builder.addStringTerm(string_term, field, 1, Weight(2));
+    builder.addStringTerm(prefix_term, field, 1, Weight(2));
+    Node::UP node = builder.build();
+    ResolveViewVisitor resolver(ViewResolver(), plain_index_env);
+    node->accept(resolver);
+    ASSERT_TRUE(node.get());
+
+    FakeSearchContext context(8);
+    context.addIdx(0).idx(0).getFake()
+        .addResult(field, string_term, FakeResult()
+                   .doc(4).pos(5).len(50).doc(8).pos(2).len(50))
+        .addResult(field, prefix_term, FakeResult()
+                   .doc(4).pos(2).len(50).doc(8).pos(5).len(50));
+
+    SearchIterator::UP iterator = getIterator(*node, context);
+    ASSERT_TRUE(iterator.get());
+    EXPECT_TRUE(!iterator->seek(4));
+    EXPECT_TRUE(iterator->seek(8));
+}
+
+void Test::requireThatPhraseIteratorsCanBeBuilt() {
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addPhrase(3, field, 0, Weight(42));
+    builder.addStringTerm(string_term, field, 1, Weight(2));
+    builder.addStringTerm(prefix_term, field, 1, Weight(2));
+    builder.addStringTerm(suffix_term, field, 1, Weight(2));
+    Node::UP node = builder.build();
+    ResolveViewVisitor resolver(ViewResolver(), plain_index_env);
+    node->accept(resolver);
+    ASSERT_TRUE(node.get());
+
+    FakeSearchContext context(9);
+    context.addIdx(0).idx(0).getFake()
+        .addResult(field, string_term, FakeResult()
+                  .doc(4).pos(3).len(50)
+                  .doc(5).pos(2).len(50)
+                  .doc(8).pos(2).len(50)
+                  .doc(9).pos(2).len(50))
+        .addResult(field, prefix_term, FakeResult()
+                   .doc(4).pos(2).len(50)
+                   .doc(5).pos(4).len(50)
+                   .doc(8).pos(3).len(50))
+        .addResult(field, suffix_term, FakeResult()
+                   .doc(4).pos(1).len(50)
+                   .doc(5).pos(5).len(50)
+                   .doc(8).pos(4).len(50));
+
+    SearchIterator::UP iterator = getIterator(*node, context);
+    ASSERT_TRUE(iterator.get());
+    EXPECT_TRUE(!iterator->seek(4));
+    EXPECT_TRUE(!iterator->seek(5));
+    EXPECT_TRUE(iterator->seek(8));
+    EXPECT_TRUE(!iterator->seek(9));
+    EXPECT_TRUE(iterator->isAtEnd());
+}
+
+void
+Test::requireThatUnknownFieldActsEmpty()
+{
+    FakeSearchContext context;
+    context.addIdx(0).idx(0).getFake()
+        .addResult(unknown_field, string_term, FakeResult()
+                   .doc(4).pos(3).len(50)
+                   .doc(5).pos(2).len(50));
+
+    ProtonNodeTypes::StringTerm
+        node(string_term, unknown_field, string_id, string_weight);
+    node.resolve(ViewResolver(), plain_index_env);
+
+    std::vector<const ITermData *> terms;
+    TermDataExtractor::extractTerms(node, terms);
+
+    SearchIterator::UP iterator = getIterator(node, context);
+
+    ASSERT_TRUE(EXPECT_EQUAL(1u, terms.size()));
+    EXPECT_EQUAL(0u, terms[0]->numFields());
+
+    ASSERT_TRUE(iterator.get());
+    EXPECT_TRUE(!iterator->seek(1));
+    EXPECT_TRUE(iterator->isAtEnd());
+}
+
+void
+Test::requireThatIllegalFieldsAreIgnored()
+{
+    ProtonNodeTypes::StringTerm
+        node(string_term, unknown_field, string_id, string_weight);
+    node.resolve(ViewResolver(), plain_index_env);
+
+    FakeRequestContext requestContext;
+    FakeSearchContext context;
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor reserve_visitor(mdl);
+    node.accept(reserve_visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, node, context);
+
+    EXPECT_EQUAL(0u, node.numFields());
+
+    MatchData::UP match_data = mdl.createMatchData();
+    EXPECT_EQUAL(0u, match_data->getNumTermFields());
+}
+
+void Test::requireThatQueryGluesEverythingTogether() {
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addStringTerm(string_term, field, 1, Weight(2));
+    string stack_dump = StackDumpCreator::create(*builder.build());
+
+    Query query;
+    query.buildTree(stack_dump, "", ViewResolver(), plain_index_env);
+    vector<const ITermData *> term_data;
+    query.extractTerms(term_data);
+    EXPECT_EQUAL(1u, term_data.size());
+
+    FakeRequestContext requestContext;
+    FakeSearchContext context;
+    context.setLimit(42);
+    MatchDataLayout mdl;
+    query.reserveHandles(requestContext, context, mdl);
+    MatchData::UP md = mdl.createMatchData();
+    EXPECT_EQUAL(1u, md->getNumTermFields());
+
+    query.optimize();
+    query.fetchPostings();
+    SearchIterator::UP search = query.createSearch(*md);
+    ASSERT_TRUE(search.get());
+}
+
+void checkQueryAddsLocation(Test &test, const string &loc_string) {
+    const string loc_field = "location";
+
+    fef_test::IndexEnvironment index_environment;
+    FieldInfo field_info(FieldType::INDEX, CollectionType::SINGLE, field, 0);
+    index_environment.getFields().push_back(field_info);
+    field_info = FieldInfo(FieldType::ATTRIBUTE, CollectionType::SINGLE,
+                           PositionDataType::getZCurveFieldName(loc_field), 1);
+    index_environment.getFields().push_back(field_info);
+
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addStringTerm(string_term, field, 1, Weight(2));
+    string stack_dump = StackDumpCreator::create(*builder.build());
+
+    Query query;
+    query.buildTree(stack_dump,
+                    loc_field + ":" + loc_string,
+                    ViewResolver(), index_environment);
+    vector<const ITermData *> term_data;
+    query.extractTerms(term_data);
+    test.EXPECT_EQUAL(1u, term_data.size());
+
+    FakeRequestContext requestContext;
+    FakeSearchContext context;
+    context.addIdx(0).setLimit(42);
+    MatchDataLayout mdl;
+    query.reserveHandles(requestContext, context, mdl);
+    MatchData::UP md = mdl.createMatchData();
+    test.EXPECT_EQUAL(2u, md->getNumTermFields());
+
+    query.fetchPostings();
+    SearchIterator::UP search = query.createSearch(*md);
+    test.ASSERT_TRUE(search.get());
+    if (!test.EXPECT_NOT_EQUAL(string::npos, search->asString().find(loc_string))) {
+        fprintf(stderr, "search (missing loc_string): %s", search->asString().c_str());
+    }
+}
+
+void Test::requireThatQueryAddsLocation() {
+    checkQueryAddsLocation(*this, "(2,10,10,3,0,1,0,0)");
+}
+
+void Test::requireThatQueryAddsLocationCutoff() {
+    checkQueryAddsLocation(*this, "[2,10,10,20,20]");
+}
+
+void
+Test::requireThatFakeFieldSearchDumpsDiffer()
+{
+    FakeRequestContext requestContext;
+    uint32_t fieldId = 0;
+    MatchDataLayout mdl;
+    TermFieldHandle handle = mdl.allocTermField(fieldId);
+    MatchData::UP match_data = mdl.createMatchData();
+
+    FakeSearchable a;
+    FakeSearchable b;
+    a.tag("a");
+    b.tag("b");
+    ProtonStringTerm n1("term1", "field1", string_id, string_weight);
+    ProtonStringTerm n2("term2", "field1", string_id, string_weight);
+    ProtonStringTerm n3("term1", "field2", string_id, string_weight);
+
+    FieldSpecList fields1;
+    FieldSpecList fields2;
+    fields1.add(FieldSpec("field1", fieldId, handle));
+    fields2.add(FieldSpec("field2", fieldId, handle));
+
+    Blueprint::UP l1(a.createBlueprint(requestContext, fields1, n1)); // reference
+    Blueprint::UP l2(a.createBlueprint(requestContext, fields1, n2)); // term
+    Blueprint::UP l3(a.createBlueprint(requestContext, fields2, n3)); // field
+    Blueprint::UP l4(b.createBlueprint(requestContext, fields1, n1)); // tag
+
+    l1->fetchPostings(true);
+    l2->fetchPostings(true);
+    l3->fetchPostings(true);
+    l4->fetchPostings(true);
+
+    SearchIterator::UP s1(l1->createSearch(*match_data, true));
+    SearchIterator::UP s2(l2->createSearch(*match_data, true));
+    SearchIterator::UP s3(l3->createSearch(*match_data, true));
+    SearchIterator::UP s4(l4->createSearch(*match_data, true));
+
+    EXPECT_NOT_EQUAL(s1->asString(), s2->asString());
+    EXPECT_NOT_EQUAL(s1->asString(), s3->asString());
+    EXPECT_NOT_EQUAL(s1->asString(), s4->asString());
+}
+
+void Test::requireThatNoDocsGiveZeroDocFrequency() {
+    ProtonStringTerm node(string_term, field, string_id, string_weight);
+    node.resolve(ViewResolver(), plain_index_env);
+    FakeSearchContext context;
+    FakeRequestContext requestContext;
+    context.setLimit(0);
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor reserve_visitor(mdl);
+    node.accept(reserve_visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, node, context);
+
+    EXPECT_EQUAL(1u, node.numFields());
+    EXPECT_EQUAL(0.0, node.field(0).getDocFreq());
+}
+
+void Test::requireThatWeakAndBlueprintsAreCreatedCorrectly() {
+    using search::queryeval::WeakAndBlueprint;
+
+    ProtonWeakAnd wand(123, "view");
+    wand.append(Node::UP(new ProtonStringTerm("foo", field, 0, Weight(3))));
+    wand.append(Node::UP(new ProtonStringTerm("bar", field, 0, Weight(7))));
+
+    ResolveViewVisitor resolve_visitor(ViewResolver(), plain_index_env);
+    wand.accept(resolve_visitor);
+
+    FakeRequestContext requestContext;
+    FakeSearchContext context;
+    context.addIdx(0).idx(0).getFake()
+        .addResult(field, "foo", FakeResult().doc(1).doc(3))
+        .addResult(field, "bar", FakeResult().doc(2).doc(3).doc(4));
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor reserve_visitor(mdl);
+    wand.accept(reserve_visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, wand, context);
+    WeakAndBlueprint *wbp = dynamic_cast<WeakAndBlueprint*>(blueprint.get());
+    ASSERT_TRUE(wbp != 0);
+    ASSERT_EQUAL(2u, wbp->getWeights().size());
+    ASSERT_EQUAL(2u, wbp->childCnt());
+    EXPECT_EQUAL(123u, wbp->getN());
+    EXPECT_EQUAL(3u, wbp->getWeights()[0]);
+    EXPECT_EQUAL(7u, wbp->getWeights()[1]);
+    EXPECT_EQUAL(2u, wbp->getChild(0).getState().estimate().estHits);
+    EXPECT_EQUAL(3u, wbp->getChild(1).getState().estimate().estHits);
+}
+
+void Test::requireThatParallelWandBlueprintsAreCreatedCorrectly() {
+    using search::queryeval::WeakAndBlueprint;
+
+    ProtonWandTerm wand(field, 42, Weight(100), 123, 9000, 1.25);
+    wand.append(Node::UP(new ProtonStringTerm("foo", field, 0, Weight(3))));
+    wand.append(Node::UP(new ProtonStringTerm("bar", field, 0, Weight(7))));
+
+    ResolveViewVisitor resolve_visitor(ViewResolver(), attribute_index_env);
+    wand.accept(resolve_visitor);
+
+    FakeRequestContext requestContext;
+    FakeSearchContext context;
+    context.setLimit(1000);
+    context.addIdx(0).idx(0).getFake()
+        .addResult(field, "foo", FakeResult().doc(1).doc(3))
+        .addResult(field, "bar", FakeResult().doc(2).doc(3).doc(4));
+
+    MatchDataLayout mdl;
+    MatchDataReserveVisitor reserve_visitor(mdl);
+    wand.accept(reserve_visitor);
+
+    Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, wand, context);
+    ParallelWeakAndBlueprint *wbp = dynamic_cast<ParallelWeakAndBlueprint*>(blueprint.get());
+    ASSERT_TRUE(wbp != nullptr);
+    EXPECT_EQUAL(9000, wbp->getScoreThreshold());
+    EXPECT_EQUAL(1.25, wbp->getThresholdBoostFactor());
+    EXPECT_EQUAL(1000u, wbp->get_docid_limit());
+}
+
+void
+Test::requireThatBlackListBlueprintCanBeUsed()
+{
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addStringTerm("foo", field, field_id, string_weight);
+    std::string stackDump = StackDumpCreator::create(*builder.build());
+
+    Query query;
+    query.buildTree(stackDump, "", ViewResolver(), plain_index_env);
+
+    FakeSearchContext context(42);
+    context.addIdx(0).idx(0).getFake()
+        .addResult(field, "foo", FakeResult().doc(1).doc(3).doc(5).doc(7).doc(9).doc(11));
+    context.setLimit(42);
+
+    query.setBlackListBlueprint(SimpleBlueprint::UP(new SimpleBlueprint(SimpleResult().addHit(3).addHit(9))));
+
+    FakeRequestContext requestContext;
+    MatchDataLayout mdl;
+    query.reserveHandles(requestContext, context, mdl);
+    MatchData::UP md = mdl.createMatchData();
+
+    query.optimize();
+    query.fetchPostings();
+    SearchIterator::UP search = query.createSearch(*md);
+    SimpleResult exp = SimpleResult().addHit(1).addHit(5).addHit(7).addHit(11);
+    SimpleResult act;
+    act.search(*search);
+    EXPECT_EQUAL(exp, act);
+}
+
+int
+Test::Main()
+{
+    setupIndexEnvironments();
+
+    TEST_INIT("query_test");
+
+    TEST_CALL(requireThatMatchDataIsReserved);
+    TEST_CALL(requireThatMatchDataIsReservedForEachFieldInAView);
+    TEST_CALL(requireThatTermsAreLookedUp);
+    TEST_CALL(requireThatTermsAreLookedUpInMultipleFieldsFromAView);
+    TEST_CALL(requireThatAttributeTermsAreLookedUpInAttributeSource);
+    TEST_CALL(requireThatAttributeTermDataHandlesAreAllocated);
+    TEST_CALL(requireThatTermDataIsFilledIn);
+    TEST_CALL(requireThatSingleIndexCanUseBlendingAsBlacklisting);
+    TEST_CALL(requireThatIteratorsAreBuiltWithBlending);
+    TEST_CALL(requireThatIteratorsAreBuiltForAllTermNodes);
+    TEST_CALL(requireThatNearIteratorsCanBeBuilt);
+    TEST_CALL(requireThatONearIteratorsCanBeBuilt);
+    TEST_CALL(requireThatPhraseIteratorsCanBeBuilt);
+    TEST_CALL(requireThatUnknownFieldActsEmpty);
+    TEST_CALL(requireThatIllegalFieldsAreIgnored);
+    TEST_CALL(requireThatQueryGluesEverythingTogether);
+    TEST_CALL(requireThatQueryAddsLocation);
+    TEST_CALL(requireThatQueryAddsLocationCutoff);
+    TEST_CALL(requireThatFakeFieldSearchDumpsDiffer);
+    TEST_CALL(requireThatNoDocsGiveZeroDocFrequency);
+    TEST_CALL(requireThatWeakAndBlueprintsAreCreatedCorrectly);
+    TEST_CALL(requireThatParallelWandBlueprintsAreCreatedCorrectly);
+    TEST_CALL(requireThatBlackListBlueprintCanBeUsed);
+
+    TEST_DONE();
+}
+
+
+}  // namespace
+}  // namespace matching
+}  // namespace proton
+
+TEST_APPHOOK(proton::matching::Test);
diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp
new file mode 100644
index 00000000000..054b70f9b98
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp
@@ -0,0 +1,486 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for querynodes.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("querynodes_test");
+
+#include <vespa/searchcore/proton/matching/querynodes.h>
+
+#include <vespa/searchcore/proton/matching/fakesearchcontext.h>
+#include <vespa/searchcore/proton/matching/blueprintbuilder.h>
+#include <vespa/searchcore/proton/matching/matchdatareservevisitor.h>
+#include <vespa/searchcore/proton/matching/resolveviewvisitor.h>
+#include <vespa/searchcore/proton/matching/viewresolver.h>
+#include <vespa/searchlib/fef/fieldinfo.h>
+#include <vespa/searchlib/fef/fieldtype.h>
+#include <vespa/searchlib/fef/matchdata.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/searchlib/fef/termfieldmatchdata.h>
+#include <vespa/searchlib/fef/termfieldmatchdataarray.h>
+#include <vespa/searchlib/fef/test/indexenvironment.h>
+#include <vespa/searchlib/query/tree/node.h>
+#include <vespa/searchlib/query/tree/querybuilder.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/queryeval/isourceselector.h>
+#include <vespa/searchlib/queryeval/nearsearch.h>
+#include <vespa/searchlib/queryeval/orsearch.h>
+#include <vespa/searchlib/queryeval/andsearch.h>
+#include <vespa/searchlib/queryeval/andnotsearch.h>
+#include <vespa/searchlib/queryeval/ranksearch.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/simple_phrase_search.h>
+#include <vespa/searchlib/queryeval/sourceblendersearch.h>
+#include <vespa/searchlib/queryeval/fake_search.h>
+#include <vespa/searchlib/queryeval/fake_requestcontext.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <cstdarg>
+#include <string>
+#include <vector>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
+
+using search::fef::CollectionType;
+using search::fef::FieldInfo;
+using search::fef::FieldType;
+using search::fef::MatchData;
+using search::fef::MatchDataLayout;
+using search::fef::TermFieldMatchData;
+using search::fef::TermFieldHandle;
+using search::fef::TermFieldMatchDataArray;
+using search::fef::test::IndexEnvironment;
+using search::query::Node;
+using search::query::QueryBuilder;
+using search::queryeval::ISourceSelector;
+using search::queryeval::NearSearch;
+using search::queryeval::ONearSearch;
+using search::queryeval::OrSearch;
+using search::queryeval::AndSearch;
+using search::queryeval::AndNotSearch;
+using search::queryeval::RankSearch;
+using search::queryeval::Blueprint;
+using search::queryeval::SearchIterator;
+using search::queryeval::SourceBlenderSearch;
+using search::queryeval::FieldSpec;
+using search::queryeval::Searchable;
+using search::queryeval::FakeSearch;
+using search::queryeval::FakeResult;
+using search::queryeval::FakeRequestContext;
+using search::queryeval::SimplePhraseSearch;
+using std::string;
+using std::vector;
+using namespace proton::matching;
+namespace fef_test = search::fef::test;
+
+namespace {
+
+template <typename T> void checkTwoFieldsTwoAttributesTwoIndexes();
+template <typename T> void checkTwoFieldsTwoAttributesOneIndex();
+template <typename T> void checkOneFieldOneAttributeTwoIndexes();
+template <typename T> void checkOneFieldNoAttributesTwoIndexes();
+template <typename T> void checkTwoFieldsNoAttributesTwoIndexes();
+template <typename T> void checkOneFieldNoAttributesOneIndex();
+
+template <typename T> void checkProperBlending();
+template <typename T> void checkProperBlendingWithParent();
+
+const string term = "term";
+const string phrase_term1 = "hello";
+const string phrase_term2 = "world";
+const string view = "view";
+const uint32_t id = 3;
+const search::query::Weight weight(7);
+const string field[] = { "field1", "field2" };
+const string attribute[] = { "attribute1", "attribute2" };
+const string source_tag[] = { "Source 1", "Source 2" };
+const string attribute_tag = "Attribute source";
+const uint32_t distance = 13;
+
+template <class SearchType>
+class Create {
+    bool _strict;
+    typename SearchType::Children _children;
+
+public:
+    explicit Create(bool strict = true) : _strict(strict) {}
+
+    Create &add(SearchIterator *s) {
+        _children.push_back(s);
+        return *this;
+    }
+
+    operator SearchIterator *() const {
+        return SearchType::create(_children, _strict);
+    }
+};
+typedef Create<OrSearch>     MyOr;
+
+class ISourceSelectorDummy : public ISourceSelector
+{
+public:
+    static SourceStore _sourceStoreDummy;
+
+    static Iterator::UP
+    makeDummyIterator()
+    {
+        return Iterator::UP(new Iterator(_sourceStoreDummy));
+    }
+};
+
+ISourceSelector::SourceStore ISourceSelectorDummy::_sourceStoreDummy("foo");
+
+
+typedef uint32_t SourceId;
+class Blender {
+    bool _strict;
+    SourceBlenderSearch::Children _children;
+
+public:
+    explicit Blender(bool strict = true) : _strict(strict) {}
+
+    Blender &add(SourceId source_id, SearchIterator *search) {
+        _children.push_back(SourceBlenderSearch::Child(search, source_id));
+        return *this;
+    }
+
+    operator SearchIterator *() const {
+        return SourceBlenderSearch::create(
+                ISourceSelectorDummy::makeDummyIterator(), _children, _strict);
+    }
+};
+
+SearchIterator *getTerm(const string &trm, const string &fld, const string &tag) {
+    static TermFieldMatchData tmd;
+    TermFieldMatchDataArray tfmda;
+    tfmda.add(&tmd);
+    return new FakeSearch(tag, fld, trm, FakeResult(), tfmda);
+}
+
+class IteratorStructureTest {
+    int _field_count;
+    int _attribute_count;
+    int _index_count;
+
+public:
+    void setFieldCount(int count) { _field_count = count; }
+    void setAttributeCount(int count) { _attribute_count = count; }
+    void setIndexCount(int count) { _index_count = count; }
+
+    string getIteratorAsString(Node &node) {
+        ViewResolver resolver;
+        for (int i = 0; i < _field_count; ++i) {
+            resolver.add(view, field[i]);
+        }
+        for (int i = 0; i < _attribute_count; ++i) {
+            resolver.add(view, attribute[i]);
+        }
+
+        fef_test::IndexEnvironment index_environment;
+        uint32_t fieldId = 0;
+        for (int i = 0; i < _field_count; ++i) {
+            FieldInfo field_info(FieldType::INDEX, CollectionType::SINGLE, field[i], fieldId++);
+            index_environment.getFields().push_back(field_info);
+        }
+        for (int i = 0; i < _attribute_count; ++i) {
+            FieldInfo field_info(FieldType::ATTRIBUTE, CollectionType::SINGLE, attribute[i], fieldId++);
+            index_environment.getFields().push_back(field_info);
+        }
+
+        ResolveViewVisitor resolve_visitor(resolver, index_environment);
+        node.accept(resolve_visitor);
+
+        FakeSearchContext context;
+        context.attr().tag(attribute_tag);
+
+        for (int i = 0; i < _index_count; ++i) {
+            context.addIdx(i).idx(i).getFake().tag(source_tag[i]);
+        }
+
+        MatchDataLayout mdl;
+        FakeRequestContext requestContext;
+        MatchDataReserveVisitor reserve_visitor(mdl);
+        node.accept(reserve_visitor);
+        MatchData::UP match_data = mdl.createMatchData();
+
+        Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, node, context);
+        blueprint->fetchPostings(true);
+        return blueprint->createSearch(*match_data, true)->asString();
+    }
+
+    template <typename Tag> string getIteratorAsString();
+};
+
+typedef QueryBuilder<ProtonNodeTypes> QB;
+struct Phrase {
+    void addToBuilder(QB& b) { b.addPhrase(2, view, id, weight); }
+};
+struct Near   { void addToBuilder(QB& b) { b.addNear(2, distance); } };
+struct ONear  { void addToBuilder(QB& b) { b.addONear(2, distance); } };
+struct Or     { void addToBuilder(QB& b) { b.addOr(2); } };
+struct And    { void addToBuilder(QB& b) { b.addAnd(2); } };
+struct AndNot { void addToBuilder(QB& b) { b.addAndNot(2); } };
+struct Rank   { void addToBuilder(QB& b) { b.addRank(2); } };
+struct Term {};
+
+template <typename Tag>
+string IteratorStructureTest::getIteratorAsString() {
+    QueryBuilder<ProtonNodeTypes> query_builder;
+    Tag().addToBuilder(query_builder);
+    query_builder.addStringTerm(phrase_term1, view, id, weight);
+    query_builder.addStringTerm(phrase_term2, view, id, weight);
+    Node::UP node = query_builder.build();
+    return getIteratorAsString(*node);
+}
+
+template <>
+string IteratorStructureTest::getIteratorAsString<Term>() {
+    ProtonStringTerm node(term, view, id, weight);
+    return getIteratorAsString(node);
+}
+
+template <typename T>
+SearchIterator *getLeaf(const string &fld, const string &tag) {
+    return getTerm(term, fld, tag);
+}
+
+template <>
+SearchIterator *getLeaf<Phrase>(const string &fld, const string &tag) {
+    SimplePhraseSearch::Children children;
+    children.push_back(getTerm(phrase_term1, fld, tag));
+    children.push_back(getTerm(phrase_term2, fld, tag));
+    static TermFieldMatchData tmd;
+    TermFieldMatchDataArray tfmda;
+    tfmda.add(&tmd).add(&tmd);
+    vector<uint32_t> eval_order(2);
+    return new SimplePhraseSearch(children, MatchData::UP(), tfmda, eval_order, tmd, true);
+}
+
+template <typename NearType>
+SearchIterator *getNearParent(SearchIterator *a, SearchIterator *b) {
+    typename NearType::Children children;
+    children.push_back(a);
+    children.push_back(b);
+    TermFieldMatchDataArray data;
+    static TermFieldMatchData tmd;
+    // we only check how many term/field combinations
+    // are below the NearType parent:
+    // two terms searching in (two index fields + two attribute fields)
+    data.add(&tmd).add(&tmd).add(&tmd).add(&tmd)
+        .add(&tmd).add(&tmd).add(&tmd).add(&tmd);
+    return new NearType(children, data, distance, true);
+}
+
+template <typename SearchType>
+SearchIterator *getSimpleParent(SearchIterator *a, SearchIterator *b) {
+    typename SearchType::Children children;
+    children.push_back(a);
+    children.push_back(b);
+    return SearchType::create(children, true);
+}
+
+template <typename T>
+SearchIterator *getParent(SearchIterator *a, SearchIterator *b);
+
+template <>
+SearchIterator *getParent<Near>(SearchIterator *a, SearchIterator *b) {
+    return getNearParent<NearSearch>(a, b);
+}
+
+template <>
+SearchIterator *getParent<ONear>(SearchIterator *a, SearchIterator *b) {
+    return getNearParent<ONearSearch>(a, b);
+}
+
+template <>
+SearchIterator *getParent<Or>(SearchIterator *a, SearchIterator *b) {
+    return getSimpleParent<OrSearch>(a, b);
+}
+
+template <>
+SearchIterator *getParent<And>(SearchIterator *a, SearchIterator *b) {
+    return getSimpleParent<AndSearch>(a, b);
+}
+
+template <>
+SearchIterator *getParent<AndNot>(SearchIterator *a, SearchIterator *b) {
+    return getSimpleParent<AndNotSearch>(a, b);
+}
+
+template <>
+SearchIterator *getParent<Rank>(SearchIterator *a, SearchIterator *b) {
+    return getSimpleParent<RankSearch>(a, b);
+}
+
+template <typename T> bool bothStrict() { return false; }
+
+template <> bool bothStrict<Or>() { return true; }
+
+template <typename T>
+void checkTwoFieldsTwoAttributesTwoIndexes() {
+    IteratorStructureTest structure_test;
+    structure_test.setFieldCount(2);
+    structure_test.setAttributeCount(2);
+    structure_test.setIndexCount(2);
+
+    SearchIterator::UP expected(
+            MyOr()
+            .add(getLeaf<T>(attribute[0], attribute_tag))
+            .add(getLeaf<T>(attribute[1], attribute_tag))
+            .add(Blender()
+                 .add(SourceId(0), MyOr()
+                      .add(getLeaf<T>(field[0], source_tag[0]))
+                      .add(getLeaf<T>(field[1], source_tag[0])))
+                 .add(SourceId(1), MyOr()
+                      .add(getLeaf<T>(field[0], source_tag[1]))
+                      .add(getLeaf<T>(field[1], source_tag[1])))));
+    EXPECT_EQUAL(expected->asString(), structure_test.getIteratorAsString<T>());
+}
+
+template <typename T>
+void checkTwoFieldsTwoAttributesOneIndex() {
+    IteratorStructureTest structure_test;
+    structure_test.setFieldCount(2);
+    structure_test.setAttributeCount(2);
+    structure_test.setIndexCount(1);
+
+    SearchIterator::UP expected(
+            MyOr()
+            .add(getLeaf<T>(attribute[0], attribute_tag))
+            .add(getLeaf<T>(attribute[1], attribute_tag))
+            .add(Blender()
+                 .add(SourceId(0), MyOr()
+                      .add(getLeaf<T>(field[0], source_tag[0]))
+                      .add(getLeaf<T>(field[1], source_tag[0])))));
+    EXPECT_EQUAL(expected->asString(), structure_test.getIteratorAsString<T>());
+}
+
+template <typename T>
+void checkOneFieldOneAttributeTwoIndexes() {
+    IteratorStructureTest structure_test;
+    structure_test.setFieldCount(1);
+    structure_test.setAttributeCount(1);
+    structure_test.setIndexCount(2);
+
+    SearchIterator::UP expected(
+            MyOr()
+            .add(getLeaf<T>(attribute[0], attribute_tag))
+            .add(Blender()
+                 .add(SourceId(0),
+                      getLeaf<T>(field[0], source_tag[0]))
+                 .add(SourceId(1),
+                      getLeaf<T>(field[0], source_tag[1]))));
+    EXPECT_EQUAL(expected->asString(), structure_test.getIteratorAsString<T>());
+}
+
+template <typename T>
+void checkOneFieldNoAttributesTwoIndexes() {
+    IteratorStructureTest structure_test;
+    structure_test.setFieldCount(1);
+    structure_test.setAttributeCount(0);
+    structure_test.setIndexCount(2);
+
+    SearchIterator::UP expected(
+            Blender()
+            .add(SourceId(0), getLeaf<T>(field[0], source_tag[0]))
+            .add(SourceId(1), getLeaf<T>(field[0], source_tag[1])));
+    EXPECT_EQUAL(expected->asString(), structure_test.getIteratorAsString<T>());
+}
+
+template <typename T>
+void checkTwoFieldsNoAttributesTwoIndexes() {
+    IteratorStructureTest structure_test;
+    structure_test.setFieldCount(2);
+    structure_test.setAttributeCount(0);
+    structure_test.setIndexCount(2);
+
+    SearchIterator::UP expected(
+            Blender()
+            .add(SourceId(0), MyOr()
+                 .add(getLeaf<T>(field[0], source_tag[0]))
+                 .add(getLeaf<T>(field[1], source_tag[0])))
+            .add(SourceId(1), MyOr()
+                 .add(getLeaf<T>(field[0], source_tag[1]))
+                 .add(getLeaf<T>(field[1], source_tag[1]))));
+    EXPECT_EQUAL(expected->asString(), structure_test.getIteratorAsString<T>());
+}
+
+template <typename T>
+void checkOneFieldNoAttributesOneIndex() {
+    IteratorStructureTest structure_test;
+    structure_test.setFieldCount(1);
+    structure_test.setAttributeCount(0);
+    structure_test.setIndexCount(1);
+
+    SearchIterator::UP expected(
+            Blender()
+            .add(SourceId(0), getLeaf<T>(field[0], source_tag[0])));
+    EXPECT_EQUAL(expected->asString(), structure_test.getIteratorAsString<T>());
+}
+
+template <typename T>
+void checkProperBlending() {
+    TEST_DO(checkTwoFieldsTwoAttributesTwoIndexes<T>());
+    TEST_DO(checkTwoFieldsTwoAttributesOneIndex<T>());
+    TEST_DO(checkOneFieldOneAttributeTwoIndexes<T>());
+    TEST_DO(checkOneFieldNoAttributesTwoIndexes<T>());
+    TEST_DO(checkTwoFieldsNoAttributesTwoIndexes<T>());
+    TEST_DO(checkOneFieldNoAttributesOneIndex<T>());
+}
+
+template <typename T>
+void checkProperBlendingWithParent() {
+    IteratorStructureTest structure_test;
+    structure_test.setFieldCount(2);
+    structure_test.setAttributeCount(2);
+    structure_test.setIndexCount(2);
+
+    SearchIterator::UP expected(
+            getParent<T>(
+                    MyOr()
+                    .add(getTerm(phrase_term1, attribute[0], attribute_tag))
+                    .add(getTerm(phrase_term1, attribute[1], attribute_tag))
+                    .add(Blender()
+                         .add(SourceId(0), MyOr()
+                              .add(getTerm(phrase_term1, field[0], source_tag[0]))
+                              .add(getTerm(phrase_term1, field[1], source_tag[0])))
+                         .add(SourceId(1), MyOr()
+                              .add(getTerm(phrase_term1, field[0], source_tag[1]))
+                              .add(getTerm(phrase_term1, field[1], source_tag[1])))),
+                    MyOr(bothStrict<T>())
+                    .add(getTerm(phrase_term2, attribute[0], attribute_tag))
+                    .add(getTerm(phrase_term2, attribute[1], attribute_tag))
+                    .add(Blender(bothStrict<T>())
+                         .add(SourceId(0), MyOr(bothStrict<T>())
+                              .add(getTerm(phrase_term2, field[0], source_tag[0]))
+                              .add(getTerm(phrase_term2, field[1], source_tag[0])))
+                         .add(SourceId(1), MyOr(bothStrict<T>())
+                              .add(getTerm(phrase_term2, field[0], source_tag[1]))
+                              .add(getTerm(phrase_term2, field[1], source_tag[1]))))));
+    EXPECT_EQUAL(expected->asString(), structure_test.getIteratorAsString<T>());
+}
+
+TEST("requireThatTermNodeSearchIteratorsGetProperBlending") {
+    TEST_DO(checkProperBlending<Term>());
+}
+
+TEST("requireThatPhrasesGetProperBlending") {
+    TEST_DO(checkProperBlending<Phrase>());
+}
+
+TEST("requireThatNearGetProperBlending") {
+    TEST_DO(checkProperBlendingWithParent<Near>());
+}
+
+TEST("requireThatONearGetProperBlending") {
+    TEST_DO(checkProperBlendingWithParent<ONear>());
+}
+
+TEST("requireThatSimpleIntermediatesGetProperBlending") {
+    TEST_DO(checkProperBlendingWithParent<And>());
+    TEST_DO(checkProperBlendingWithParent<AndNot>());
+    TEST_DO(checkProperBlendingWithParent<Or>());
+    TEST_DO(checkProperBlendingWithParent<Rank>());
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp b/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp
new file mode 100644
index 00000000000..212762389f0
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp
@@ -0,0 +1,142 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for resolveviewvisitor.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("resolveviewvisitor_test");
+
+#include <vespa/searchlib/fef/test/indexenvironment.h>
+#include <vespa/searchcore/proton/matching/querynodes.h>
+#include <vespa/searchcore/proton/matching/resolveviewvisitor.h>
+#include <vespa/searchcore/proton/matching/viewresolver.h>
+#include <vespa/searchlib/query/tree/node.h>
+#include <vespa/searchlib/query/tree/querybuilder.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <string>
+
+namespace fef_test = search::fef::test;
+using search::fef::CollectionType;
+using search::fef::FieldInfo;
+using search::fef::FieldType;
+using search::fef::test::IndexEnvironment;
+using search::query::Node;
+using search::query::QueryBuilder;
+using std::string;
+using namespace proton::matching;
+
+namespace {
+
+const string term = "term";
+const string view = "view";
+const string field1 = "field1";
+const string field2 = "field2";
+const uint32_t id = 1;
+const search::query::Weight weight(2);
+
+ViewResolver getResolver(const string &test_view) {
+    ViewResolver resolver;
+    resolver.add(test_view, field1);
+    resolver.add(test_view, field2);
+    return resolver;
+}
+
+struct Fixture {
+    IndexEnvironment index_environment;
+
+    Fixture() {
+        index_environment.getFields().push_back(FieldInfo(
+                        FieldType::INDEX, CollectionType::SINGLE, field1, 0));
+        index_environment.getFields().push_back(FieldInfo(
+                        FieldType::INDEX, CollectionType::SINGLE, field2, 1));
+    }
+};
+
+TEST_F("requireThatFieldsResolveToThemselves", Fixture) {
+    ViewResolver resolver = getResolver(view);
+
+    QueryBuilder<ProtonNodeTypes> builder;
+    ProtonTermData &base = builder.addStringTerm(term, field1, id, weight);
+    Node::UP node = builder.build();
+
+    ResolveViewVisitor visitor(resolver, f.index_environment);
+    node->accept(visitor);
+
+    EXPECT_EQUAL(1u, base.numFields());
+    EXPECT_EQUAL(field1, base.field(0).field_name);
+}
+
+void checkResolveAlias(const string &view_name, const string &alias,
+                       const Fixture &f) {
+    ViewResolver resolver = getResolver(view_name);
+
+    QueryBuilder<ProtonNodeTypes> builder;
+    ProtonTermData &base = builder.addStringTerm(term, alias, id, weight);
+    Node::UP node = builder.build();
+
+    ResolveViewVisitor visitor(resolver, f.index_environment);
+    node->accept(visitor);
+
+    ASSERT_EQUAL(2u, base.numFields());
+    EXPECT_EQUAL(field1, base.field(0).field_name);
+    EXPECT_EQUAL(field2, base.field(1).field_name);
+}
+
+TEST_F("requireThatViewsCanResolveToMultipleFields", Fixture) {
+    checkResolveAlias(view, view, f);
+}
+
+TEST_F("requireThatEmptyViewResolvesAsDefault", Fixture) {
+    const string default_view = "default";
+    const string empty_view = "";
+    checkResolveAlias(default_view, empty_view, f);
+}
+
+TEST_F("requireThatWeCanForceFilterField", Fixture) {
+    ViewResolver resolver = getResolver(view);
+    f.index_environment.getFields().back().setFilter(true);
+    ResolveViewVisitor visitor(resolver, f.index_environment);
+
+    { // use filter field settings from index environment
+        QueryBuilder<ProtonNodeTypes> builder;
+        ProtonStringTerm &sterm =
+            builder.addStringTerm(term, view, id, weight);
+        Node::UP node = builder.build();
+        node->accept(visitor);
+        ASSERT_EQUAL(2u, sterm.numFields());
+        EXPECT_TRUE(!sterm.field(0).filter_field);
+        EXPECT_TRUE(sterm.field(1).filter_field);
+    }
+    { // force filter on all fields
+        QueryBuilder<ProtonNodeTypes> builder;
+        ProtonStringTerm &sterm =
+            builder.addStringTerm(term, view, id, weight);
+        sterm.setPositionData(false); // force filter
+        Node::UP node = builder.build();
+        node->accept(visitor);
+        ASSERT_EQUAL(2u, sterm.numFields());
+        EXPECT_TRUE(sterm.field(0).filter_field);
+        EXPECT_TRUE(sterm.field(1).filter_field);
+    }
+}
+
+TEST_F("require that equiv nodes resolve view from children", Fixture) {
+    ViewResolver resolver;
+    resolver.add(view, field1);
+
+    QueryBuilder<ProtonNodeTypes> builder;
+    ProtonTermData &base = builder.addEquiv(2, id, weight);
+    builder.addStringTerm(term, view, 42, weight);
+    builder.addStringTerm(term, field2, 43, weight);
+    Node::UP node = builder.build();
+
+    ResolveViewVisitor visitor(resolver, f.index_environment);
+    node->accept(visitor);
+
+    ASSERT_EQUAL(2u, base.numFields());
+    EXPECT_EQUAL(field1, base.field(0).field_name);
+    EXPECT_EQUAL(field2, base.field(1).field_name);
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp
new file mode 100644
index 00000000000..078a6985fc4
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for sessionmanager.
+
+#include <vespa/log/log.h>
+LOG_SETUP("sessionmanager_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
+#include <vespa/searchcore/proton/matching/session_manager_explorer.h>
+#include <vespa/searchcore/proton/matching/search_session.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/data/slime/slime.h>
+
+using vespalib::string;
+using namespace proton;
+using namespace proton::matching;
+using vespalib::StateExplorer;
+
+namespace {
+
+void checkStats(SessionManager::Stats stats, uint32_t numInsert,
+                uint32_t numPick, uint32_t numDropped, uint32_t numCached,
+                uint32_t numTimedout) {
+    EXPECT_EQUAL(numInsert, stats.numInsert);
+    EXPECT_EQUAL(numPick, stats.numPick);
+    EXPECT_EQUAL(numDropped, stats.numDropped);
+    EXPECT_EQUAL(numCached, stats.numCached);
+    EXPECT_EQUAL(numTimedout, stats.numTimedout);
+}
+
+
+TEST("require that SessionManager handles SearchSessions.") {
+    string session_id("foo");
+    fastos::TimeStamp doom(1000);
+    MatchToolsFactory::UP mtf;
+    SearchSession::OwnershipBundle owned_objects;
+    SearchSession::SP session(
+            new SearchSession(session_id, doom, std::move(mtf),
+                              std::move(owned_objects)));
+
+    SessionManager session_manager(10);
+    TEST_DO(checkStats(session_manager.getSearchStats(), 0, 0, 0, 0, 0));
+    session_manager.insert(std::move(session));
+    TEST_DO(checkStats(session_manager.getSearchStats(), 1, 0, 0, 1, 0));
+    session = session_manager.pickSearch(session_id);
+    EXPECT_TRUE(session.get());
+    TEST_DO(checkStats(session_manager.getSearchStats(), 0, 1, 0, 1, 0));
+    session_manager.insert(std::move(session));
+    TEST_DO(checkStats(session_manager.getSearchStats(), 1, 0, 0, 1, 0));
+    session_manager.pruneTimedOutSessions(500);
+    TEST_DO(checkStats(session_manager.getSearchStats(), 0, 0, 0, 1, 0));
+    session_manager.pruneTimedOutSessions(2000);
+    TEST_DO(checkStats(session_manager.getSearchStats(), 0, 0, 0, 0, 1));
+
+    session = session_manager.pickSearch(session_id);
+    EXPECT_FALSE(session.get());
+}
+
+TEST("require that SessionManager can be explored") {
+    fastos::TimeStamp doom(1000);
+    SessionManager session_manager(10);
+    session_manager.insert(SearchSession::SP(new SearchSession("foo", doom,
+        MatchToolsFactory::UP(), SearchSession::OwnershipBundle())));
+    session_manager.insert(SearchSession::SP(new SearchSession("bar", doom,
+        MatchToolsFactory::UP(), SearchSession::OwnershipBundle())));
+    session_manager.insert(SearchSession::SP(new SearchSession("baz", doom,
+        MatchToolsFactory::UP(), SearchSession::OwnershipBundle())));
+    SessionManagerExplorer explorer(session_manager);
+    EXPECT_EQUAL(std::vector<vespalib::string>({"search"}),
+                 explorer.get_children_names());
+    std::unique_ptr<StateExplorer> search = explorer.get_child("search");
+    ASSERT_TRUE(search.get() != nullptr);
+    vespalib::Slime state;
+    vespalib::Slime full_state;
+    search->get_state(vespalib::slime::SlimeInserter(state), false);
+    search->get_state(vespalib::slime::SlimeInserter(full_state), true);
+    EXPECT_EQUAL(3, state.get()["numSessions"].asLong());
+    EXPECT_EQUAL(3, full_state.get()["numSessions"].asLong());
+    EXPECT_EQUAL(0u, state.get()["sessions"].entries());
+    EXPECT_EQUAL(3u, full_state.get()["sessions"].entries());
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp
new file mode 100644
index 00000000000..d61267b7d31
--- /dev/null
+++ b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp
@@ -0,0 +1,167 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for TermDataExtractor.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("termdataextractor_test");
+
+#include <vespa/searchcore/proton/matching/querynodes.h>
+#include <vespa/searchcore/proton/matching/resolveviewvisitor.h>
+#include <vespa/searchcore/proton/matching/termdataextractor.h>
+#include <vespa/searchcore/proton/matching/viewresolver.h>
+#include <vespa/searchlib/fef/tablemanager.h>
+#include <vespa/searchlib/fef/itermdata.h>
+#include <vespa/searchlib/fef/test/indexenvironment.h>
+#include <vespa/searchlib/query/tree/location.h>
+#include <vespa/searchlib/query/tree/point.h>
+#include <vespa/searchlib/query/tree/querybuilder.h>
+#include <vespa/searchlib/query/weight.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <string>
+#include <vector>
+
+namespace fef_test = search::fef::test;
+using search::fef::CollectionType;
+using search::fef::FieldInfo;
+using search::fef::FieldType;
+using search::fef::ITermData;
+using search::fef::IIndexEnvironment;
+using search::query::Location;
+using search::query::Node;
+using search::query::Point;
+using search::query::QueryBuilder;
+using search::query::Range;
+using search::query::Weight;
+using std::string;
+using std::vector;
+using namespace proton::matching;
+
+namespace search { class AttributeManager; }
+
+namespace {
+
+class Test : public vespalib::TestApp {
+    void requireThatTermsAreAdded();
+    void requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm();
+    void requireThatUnrankedTermsAreSkipped();
+    void requireThatNegativeTermsAreSkipped();
+
+public:
+    int Main();
+};
+
+int
+Test::Main()
+{
+    TEST_INIT("termdataextractor_test");
+
+    TEST_DO(requireThatTermsAreAdded());
+    TEST_DO(requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm());
+    TEST_DO(requireThatUnrankedTermsAreSkipped());
+    TEST_DO(requireThatNegativeTermsAreSkipped());
+
+    TEST_DONE();
+}
+
+const string field = "field";
+const uint32_t id[] = { 10, 11, 12, 13, 14, 15, 16, 17, 18 };
+
+Node::UP getQuery(const ViewResolver &resolver)
+{
+    QueryBuilder<ProtonNodeTypes> query_builder;
+    query_builder.addAnd(8);
+    query_builder.addNumberTerm("0.0", field, id[0], Weight(0));
+    query_builder.addPrefixTerm("foo", field, id[1], Weight(0));
+    query_builder.addStringTerm("bar", field, id[2], Weight(0));
+    query_builder.addSubstringTerm("baz", field, id[3], Weight(0));
+    query_builder.addSuffixTerm("qux", field, id[4], Weight(0));
+    query_builder.addRangeTerm(Range(), field, id[5], Weight(0));
+    query_builder.addWeightedSetTerm(1, field, id[6], Weight(0));
+    {
+        // weighted token
+        query_builder.addStringTerm("bar", field, id[3], Weight(0));
+    }
+
+    query_builder.addLocationTerm(Location(Point(10, 10), 3, 0),
+                                  field, id[7], Weight(0));
+    Node::UP node = query_builder.build();
+
+    fef_test::IndexEnvironment index_environment;
+    index_environment.getFields().push_back(FieldInfo(FieldType::INDEX, CollectionType::SINGLE, field, 0));
+    index_environment.getFields().push_back(FieldInfo(FieldType::INDEX, CollectionType::SINGLE, "foo", 1));
+    index_environment.getFields().push_back(FieldInfo(FieldType::INDEX, CollectionType::SINGLE, "bar", 2));
+
+    ResolveViewVisitor visitor(resolver, index_environment);
+    node->accept(visitor);
+
+    return node;
+}
+
+void Test::requireThatTermsAreAdded() {
+    Node::UP node = getQuery(ViewResolver());
+
+    vector<const ITermData *> term_data;
+    TermDataExtractor::extractTerms(*node, term_data);
+    EXPECT_EQUAL(7u, term_data.size());
+    for (int i = 0; i < 7; ++i) {
+        EXPECT_EQUAL(id[i], term_data[i]->getUniqueId());
+        EXPECT_EQUAL(1u, term_data[i]->numFields());
+    }
+}
+
+void Test::requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm() {
+    ViewResolver resolver;
+    resolver.add(field, "foo");
+    resolver.add(field, "bar");
+    Node::UP node = getQuery(resolver);
+
+    vector<const ITermData *> term_data;
+    TermDataExtractor::extractTerms(*node, term_data);
+    EXPECT_EQUAL(7u, term_data.size());
+    for (int i = 0; i < 7; ++i) {
+        EXPECT_EQUAL(id[i], term_data[i]->getUniqueId());
+        EXPECT_EQUAL(2u, term_data[i]->numFields());
+    }
+}
+
+void
+Test::requireThatUnrankedTermsAreSkipped()
+{
+    QueryBuilder<ProtonNodeTypes> query_builder;
+    query_builder.addAnd(2);
+    query_builder.addStringTerm("term1", field, id[0], Weight(0));
+    query_builder.addStringTerm("term2", field, id[1], Weight(0))
+        .setRanked(false);
+    Node::UP node = query_builder.build();
+
+    vector<const ITermData *> term_data;
+    TermDataExtractor::extractTerms(*node, term_data);
+    EXPECT_EQUAL(1u, term_data.size());
+    ASSERT_TRUE(term_data.size() >= 1);
+    EXPECT_EQUAL(id[0], term_data[0]->getUniqueId());
+}
+
+void
+Test::requireThatNegativeTermsAreSkipped()
+{
+    QueryBuilder<ProtonNodeTypes> query_builder;
+    query_builder.addAnd(2);
+    query_builder.addStringTerm("term1", field, id[0], Weight(0));
+    query_builder.addAndNot(2);
+    query_builder.addStringTerm("term2", field, id[1], Weight(0));
+    query_builder.addAndNot(2);
+    query_builder.addStringTerm("term3", field, id[2], Weight(0));
+    query_builder.addStringTerm("term4", field, id[3], Weight(0));
+    Node::UP node = query_builder.build();
+
+    vector<const ITermData *> term_data;
+    TermDataExtractor::extractTerms(*node, term_data);
+    EXPECT_EQUAL(2u, term_data.size());
+    ASSERT_TRUE(term_data.size() >= 2);
+    EXPECT_EQUAL(id[0], term_data[0]->getUniqueId());
+    EXPECT_EQUAL(id[1], term_data[1]->getUniqueId());
+}
+
+}  // namespace
+
+TEST_APPHOOK(Test);
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore
new file mode 100644
index 00000000000..84c97c63aca
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore
@@ -0,0 +1 @@
+searchcore_documentdb_job_trackers_test_app
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt
new file mode 100644
index 00000000000..bf77c583468
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_documentdb_job_trackers_test_app
+    SOURCES
+    documentdb_job_trackers_test.cpp
+    DEPENDS
+    searchcore_proton_metrics
+)
+vespa_add_test(NAME searchcore_documentdb_job_trackers_test_app COMMAND searchcore_documentdb_job_trackers_test_app)
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/DESC b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/DESC
new file mode 100644
index 00000000000..ccd322886ea
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/DESC
@@ -0,0 +1 @@
+documentdb job trackers test. Take a look at documentdb_job_trackers_test.cpp for details.
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/FILES b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/FILES
new file mode 100644
index 00000000000..a63504feca2
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/FILES
@@ -0,0 +1 @@
+documentdb_job_trackers_test.cpp
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
new file mode 100644
index 00000000000..3269fe84dcd
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("documentdb_job_trackers_test");
+
+#include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h>
+#include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h>
+#include <vespa/searchcore/proton/test/dummy_flush_target.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace proton;
+using namespace searchcorespi;
+
+constexpr double EPS = 0.000001;
+
+typedef IFlushTarget::Type FTT;
+typedef IFlushTarget::Component FTC;
+
+struct MFT : public test::DummyFlushTarget
+{
+    MFT(FTT type, FTC component) : test::DummyFlushTarget("", type, component) {}
+};
+
+struct AttributeFlush : public MFT { AttributeFlush() : MFT(FTT::SYNC, FTC::ATTRIBUTE) {} };
+struct MemoryIndexFlush : public MFT { MemoryIndexFlush() : MFT(FTT::FLUSH, FTC::INDEX) {} };
+struct DiskIndexFusion : public MFT { DiskIndexFusion() : MFT(FTT::GC, FTC::INDEX) {} };
+struct DocStoreFlush : public MFT { DocStoreFlush() : MFT(FTT::SYNC, FTC::DOCUMENT_STORE) {} };
+struct DocStoreCompaction : public MFT { DocStoreCompaction() : MFT(FTT::GC, FTC::DOCUMENT_STORE) {} };
+struct OtherFlush : public MFT { OtherFlush() : MFT(FTT::FLUSH, FTC::OTHER) {} };
+
+struct Fixture
+{
+    DocumentDBJobTrackers _trackers;
+    DocumentDBTaggedMetrics::JobMetrics _metrics;
+    Fixture()
+        : _trackers(),
+          _metrics(nullptr)
+    {
+    }
+};
+
+void
+startJobs(IJobTracker &tracker, uint32_t numJobs)
+{
+    for (uint32_t i = 0; i < numJobs; ++i) {
+        tracker.start();
+    }
+}
+
+TEST_F("require that job metrics are updated", Fixture)
+{
+    startJobs(f._trackers.getAttributeFlush(), 1);
+    startJobs(f._trackers.getMemoryIndexFlush(), 2);
+    startJobs(f._trackers.getDiskIndexFusion(), 3);
+    startJobs(f._trackers.getDocumentStoreFlush(), 4);
+    startJobs(f._trackers.getDocumentStoreCompact(), 5);
+    startJobs(*f._trackers.getBucketMove(), 6);
+    startJobs(*f._trackers.getLidSpaceCompact(), 7);
+    startJobs(*f._trackers.getRemovedDocumentsPrune(), 8);
+
+    // Update metrics 2 times to ensure that all jobs are running
+    // in the last interval we actually care about.
+    f._trackers.updateMetrics(f._metrics);
+    FastOS_Thread::Sleep(100);
+    f._trackers.updateMetrics(f._metrics);
+
+    EXPECT_APPROX(1.0, f._metrics.attributeFlush.getLast(), EPS);
+    EXPECT_APPROX(2.0, f._metrics.memoryIndexFlush.getLast(), EPS);
+    EXPECT_APPROX(3.0, f._metrics.diskIndexFusion.getLast(), EPS);
+    EXPECT_APPROX(4.0, f._metrics.documentStoreFlush.getLast(), EPS);
+    EXPECT_APPROX(5.0, f._metrics.documentStoreCompact.getLast(), EPS);
+    EXPECT_APPROX(6.0, f._metrics.bucketMove.getLast(), EPS);
+    EXPECT_APPROX(7.0, f._metrics.lidSpaceCompact.getLast(), EPS);
+    EXPECT_APPROX(8.0, f._metrics.removedDocumentsPrune.getLast(), EPS);
+    EXPECT_APPROX(36.0, f._metrics.total.getLast(), EPS);
+}
+
+bool
+assertFlushTarget(const IJobTracker &tracker, const IFlushTarget &target)
+{
+    const JobTrackedFlushTarget *tracked =
+            dynamic_cast<const JobTrackedFlushTarget *>(&target);
+    if (!EXPECT_TRUE(tracked != nullptr)) return false;
+    if (!EXPECT_EQUAL(&tracker, &tracked->getTracker())) return false;
+    return true;
+}
+
+TEST_F("require that known flush targets are tracked", Fixture)
+{
+    IFlushTarget::List input;
+    input.push_back(IFlushTarget::SP(new AttributeFlush()));
+    input.push_back(IFlushTarget::SP(new MemoryIndexFlush()));
+    input.push_back(IFlushTarget::SP(new DiskIndexFusion()));
+    input.push_back(IFlushTarget::SP(new DocStoreFlush()));
+    input.push_back(IFlushTarget::SP(new DocStoreCompaction()));
+
+    IFlushTarget::List output = f._trackers.trackFlushTargets(input);
+    EXPECT_EQUAL(5u, output.size());
+    EXPECT_TRUE(assertFlushTarget(f._trackers.getAttributeFlush(), *output[0]));
+    EXPECT_TRUE(assertFlushTarget(f._trackers.getMemoryIndexFlush(), *output[1]));
+    EXPECT_TRUE(assertFlushTarget(f._trackers.getDiskIndexFusion(), *output[2]));
+    EXPECT_TRUE(assertFlushTarget(f._trackers.getDocumentStoreFlush(), *output[3]));
+    EXPECT_TRUE(assertFlushTarget(f._trackers.getDocumentStoreCompact(), *output[4]));
+}
+
+TEST_F("require that un-known flush targets are not tracked", Fixture)
+{
+    IFlushTarget::List input;
+    input.push_back(IFlushTarget::SP(new OtherFlush()));
+
+    IFlushTarget::List output = f._trackers.trackFlushTargets(input);
+    EXPECT_EQUAL(1u, output.size());
+    EXPECT_EQUAL(&*output[0].get(), &*input[0]);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore b/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore
new file mode 100644
index 00000000000..2e02ec8191b
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore
@@ -0,0 +1 @@
+searchcore_job_load_sampler_test_app
diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt
new file mode 100644
index 00000000000..478a7201228
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_job_load_sampler_test_app
+    SOURCES
+    job_load_sampler_test.cpp
+    DEPENDS
+    searchcore_proton_metrics
+)
+vespa_add_test(NAME searchcore_job_load_sampler_test_app COMMAND searchcore_job_load_sampler_test_app)
diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/DESC b/searchcore/src/tests/proton/metrics/job_load_sampler/DESC
new file mode 100644
index 00000000000..966bcdf83f6
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_load_sampler/DESC
@@ -0,0 +1 @@
+job load sampler test. Take a look at job_load_sampler_test.cpp for details.
diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/FILES b/searchcore/src/tests/proton/metrics/job_load_sampler/FILES
new file mode 100644
index 00000000000..1112ae6c5da
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_load_sampler/FILES
@@ -0,0 +1 @@
+job_load_sampler_test.cpp
diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp b/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp
new file mode 100644
index 00000000000..b8fa728927f
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("job_load_sampler_test");
+
+#include <vespa/searchcore/proton/metrics/job_load_sampler.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace proton;
+
+constexpr double EPS = 0.000001;
+
+struct Fixture
+{
+    JobLoadSampler _sampler;
+    Fixture()
+        : _sampler(10)
+    {
+    }
+    Fixture &start(double now) {
+        _sampler.startJob(now);
+        return *this;
+    }
+    Fixture &end(double now) {
+        _sampler.endJob(now);
+        return *this;
+    }
+    double sample(double now) {
+        return _sampler.sampleLoad(now);
+    }
+};
+
+TEST_F("require that empty sampler gives 0 load", Fixture)
+{
+    EXPECT_APPROX(0.0, f.sample(11), EPS);
+}
+
+TEST_F("require that empty time interval gives 0 load", Fixture)
+{
+    EXPECT_APPROX(0.0, f.sample(10), EPS);
+}
+
+TEST_F("require that job that starts and ends in interval gets correct load", Fixture)
+{
+    f.start(12).end(17);
+    EXPECT_APPROX(0.5, f.sample(20), EPS);
+    EXPECT_APPROX(0.0, f.sample(21), EPS);
+}
+
+TEST_F("require that job that starts in interval gets correct load", Fixture)
+{
+    f.start(12);
+    EXPECT_APPROX(0.8, f.sample(20), EPS);
+    EXPECT_APPROX(1.0, f.sample(21), EPS);
+}
+
+TEST_F("require that job that ends in interval gets correct load", Fixture)
+{
+    f.start(12).sample(20);
+    f.end(27);
+    EXPECT_APPROX(0.7, f.sample(30), EPS);
+    EXPECT_APPROX(0.0, f.sample(31), EPS);
+}
+
+TEST_F("require that job that runs in complete interval gets correct load", Fixture)
+{
+    f.start(12).sample(20);
+    EXPECT_APPROX(1.0, f.sample(30), EPS);
+    EXPECT_APPROX(1.0, f.sample(31), EPS);
+}
+
+TEST_F("require that multiple jobs that starts and ends in interval gets correct load", Fixture)
+{
+    // job1: 12->17: 0.5
+    // job2: 14->16: 0.2
+    f.start(12).start(14).end(16).end(17);
+    EXPECT_APPROX(0.7, f.sample(20), EPS);
+}
+
+TEST_F("require that multiple jobs that starts and ends in several intervals gets correct load", Fixture)
+{
+    // job1: 12->22
+    // job2: 14->34
+    // job3: 25->45
+    f.start(12).start(14);
+    EXPECT_APPROX(1.4, f.sample(20), EPS);
+    f.end(22).start(25);
+    EXPECT_APPROX(1.7, f.sample(30), EPS);
+    f.end(34);
+    EXPECT_APPROX(1.4, f.sample(40), EPS);
+    f.end(45);
+    EXPECT_APPROX(0.5, f.sample(50), EPS);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore b/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore
new file mode 100644
index 00000000000..85e6097878b
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore
@@ -0,0 +1 @@
+searchcore_job_tracked_flush_test_app
diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt
new file mode 100644
index 00000000000..f4544740f8e
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_job_tracked_flush_test_app
+    SOURCES
+    job_tracked_flush_test.cpp
+    DEPENDS
+    searchcore_proton_metrics
+)
+vespa_add_test(NAME searchcore_job_tracked_flush_test_app COMMAND searchcore_job_tracked_flush_test_app)
diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/DESC b/searchcore/src/tests/proton/metrics/job_tracked_flush/DESC
new file mode 100644
index 00000000000..b62528ff8b4
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/DESC
@@ -0,0 +1,2 @@
+job tracked flush target/task test. Take a look at job_tracked_flush_test.cpp for details.
+
diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/FILES b/searchcore/src/tests/proton/metrics/job_tracked_flush/FILES
new file mode 100644
index 00000000000..09f32789c94
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/FILES
@@ -0,0 +1 @@
+job_tracked_flush_test.cpp
diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp b/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp
new file mode 100644
index 00000000000..cf35ba0b505
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp
@@ -0,0 +1,139 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("job_tracked_flush_test");
+
+#include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h>
+#include <vespa/searchcore/proton/metrics/job_tracked_flush_task.h>
+#include <vespa/searchcore/proton/test/dummy_flush_target.h>
+#include <vespa/searchcore/proton/test/simple_job_tracker.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/sync.h>
+
+using namespace proton;
+using namespace searchcorespi;
+using search::SerialNum;
+using test::SimpleJobTracker;
+using vespalib::makeTask;
+using vespalib::makeClosure;
+using vespalib::CountDownLatch;
+using vespalib::Gate;
+using vespalib::ThreadStackExecutor;
+
+struct MyFlushTask : public searchcorespi::FlushTask
+{
+    Gate &_execGate;
+    MyFlushTask(Gate &execGate) : _execGate(execGate) {}
+
+    // Implements searchcorespi::FlushTask
+    virtual void run() {
+        _execGate.await(5000);
+    }
+    virtual search::SerialNum getFlushSerial() const { return 5; }
+};
+
+struct MyFlushTarget : public test::DummyFlushTarget
+{
+    typedef std::shared_ptr<MyFlushTarget> SP;
+    SerialNum _initFlushSerial;
+    Gate _execGate;
+    Gate _initGate;
+    MyFlushTarget()
+        : test::DummyFlushTarget("mytarget", Type::FLUSH, Component::OTHER),
+          _initFlushSerial(0),
+          _execGate(),
+          _initGate()
+    {}
+
+    // Implements searchcorespi::IFlushTarget
+    virtual FlushTask::UP initFlush(SerialNum currentSerial) {
+        if (currentSerial > 0) {
+            _initFlushSerial = currentSerial;
+            _initGate.await(5000);
+            return FlushTask::UP(new MyFlushTask(_execGate));
+        }
+        return FlushTask::UP();
+    }
+};
+
+struct Fixture
+{
+    SimpleJobTracker::SP _tracker;
+    MyFlushTarget::SP _target;
+    JobTrackedFlushTarget _trackedFlush;
+    FlushTask::UP _task;
+    Gate _taskGate;
+    ThreadStackExecutor _exec;
+    Fixture(uint32_t numJobTrackings = 1)
+        : _tracker(new SimpleJobTracker(numJobTrackings)),
+          _target(new MyFlushTarget()),
+          _trackedFlush(_tracker, _target),
+          _task(),
+          _taskGate(),
+          _exec(1, 64000)
+    {
+    }
+    void initFlush(SerialNum currentSerial) {
+        _task = _trackedFlush.initFlush(currentSerial);
+        _taskGate.countDown();
+    }
+};
+
+constexpr SerialNum FLUSH_SERIAL = 10;
+
+TEST_F("require that flush target name, type and component is preserved", Fixture)
+{
+    EXPECT_EQUAL("mytarget", f._trackedFlush.getName());
+    EXPECT_TRUE(IFlushTarget::Type::FLUSH == f._trackedFlush.getType());
+    EXPECT_TRUE(IFlushTarget::Component::OTHER == f._trackedFlush.getComponent());
+}
+
+TEST_F("require that flush task init is tracked", Fixture)
+{
+    EXPECT_EQUAL(1u, f._tracker->_started.getCount());
+    EXPECT_EQUAL(1u, f._tracker->_ended.getCount());
+
+    f._exec.execute(makeTask(makeClosure(&f, &Fixture::initFlush, FLUSH_SERIAL)));
+    f._tracker->_started.await(5000);
+    EXPECT_EQUAL(0u, f._tracker->_started.getCount());
+    EXPECT_EQUAL(1u, f._tracker->_ended.getCount());
+
+    f._target->_initGate.countDown();
+    f._taskGate.await(5000);
+    EXPECT_EQUAL(0u, f._tracker->_ended.getCount());
+    {
+        JobTrackedFlushTask *trackedTask = dynamic_cast<JobTrackedFlushTask *>(f._task.get());
+        EXPECT_TRUE(trackedTask != nullptr);
+        EXPECT_EQUAL(5u, trackedTask->getFlushSerial());
+    }
+    EXPECT_EQUAL(FLUSH_SERIAL, f._target->_initFlushSerial);
+}
+
+TEST_F("require that flush task execution is tracked", Fixture(2))
+{
+    f._exec.execute(makeTask(makeClosure(&f, &Fixture::initFlush, FLUSH_SERIAL)));
+    f._target->_initGate.countDown();
+    f._taskGate.await(5000);
+
+    EXPECT_EQUAL(1u, f._tracker->_started.getCount());
+    EXPECT_EQUAL(1u, f._tracker->_ended.getCount());
+
+    f._exec.execute(std::move(f._task));
+    f._tracker->_started.await(5000);
+    EXPECT_EQUAL(0u, f._tracker->_started.getCount());
+    EXPECT_EQUAL(1u, f._tracker->_ended.getCount());
+
+    f._target->_execGate.countDown();
+    f._tracker->_ended.await(5000);
+    EXPECT_EQUAL(0u, f._tracker->_ended.getCount());
+}
+
+TEST_F("require that nullptr flush task is not tracked", Fixture)
+{
+    FlushTask::UP task = f._trackedFlush.initFlush(0);
+    EXPECT_TRUE(task.get() == nullptr);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/.gitignore b/searchcore/src/tests/proton/metrics/metrics_engine/.gitignore
new file mode 100644
index 00000000000..98ae77cb458
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/metrics_engine/.gitignore
@@ -0,0 +1 @@
+searchcore_metrics_engine_test_app
diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt b/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt
new file mode 100644
index 00000000000..e50e584e578
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_metrics_engine_test_app
+    SOURCES
+    metrics_engine_test.cpp
+    DEPENDS
+    searchcore_flushengine
+    searchcore_proton_metrics
+)
+vespa_add_test(NAME searchcore_metrics_engine_test_app COMMAND searchcore_metrics_engine_test_app)
diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/DESC b/searchcore/src/tests/proton/metrics/metrics_engine/DESC
new file mode 100644
index 00000000000..2efe31d45d3
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/metrics_engine/DESC
@@ -0,0 +1 @@
+metrics engine test. Take a look at metrics_engine_test.cpp for details.
diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/FILES b/searchcore/src/tests/proton/metrics/metrics_engine/FILES
new file mode 100644
index 00000000000..ac033a53070
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/metrics_engine/FILES
@@ -0,0 +1 @@
+metrics_engine_test.cpp
diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp
new file mode 100644
index 00000000000..a70ce5a5333
--- /dev/null
+++ b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for metrics_engine.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("metrics_engine_test");
+
+#include <vespa/searchcore/proton/metrics/metrics_engine.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace proton;
+
+namespace {
+
+TEST("require that the metric proton.diskusage is the sum of the documentDB "
+     "diskusage metrics.") {
+    MetricsEngine metrics_engine;
+
+    DocumentDBMetricsCollection metrics1("type1", 1);
+    DocumentDBMetricsCollection metrics2("type2", 1);
+    metrics1.getMetrics().index.diskUsage.addValue(100);
+    metrics2.getMetrics().index.diskUsage.addValue(1000);
+
+    metrics_engine.addDocumentDBMetrics(metrics1);
+    metrics_engine.addDocumentDBMetrics(metrics2);
+
+    EXPECT_EQUAL(1100, metrics_engine.legacyRoot().diskUsage.getLongValue("value"));
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/persistenceconformance/.gitignore b/searchcore/src/tests/proton/persistenceconformance/.gitignore
new file mode 100644
index 00000000000..9b6330d1531
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceconformance/.gitignore
@@ -0,0 +1 @@
+/vlog.txt
diff --git a/searchcore/src/tests/proton/persistenceconformance/CMakeLists.txt b/searchcore/src/tests/proton/persistenceconformance/CMakeLists.txt
new file mode 100644
index 00000000000..f71dbf3c1ba
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceconformance/CMakeLists.txt
@@ -0,0 +1,6 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_test(
+    NAME searchcore_persistenceconformance_test_app
+    COMMAND ../../../apps/tests/searchcore_persistenceconformance_test_app
+    ENVIRONMENT "VESPA_LOG_TARGET=file:vlog.txt"
+)
diff --git a/searchcore/src/tests/proton/persistenceconformance/DESC b/searchcore/src/tests/proton/persistenceconformance/DESC
new file mode 100644
index 00000000000..98392d5a316
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceconformance/DESC
@@ -0,0 +1 @@
+Persistence provider conformance test for proton integration. Take a look at persistenceconformance_test.cpp for details.
diff --git a/searchcore/src/tests/proton/persistenceconformance/FILES b/searchcore/src/tests/proton/persistenceconformance/FILES
new file mode 100644
index 00000000000..6912ecd1d9b
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceconformance/FILES
@@ -0,0 +1 @@
+persistenceconformance_test.cpp
diff --git a/searchcore/src/tests/proton/persistenceengine/.gitignore b/searchcore/src/tests/proton/persistenceengine/.gitignore
new file mode 100644
index 00000000000..93d1e27e9c6
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceengine/.gitignore
@@ -0,0 +1 @@
+searchcore_persistenceengine_test_app
diff --git a/searchcore/src/tests/proton/persistenceengine/CMakeLists.txt b/searchcore/src/tests/proton/persistenceengine/CMakeLists.txt
new file mode 100644
index 00000000000..b5e42ac5075
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceengine/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_persistenceengine_test_app
+    SOURCES
+    persistenceengine_test.cpp
+    DEPENDS
+    searchcore_persistenceengine
+    searchcore_pcommon
+    searchcore_proton_metrics
+)
+vespa_add_test(NAME searchcore_persistenceengine_test_app COMMAND searchcore_persistenceengine_test_app)
diff --git a/searchcore/src/tests/proton/persistenceengine/DESC b/searchcore/src/tests/proton/persistenceengine/DESC
new file mode 100644
index 00000000000..ec363711b48
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceengine/DESC
@@ -0,0 +1 @@
+persistenceengine test. Take a look at persistenceengine_test.cpp for details.
diff --git a/searchcore/src/tests/proton/persistenceengine/FILES b/searchcore/src/tests/proton/persistenceengine/FILES
new file mode 100644
index 00000000000..12d47ca9632
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceengine/FILES
@@ -0,0 +1 @@
+persistenceengine_test.cpp
diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
new file mode 100644
index 00000000000..4346f7d43c1
--- /dev/null
+++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
@@ -0,0 +1,828 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("persistenceengine_test");
+
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/persistence/spi/documentselection.h>
+#include <vespa/searchcore/proton/persistenceengine/bucket_guard.h>
+#include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h>
+#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/document/fieldset/fieldsets.h>
+#include <set>
+
+using document::BucketId;
+using document::Document;
+using document::DocumentId;
+using document::DocumentType;
+using search::DocumentMetaData;
+using storage::spi::BucketChecksum;
+using storage::spi::BucketInfo;
+using storage::spi::ClusterState;
+using storage::spi::DocumentSelection;
+using storage::spi::GetResult;
+using namespace proton;
+using namespace vespalib;
+
+DocumentType
+createDocType(const vespalib::string &name, int32_t id)
+{
+    return DocumentType(name, id);
+}
+
+
+document::Document::SP
+createDoc(const DocumentType &docType, const DocumentId &docId)
+{
+    return document::Document::SP(new document::Document(docType, docId));
+}
+
+
+document::DocumentUpdate::SP
+createUpd(const DocumentType& docType, const DocumentId &docId)
+{
+    return document::DocumentUpdate::SP(new document::DocumentUpdate(docType, docId));
+}
+
+
+document::Document::UP
+clone(const document::Document::SP &doc)
+{
+    return document::Document::UP(doc->clone());
+}
+
+
+document::DocumentUpdate::UP
+clone(const document::DocumentUpdate::SP &upd)
+{
+    return document::DocumentUpdate::UP(upd->clone());
+}
+
+
+storage::spi::ClusterState
+createClusterState(const storage::lib::State& nodeState =
+                   storage::lib::State::UP)
+{
+    using storage::lib::Distribution;
+    using storage::lib::Node;
+    using storage::lib::NodeState;
+    using storage::lib::NodeType;
+    using storage::lib::State;
+    using vespa::config::content::StorDistributionConfigBuilder;
+    typedef StorDistributionConfigBuilder::Group Group;
+    typedef Group::Nodes Nodes;
+    storage::lib::ClusterState cstate;
+    StorDistributionConfigBuilder dc;
+
+    cstate.setNodeState(Node(NodeType::STORAGE, 0),
+                        NodeState(NodeType::STORAGE,
+                                  nodeState,
+                                  "dummy desc",
+                                  1.0,
+                                  1));
+    cstate.setClusterState(State::UP);
+    dc.redundancy = 1;
+    dc.readyCopies = 1;
+    dc.group.push_back(Group());
+    Group &g(dc.group[0]);
+    g.index = "invalid";
+    g.name = "invalid";
+    g.capacity = 1.0;
+    g.partitions = "";
+    g.nodes.push_back(Nodes());
+    Nodes &n(g.nodes[0]);
+    n.index = 0;
+    Distribution dist(dc);
+    return ClusterState(cstate, 0, dist);
+}
+
+
+struct MyDocumentRetriever : DocumentRetrieverBaseForTest {
+    document::DocumentTypeRepo repo;
+    const Document *document;
+    Timestamp timestamp;
+    DocumentId &last_doc_id;
+
+    MyDocumentRetriever(const Document *d, Timestamp ts, DocumentId &last_id)
+        : repo(), document(d), timestamp(ts), last_doc_id(last_id) {}
+    virtual const document::DocumentTypeRepo &getDocumentTypeRepo() const {
+        return repo;
+    }
+    virtual void getBucketMetaData(const storage::spi::Bucket &,
+                                   search::DocumentMetaData::Vector &v) const {
+        if (document != 0) {
+            v.push_back(getDocumentMetaData(document->getId()));
+        }
+    }
+    virtual DocumentMetaData getDocumentMetaData(const DocumentId &id) const {
+        last_doc_id = id;
+        if (document != 0) {
+            return DocumentMetaData(1, timestamp, document::BucketId(1),
+                                    document->getId().getGlobalId());
+        }
+        return DocumentMetaData();
+    }
+    virtual document::Document::UP getDocument(search::DocumentIdT) const {
+        if (document != 0) {
+            return Document::UP(document->clone());
+        }
+        return Document::UP();
+    }
+
+    virtual CachedSelect::SP
+    parseSelect(const vespalib::string &) const
+    {
+        return CachedSelect::SP();
+    }
+};
+
+struct MyHandler : public IPersistenceHandler, IBucketFreezer {
+    bool                         initialized;
+    Bucket                       lastBucket;
+    Timestamp                    lastTimestamp;
+    DocumentId                   lastDocId;
+    Timestamp                    existingTimestamp;
+    const ClusterState*          lastCalc;
+    storage::spi::BucketInfo::ActiveState lastBucketState;
+    BucketIdListResult::List     bucketList;
+    Result                       bucketStateResult;
+    BucketInfo                   bucketInfo;
+    Result                       deleteBucketResult;
+    BucketIdListResult::List     modBucketList;
+    Result                       _splitResult;
+    Result                       _joinResult;
+    Result                       _createBucketResult;
+    const Document *document;
+    std::multiset<uint64_t> frozen;
+    std::multiset<uint64_t> was_frozen;
+
+    MyHandler()
+        : initialized(false),
+          lastBucket(),
+          lastTimestamp(),
+          lastDocId(),
+          existingTimestamp(),
+          lastCalc(NULL),
+          lastBucketState(),
+          bucketList(),
+          bucketStateResult(),
+          bucketInfo(),
+          deleteBucketResult(),
+          modBucketList(),
+          _splitResult(),
+          _joinResult(),
+          _createBucketResult(),
+          document(0),
+          frozen(),
+          was_frozen()
+    {
+    }
+
+    void setExistingTimestamp(Timestamp ts) {
+        existingTimestamp = ts;
+    }
+    void setDocument(const Document &doc, Timestamp ts) {
+        document = &doc;
+        setExistingTimestamp(ts);
+    }
+    void handle(FeedToken token, const Bucket &bucket, Timestamp timestamp, const DocumentId &docId) {
+        lastBucket = bucket;
+        lastTimestamp = timestamp;
+        lastDocId = docId;
+        token.ack();
+    }
+
+    virtual void initialize() { initialized = true; }
+
+    virtual void handlePut(FeedToken token, const Bucket& bucket,
+                           Timestamp timestamp, const document::Document::SP& doc) {
+        token.setResult(ResultUP(new storage::spi::Result()), false);
+        handle(token, bucket, timestamp, doc->getId());
+    }
+
+    virtual void handleUpdate(FeedToken token, const Bucket& bucket,
+                              Timestamp timestamp, const document::DocumentUpdate::SP& upd) {
+        token.setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)),
+                        existingTimestamp > 0);
+        handle(token, bucket, timestamp, upd->getId());
+    }
+
+    virtual void handleRemove(FeedToken token, const Bucket& bucket,
+                              Timestamp timestamp, const DocumentId& id) {
+        bool wasFound = existingTimestamp > 0;
+        token.setResult(ResultUP(new storage::spi::RemoveResult(wasFound)), wasFound);
+        handle(token, bucket, timestamp, id);
+    }
+
+    virtual void handleListBuckets(IBucketIdListResultHandler &resultHandler) {
+        resultHandler.handle(BucketIdListResult(bucketList));
+    }
+
+    virtual void handleSetClusterState(const ClusterState &calc,
+                                       IGenericResultHandler &resultHandler) {
+        lastCalc = &calc;
+        resultHandler.handle(Result());
+    }
+
+    virtual void handleSetActiveState(const Bucket &bucket,
+                                       storage::spi::BucketInfo::ActiveState newState,
+                                       IGenericResultHandler &resultHandler) {
+        lastBucket = bucket;
+        lastBucketState = newState;
+        resultHandler.handle(bucketStateResult);
+    }
+
+    virtual void handleGetBucketInfo(const Bucket &,
+                                     IBucketInfoResultHandler &resultHandler) {
+        resultHandler.handle(BucketInfoResult(bucketInfo));
+    }
+
+    virtual void
+    handleCreateBucket(FeedToken token,
+                       const storage::spi::Bucket &)
+    {
+        token.setResult(ResultUP(new Result(_createBucketResult)), true);
+        token.ack();
+    }
+
+    virtual void handleDeleteBucket(FeedToken token,
+                                    const storage::spi::Bucket &) {
+        token.setResult(ResultUP(new Result(deleteBucketResult)), true);
+        token.ack();
+    }
+
+    virtual void handleGetModifiedBuckets(IBucketIdListResultHandler &resultHandler) {
+        resultHandler.handle(BucketIdListResult(modBucketList));
+    }
+
+    virtual void
+    handleSplit(FeedToken token,
+                const storage::spi::Bucket &source,
+                const storage::spi::Bucket &target1,
+                const storage::spi::Bucket &target2)
+    {
+        (void) source;
+        (void) target1;
+        (void) target2;
+        token.setResult(ResultUP(new Result(_splitResult)), true);
+        token.ack();
+    }
+
+    virtual void
+    handleJoin(FeedToken token,
+               const storage::spi::Bucket &source1,
+               const storage::spi::Bucket &source2,
+               const storage::spi::Bucket &target)
+    {
+        (void) source1;
+        (void) source2;
+        (void) target;
+        token.setResult(ResultUP(new Result(_joinResult)), true);
+        token.ack();
+    }
+
+    virtual RetrieversSP getDocumentRetrievers() {
+        RetrieversSP ret(new std::vector<IDocumentRetriever::SP>);
+        ret->push_back(IDocumentRetriever::SP(new MyDocumentRetriever(
+                                0, Timestamp(), lastDocId)));
+        ret->push_back(IDocumentRetriever::SP(new MyDocumentRetriever(
+                                document, existingTimestamp, lastDocId)));
+        return ret;
+    }
+
+    virtual BucketGuard::UP lockBucket(const storage::spi::Bucket &b) {
+        return BucketGuard::UP(new BucketGuard(b.getBucketId(), *this));
+    }
+
+    virtual void
+    handleListActiveBuckets(IBucketIdListResultHandler &resultHandler)
+    {
+        BucketIdListResult::List list;
+        resultHandler.handle(BucketIdListResult(list));
+    }
+
+    virtual void
+    handlePopulateActiveBuckets(document::BucketId::List &buckets,
+                                IGenericResultHandler &resultHandler)
+    {
+        (void) buckets;
+        resultHandler.handle(Result());
+    }
+
+    virtual void freezeBucket(BucketId bucket) {
+        frozen.insert(bucket.getId());
+        was_frozen.insert(bucket.getId());
+    }
+    virtual void thawBucket(BucketId bucket) {
+        std::multiset<uint64_t>::iterator it = frozen.find(bucket.getId());
+        ASSERT_TRUE(it != frozen.end());
+        frozen.erase(it);
+    }
+    bool isFrozen(const Bucket &bucket) {
+        return frozen.find(bucket.getBucketId().getId()) != frozen.end();
+    }
+    bool wasFrozen(const Bucket &bucket) {
+        return was_frozen.find(bucket.getBucketId().getId())
+            != was_frozen.end();
+    }
+};
+
+
+struct HandlerSet {
+    IPersistenceHandler::SP phandler1;
+    IPersistenceHandler::SP phandler2;
+    MyHandler              &handler1;
+    MyHandler              &handler2;
+    HandlerSet() :
+        phandler1(new MyHandler()),
+        phandler2(new MyHandler()),
+        handler1(static_cast<MyHandler &>(*phandler1.get())),
+        handler2(static_cast<MyHandler &>(*phandler2.get()))
+    {}
+};
+
+
+DocumentType type1(createDocType("type1", 1));
+DocumentType type2(createDocType("type2", 2));
+DocumentType type3(createDocType("type3", 3));
+DocumentId docId0;
+DocumentId docId1("id:type1:type1::1");
+DocumentId docId2("id:type2:type2::1");
+DocumentId docId3("id:type3:type3::1");
+Document::SP doc1(createDoc(type1, docId1));
+Document::SP doc2(createDoc(type2, docId2));
+Document::SP doc3(createDoc(type3, docId3));
+Document::SP old_doc(createDoc(type1, DocumentId("doc:old:id-scheme")));
+document::DocumentUpdate::SP upd1(createUpd(type1, docId1));
+document::DocumentUpdate::SP upd2(createUpd(type2, docId2));
+document::DocumentUpdate::SP upd3(createUpd(type3, docId3));
+PartitionId partId(0);
+BucketId bckId1(1);
+BucketId bckId2(2);
+BucketId bckId3(3);
+Bucket bucket0;
+Bucket bucket1(bckId1, partId);
+Bucket bucket2(bckId2, partId);
+BucketChecksum checksum1(1);
+BucketChecksum checksum2(2);
+BucketChecksum checksum3(1+2);
+BucketInfo bucketInfo1(checksum1, 1, 0, 1, 0);
+BucketInfo bucketInfo2(checksum2, 2, 0, 2, 0);
+BucketInfo bucketInfo3(checksum3, 3, 0, 3, 0);
+Timestamp tstamp0;
+Timestamp tstamp1(1);
+Timestamp tstamp2(2);
+Timestamp tstamp3(3);
+DocumentSelection doc_sel("");
+Selection selection(doc_sel);
+
+
+class SimplePersistenceEngineOwner : public IPersistenceEngineOwner
+{
+    virtual void
+    setClusterState(const storage::spi::ClusterState &calc)
+    {
+        (void) calc;
+    }
+};
+
+struct SimpleResourceWriteFilter : public IResourceWriteFilter
+{
+    bool _acceptWriteOperation;
+    vespalib::string _message;
+    SimpleResourceWriteFilter()
+        : _acceptWriteOperation(true),
+          _message()
+    {}
+
+    virtual bool acceptWriteOperation() const override { return _acceptWriteOperation; }
+    virtual State getAcceptState() const override {
+        return IResourceWriteFilter::State(acceptWriteOperation(), _message);
+    }
+};
+
+
+struct SimpleFixture {
+    SimplePersistenceEngineOwner _owner;
+    SimpleResourceWriteFilter _writeFilter;
+    PersistenceEngine engine;
+    HandlerSet hset;
+    SimpleFixture()
+        : _owner(),
+          engine(_owner, _writeFilter, -1, false),
+          hset()
+    {
+        engine.putHandler(DocTypeName(doc1->getType()), hset.phandler1);
+        engine.putHandler(DocTypeName(doc2->getType()), hset.phandler2);
+    }
+};
+
+
+void
+assertHandler(const Bucket &expBucket, Timestamp expTimestamp,
+              const DocumentId &expDocId, const MyHandler &handler)
+{
+    EXPECT_EQUAL(expBucket, handler.lastBucket);
+    EXPECT_EQUAL(expTimestamp, handler.lastTimestamp);
+    EXPECT_EQUAL(expDocId, handler.lastDocId);
+}
+
+
+TEST_F("require that getPartitionStates() prepares all handlers", SimpleFixture)
+{
+    EXPECT_FALSE(f.hset.handler1.initialized);
+    EXPECT_FALSE(f.hset.handler2.initialized);
+    f.engine.initialize();
+    EXPECT_TRUE(f.hset.handler1.initialized);
+    EXPECT_TRUE(f.hset.handler2.initialized);
+}
+
+
+TEST_F("require that puts are routed to handler", SimpleFixture)
+{
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    f.engine.put(bucket1, tstamp1, doc1, context);
+    assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
+    assertHandler(bucket0, tstamp0, docId0, f.hset.handler2);
+
+    f.engine.put(bucket1, tstamp1, doc2, context);
+    assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
+    assertHandler(bucket1, tstamp1, docId2, f.hset.handler2);
+
+    EXPECT_EQUAL(
+            Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"),
+            f.engine.put(bucket1, tstamp1, doc3, context));
+}
+
+
+TEST_F("require that puts with old id scheme are rejected", SimpleFixture) {
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    EXPECT_EQUAL(
+            Result(Result::PERMANENT_ERROR, "Old id scheme not supported in "
+                   "elastic mode (doc:old:id-scheme)"),
+            f.engine.put(bucket1, tstamp1, old_doc, context));
+}
+
+
+TEST_F("require that put is rejected if resource limit is reached", SimpleFixture)
+{
+    f._writeFilter._acceptWriteOperation = false;
+    f._writeFilter._message = "Disk is full";
+
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    EXPECT_EQUAL(
+            Result(Result::RESOURCE_EXHAUSTED,
+                   "Put operation rejected for document 'doc:old:id-scheme': 'Disk is full'"),
+            f.engine.put(bucket1, tstamp1, old_doc, context));
+}
+
+
+TEST_F("require that updates are routed to handler", SimpleFixture)
+{
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    f.hset.handler1.setExistingTimestamp(tstamp2);
+    UpdateResult ur = f.engine.update(bucket1, tstamp1, upd1, context);
+    assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
+    assertHandler(bucket0, tstamp0, docId0, f.hset.handler2);
+    EXPECT_EQUAL(tstamp2, ur.getExistingTimestamp());
+
+    f.hset.handler2.setExistingTimestamp(tstamp3);
+    ur = f.engine.update(bucket1, tstamp1, upd2, context);
+    assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
+    assertHandler(bucket1, tstamp1, docId2, f.hset.handler2);
+    EXPECT_EQUAL(tstamp3, ur.getExistingTimestamp());
+
+    EXPECT_EQUAL(
+            Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"),
+            f.engine.update(bucket1, tstamp1, upd3, context));
+}
+
+
+TEST_F("require that update is rejected if resource limit is reached", SimpleFixture)
+{
+    f._writeFilter._acceptWriteOperation = false;
+    f._writeFilter._message = "Disk is full";
+
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+
+    EXPECT_EQUAL(
+            Result(Result::RESOURCE_EXHAUSTED,
+                   "Update operation rejected for document 'id:type1:type1::1': 'Disk is full'"),
+            f.engine.update(bucket1, tstamp1, upd1, context));
+}
+
+
+TEST_F("require that removes are routed to handlers", SimpleFixture)
+{
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    RemoveResult rr = f.engine.remove(bucket1, tstamp1, docId3, context);
+    assertHandler(bucket0, tstamp0, docId0, f.hset.handler1);
+    assertHandler(bucket0, tstamp0, docId0, f.hset.handler2);
+    EXPECT_FALSE(rr.wasFound());
+
+    f.hset.handler1.setExistingTimestamp(tstamp2);
+    rr = f.engine.remove(bucket1, tstamp1, docId1, context);
+    assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
+    assertHandler(bucket0, tstamp0, docId0, f.hset.handler2);
+    EXPECT_TRUE(rr.wasFound());
+
+    f.hset.handler1.setExistingTimestamp(tstamp0);
+    f.hset.handler2.setExistingTimestamp(tstamp3);
+    rr = f.engine.remove(bucket1, tstamp1, docId2, context);
+    assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
+    assertHandler(bucket1, tstamp1, docId2, f.hset.handler2);
+    EXPECT_TRUE(rr.wasFound());
+
+    f.hset.handler2.setExistingTimestamp(tstamp0);
+    rr = f.engine.remove(bucket1, tstamp1, docId2, context);
+    assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
+    assertHandler(bucket1, tstamp1, docId2, f.hset.handler2);
+    EXPECT_FALSE(rr.wasFound());
+}
+
+
+TEST_F("require that remove is NOT rejected if resource limit is reached", SimpleFixture)
+{
+    f._writeFilter._acceptWriteOperation = false;
+    f._writeFilter._message = "Disk is full";
+
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+
+    EXPECT_EQUAL(RemoveResult(false), f.engine.remove(bucket1, tstamp1, docId1, context));
+}
+
+
+TEST_F("require that listBuckets() is routed to handlers and merged", SimpleFixture)
+{
+    f.hset.handler1.bucketList.push_back(bckId1);
+    f.hset.handler1.bucketList.push_back(bckId2);
+    f.hset.handler2.bucketList.push_back(bckId2);
+    f.hset.handler2.bucketList.push_back(bckId3);
+
+    EXPECT_TRUE(f.engine.listBuckets(PartitionId(1)).getList().empty());
+    BucketIdListResult result = f.engine.listBuckets(partId);
+    const BucketIdListResult::List &bucketList = result.getList();
+    EXPECT_EQUAL(3u, bucketList.size());
+    EXPECT_EQUAL(bckId1, bucketList[0]);
+    EXPECT_EQUAL(bckId2, bucketList[1]);
+    EXPECT_EQUAL(bckId3, bucketList[2]);
+}
+
+
+TEST_F("require that setClusterState() is routed to handlers", SimpleFixture)
+{
+    ClusterState state(createClusterState());
+
+    f.engine.setClusterState(state);
+    EXPECT_EQUAL(&state, f.hset.handler1.lastCalc);
+    EXPECT_EQUAL(&state, f.hset.handler2.lastCalc);
+}
+
+
+TEST_F("require that setActiveState() is routed to handlers and merged", SimpleFixture)
+{
+    f.hset.handler1.bucketStateResult = Result(Result::TRANSIENT_ERROR, "err1");
+    f.hset.handler2.bucketStateResult = Result(Result::PERMANENT_ERROR, "err2");
+
+    Result result = f.engine.setActiveState(bucket1,
+                                             storage::spi::BucketInfo::NOT_ACTIVE);
+    EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode());
+    EXPECT_EQUAL("err1, err2", result.getErrorMessage());
+    EXPECT_EQUAL(storage::spi::BucketInfo::NOT_ACTIVE, f.hset.handler1.lastBucketState);
+    EXPECT_EQUAL(storage::spi::BucketInfo::NOT_ACTIVE, f.hset.handler2.lastBucketState);
+
+    f.engine.setActiveState(bucket1, storage::spi::BucketInfo::ACTIVE);
+    EXPECT_EQUAL(storage::spi::BucketInfo::ACTIVE, f.hset.handler1.lastBucketState);
+    EXPECT_EQUAL(storage::spi::BucketInfo::ACTIVE, f.hset.handler2.lastBucketState);
+}
+
+
+TEST_F("require that getBucketInfo() is routed to handlers and merged", SimpleFixture)
+{
+    f.hset.handler1.bucketInfo = bucketInfo1;
+    f.hset.handler2.bucketInfo = bucketInfo2;
+
+    BucketInfoResult result = f.engine.getBucketInfo(bucket1);
+    EXPECT_EQUAL(bucketInfo3, result.getBucketInfo());
+}
+
+
+TEST_F("require that createBucket() is routed to handlers and merged",
+       SimpleFixture)
+{
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    f.hset.handler1._createBucketResult =
+        Result(Result::TRANSIENT_ERROR, "err1a");
+    f.hset.handler2._createBucketResult =
+        Result(Result::PERMANENT_ERROR, "err2a");
+
+    Result result = f.engine.createBucket(bucket1, context);
+    EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode());
+    EXPECT_EQUAL("err1a, err2a", result.getErrorMessage());
+}
+
+
+TEST_F("require that deleteBucket() is routed to handlers and merged", SimpleFixture)
+{
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    f.hset.handler1.deleteBucketResult = Result(Result::TRANSIENT_ERROR, "err1");
+    f.hset.handler2.deleteBucketResult = Result(Result::PERMANENT_ERROR, "err2");
+
+    Result result = f.engine.deleteBucket(bucket1, context);
+    EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode());
+    EXPECT_EQUAL("err1, err2", result.getErrorMessage());
+}
+
+
+TEST_F("require that getModifiedBuckets() is routed to handlers and merged", SimpleFixture)
+{
+    f.hset.handler1.modBucketList.push_back(bckId1);
+    f.hset.handler1.modBucketList.push_back(bckId2);
+    f.hset.handler2.modBucketList.push_back(bckId2);
+    f.hset.handler2.modBucketList.push_back(bckId3);
+
+    BucketIdListResult result = f.engine.getModifiedBuckets();
+    const BucketIdListResult::List &bucketList = result.getList();
+    EXPECT_EQUAL(3u, bucketList.size());
+    EXPECT_EQUAL(bckId1, bucketList[0]);
+    EXPECT_EQUAL(bckId2, bucketList[1]);
+    EXPECT_EQUAL(bckId3, bucketList[2]);
+}
+
+
+TEST_F("require that get is sent to all handlers", SimpleFixture) {
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    GetResult result = f.engine.get(bucket1, document::AllFields(), docId1,
+                                    context);
+
+    EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId);
+    EXPECT_EQUAL(docId1, f.hset.handler2.lastDocId);
+}
+
+TEST_F("require that get freezes the bucket", SimpleFixture) {
+    EXPECT_FALSE(f.hset.handler1.wasFrozen(bucket1));
+    EXPECT_FALSE(f.hset.handler2.wasFrozen(bucket1));
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    f.engine.get(bucket1, document::AllFields(), docId1, context);
+    EXPECT_TRUE(f.hset.handler1.wasFrozen(bucket1));
+    EXPECT_TRUE(f.hset.handler2.wasFrozen(bucket1));
+    EXPECT_FALSE(f.hset.handler1.isFrozen(bucket1));
+    EXPECT_FALSE(f.hset.handler2.isFrozen(bucket1));
+}
+
+TEST_F("require that get returns the first document found", SimpleFixture) {
+    f.hset.handler1.setDocument(*doc1, tstamp1);
+    f.hset.handler2.setDocument(*doc2, tstamp2);
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    GetResult result = f.engine.get(bucket1, document::AllFields(), docId1,
+                                    context);
+
+    EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId);
+    EXPECT_EQUAL(DocumentId(), f.hset.handler2.lastDocId);
+
+    EXPECT_EQUAL(tstamp1, result.getTimestamp());
+    ASSERT_TRUE(result.hasDocument());
+    EXPECT_EQUAL(*doc1, result.getDocument());
+}
+
+TEST_F("require that createIterator does", SimpleFixture) {
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    CreateIteratorResult result =
+        f.engine.createIterator(bucket1, document::AllFields(), selection,
+                                storage::spi::NEWEST_DOCUMENT_ONLY, context);
+    EXPECT_FALSE(result.hasError());
+    EXPECT_TRUE(result.getIteratorId());
+
+    uint64_t max_size = 1024;
+    IterateResult it_result =
+        f.engine.iterate(result.getIteratorId(), max_size, context);
+    EXPECT_FALSE(it_result.hasError());
+}
+
+TEST_F("require that iterator ids are unique", SimpleFixture) {
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    CreateIteratorResult result =
+        f.engine.createIterator(bucket1, document::AllFields(), selection,
+                                storage::spi::NEWEST_DOCUMENT_ONLY, context);
+    CreateIteratorResult result2 =
+        f.engine.createIterator(bucket1, document::AllFields(), selection,
+                                storage::spi::NEWEST_DOCUMENT_ONLY, context);
+    EXPECT_FALSE(result.hasError());
+    EXPECT_FALSE(result2.hasError());
+    EXPECT_NOT_EQUAL(result.getIteratorId(), result2.getIteratorId());
+}
+
+TEST_F("require that iterate requires valid iterator", SimpleFixture) {
+    uint64_t max_size = 1024;
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    IterateResult it_result = f.engine.iterate(IteratorId(1), max_size,
+                                               context);
+    EXPECT_TRUE(it_result.hasError());
+    EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode());
+    EXPECT_EQUAL("Unknown iterator with id 1", it_result.getErrorMessage());
+
+    CreateIteratorResult result =
+        f.engine.createIterator(bucket1, document::AllFields(), selection,
+                                storage::spi::NEWEST_DOCUMENT_ONLY, context);
+    EXPECT_TRUE(result.getIteratorId());
+
+    it_result = f.engine.iterate(result.getIteratorId(), max_size, context);
+    EXPECT_FALSE(it_result.hasError());
+}
+
+TEST_F("require that iterate returns documents", SimpleFixture) {
+    f.hset.handler1.setDocument(*doc1, tstamp1);
+    f.hset.handler2.setDocument(*doc2, tstamp2);
+
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    uint64_t max_size = 1024;
+    CreateIteratorResult result =
+        f.engine.createIterator(bucket1, document::AllFields(), selection,
+                                storage::spi::NEWEST_DOCUMENT_ONLY, context);
+    EXPECT_TRUE(result.getIteratorId());
+
+    IterateResult it_result =
+        f.engine.iterate(result.getIteratorId(), max_size, context);
+    EXPECT_FALSE(it_result.hasError());
+    EXPECT_EQUAL(2u, it_result.getEntries().size());
+}
+
+TEST_F("require that destroyIterator prevents iteration", SimpleFixture) {
+    f.hset.handler1.setDocument(*doc1, tstamp1);
+
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    CreateIteratorResult create_result =
+        f.engine.createIterator(bucket1, document::AllFields(), selection,
+                                storage::spi::NEWEST_DOCUMENT_ONLY, context);
+    EXPECT_TRUE(create_result.getIteratorId());
+
+    Result result = f.engine.destroyIterator(create_result.getIteratorId(),
+                                             context);
+    EXPECT_FALSE(result.hasError());
+
+    uint64_t max_size = 1024;
+    IterateResult it_result =
+        f.engine.iterate(create_result.getIteratorId(), max_size, context);
+    EXPECT_TRUE(it_result.hasError());
+    EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode());
+    string msg_prefix = "Unknown iterator with id";
+    EXPECT_EQUAL(msg_prefix,
+                 it_result.getErrorMessage().substr(0, msg_prefix.size()));
+}
+
+TEST_F("require that buckets are frozen during iterator life", SimpleFixture) {
+    EXPECT_FALSE(f.hset.handler1.isFrozen(bucket1));
+    EXPECT_FALSE(f.hset.handler2.isFrozen(bucket1));
+    storage::spi::LoadType loadType(0, "default");
+    Context context(loadType, storage::spi::Priority(0),
+                    storage::spi::Trace::TraceLevel(0));
+    CreateIteratorResult create_result =
+        f.engine.createIterator(bucket1, document::AllFields(), selection,
+                                storage::spi::NEWEST_DOCUMENT_ONLY, context);
+    EXPECT_TRUE(f.hset.handler1.isFrozen(bucket1));
+    EXPECT_TRUE(f.hset.handler2.isFrozen(bucket1));
+    f.engine.destroyIterator(create_result.getIteratorId(), context);
+    EXPECT_FALSE(f.hset.handler1.isFrozen(bucket1));
+    EXPECT_FALSE(f.hset.handler2.isFrozen(bucket1));
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
+
diff --git a/searchcore/src/tests/proton/proton/CMakeLists.txt b/searchcore/src/tests/proton/proton/CMakeLists.txt
new file mode 100644
index 00000000000..5c90dd5bfcc
--- /dev/null
+++ b/searchcore/src/tests/proton/proton/CMakeLists.txt
@@ -0,0 +1 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/.gitignore b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/.gitignore
new file mode 100644
index 00000000000..5d662ccaf21
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/.gitignore
@@ -0,0 +1 @@
+searchcore_attribute_reprocessing_initializer_test_app
diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt
new file mode 100644
index 00000000000..5e17a1d6606
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_reprocessing_initializer_test_app
+    SOURCES
+    attribute_reprocessing_initializer_test.cpp
+    DEPENDS
+    searchcore_reprocessing
+    searchcore_attribute
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_attribute_reprocessing_initializer_test_app COMMAND searchcore_attribute_reprocessing_initializer_test_app)
diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/DESC b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/DESC
new file mode 100644
index 00000000000..2ae1d0c8164
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/DESC
@@ -0,0 +1,2 @@
+Test for attribute reprocessing initializer. Take a look at attribute_reprocessing_initializer_test.cpp for details.
+
diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/FILES b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/FILES
new file mode 100644
index 00000000000..6c9084f176d
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/FILES
@@ -0,0 +1 @@
+attribute_reprocessing_initializer_test.cpp
diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp
new file mode 100644
index 00000000000..4670719897a
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp
@@ -0,0 +1,247 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_reprocessing_initializer_test");
+
+#include <vespa/searchcore/proton/attribute/attribute_populator.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/document_field_populator.h>
+#include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h>
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h>
+#include <vespa/searchcore/proton/test/attribute_utils.h>
+#include <vespa/searchcore/proton/test/directory_handler.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+
+using namespace proton;
+using namespace search;
+using namespace search::index;
+
+const vespalib::string TEST_DIR = "test_output";
+const SerialNum INIT_SERIAL_NUM = 10;
+typedef std::vector<vespalib::string> StringVector;
+typedef std::set<vespalib::string> StringSet;
+typedef AttributeReprocessingInitializer::Config ARIConfig;
+
+struct MyReprocessingHandler : public IReprocessingHandler
+{
+    IReprocessingReader::SP _reader;
+    std::vector<IReprocessingRewriter::SP> _rewriters;
+    MyReprocessingHandler() : _reader(), _rewriters() {}
+    virtual void addReader(const IReprocessingReader::SP &reader) {
+        _reader = reader;
+    }
+    virtual void addRewriter(const IReprocessingRewriter::SP &rewriter) {
+        _rewriters.push_back(rewriter);
+    }
+};
+
+struct MyDocTypeInspector : public IDocumentTypeInspector
+{
+    typedef std::shared_ptr<MyDocTypeInspector> SP;
+    std::set<vespalib::string> _fields;
+    MyDocTypeInspector() : _fields() {}
+    virtual bool hasField(const vespalib::string &name) const {
+        return _fields.count(name) > 0;
+    }
+};
+
+struct MyConfig
+{
+    DummyFileHeaderContext _fileHeaderContext;
+    ForegroundTaskExecutor _attributeFieldWriter;
+    AttributeManager::SP _mgr;
+    search::index::Schema _schema;
+    MyDocTypeInspector::SP _inspector;
+    MyConfig()
+        : _fileHeaderContext(),
+          _attributeFieldWriter(),
+          _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(),
+                                    _fileHeaderContext,
+                                    _attributeFieldWriter)),
+          _schema(),
+          _inspector(new MyDocTypeInspector())
+    {
+    }
+    void addFields(const StringVector &fields) {
+        for (auto field : fields) {
+            _inspector->_fields.insert(field);
+        }
+    }
+    void addAttrs(const StringVector &attrs) {
+        for (auto attr : attrs) {
+            if (attr == "tensor") {
+                _mgr->addAttribute(attr, test::AttributeUtils::getTensorConfig(), 1);
+                _schema.addAttributeField(Schema::AttributeField(attr, Schema::TENSOR));
+            } else if (attr == "predicate") {
+                _mgr->addAttribute(attr, test::AttributeUtils::getPredicateConfig(), 1);
+                _schema.addAttributeField(Schema::AttributeField(attr, Schema::BOOLEANTREE));
+            } else {
+                _mgr->addAttribute(attr, test::AttributeUtils::getStringConfig(), 1);
+                _schema.addAttributeField(Schema::AttributeField(attr, Schema::STRING));
+            }
+        }
+    }
+    void addIndexField(const vespalib::string &name) {
+        _schema.addIndexField(Schema::IndexField(name, Schema::STRING));
+    }
+};
+
+struct Fixture
+{
+    test::DirectoryHandler _dirHandler;
+    DummyFileHeaderContext _fileHeaderContext;
+    ForegroundTaskExecutor _attributeFieldWriter;
+    AttributeManager::SP _mgr;
+    MyConfig _oldCfg;
+    MyConfig _newCfg;
+    AttributeReprocessingInitializer::UP _initializer;
+    MyReprocessingHandler _handler;
+    Fixture()
+        : _dirHandler(TEST_DIR),
+          _fileHeaderContext(),
+          _attributeFieldWriter(),
+          _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(),
+                                    _fileHeaderContext,
+                                    _attributeFieldWriter)),
+          _initializer(),
+          _handler()
+    {
+    }
+    void init() {
+        _initializer.reset(new AttributeReprocessingInitializer
+                (ARIConfig(_newCfg._mgr, _newCfg._schema, _newCfg._inspector),
+                        ARIConfig(_oldCfg._mgr, _oldCfg._schema, _oldCfg._inspector), "test"));
+        _initializer->initialize(_handler);
+    }
+    Fixture &addOldConfig(const StringVector &fields,
+                          const StringVector &attrs) {
+        return addConfig(fields, attrs, _oldCfg);
+    }
+    Fixture &addNewConfig(const StringVector &fields,
+                          const StringVector &attrs) {
+        return addConfig(fields, attrs, _newCfg);
+    }
+    Fixture &addConfig(const StringVector &fields,
+                       const StringVector &attrs,
+                       MyConfig &cfg) {
+        cfg.addFields(fields);
+        cfg.addAttrs(attrs);
+        return *this;
+    }
+    bool assertAttributes(const StringSet &expAttrs) {
+        if (expAttrs.empty()) {
+            if (!EXPECT_TRUE(_handler._reader.get() == nullptr)) return false;
+        } else {
+            const AttributePopulator &populator =
+                    dynamic_cast<const AttributePopulator &>(*_handler._reader);
+            std::vector<search::AttributeVector *> attrList =
+                populator.getWriter().getWritableAttributes();
+            std::set<vespalib::string> actAttrs;
+            for (const auto attr : attrList) {
+                actAttrs.insert(attr->getName());
+            }
+            if (!EXPECT_EQUAL(expAttrs, actAttrs)) return false;
+        }
+        return true;
+    }
+    bool assertFields(const StringSet &expFields) {
+        if (expFields.empty()) {
+            if (!EXPECT_EQUAL(0u, _handler._rewriters.size())) return false;
+        } else {
+            StringSet actFields;
+            for (auto rewriter : _handler._rewriters) {
+                const DocumentFieldPopulator &populator =
+                    dynamic_cast<const DocumentFieldPopulator &>(*rewriter);
+                actFields.insert(populator.getAttribute().getName());
+            }
+            if (!EXPECT_EQUAL(expFields, actFields)) return false;
+        }
+        return true;
+    }
+};
+
+TEST_F("require that new field does NOT require attribute populate", Fixture)
+{
+    f.addOldConfig({}, {}).addNewConfig({"a"}, {"a"}).init();
+    EXPECT_TRUE(f.assertAttributes({}));
+}
+
+TEST_F("require that added attribute aspect does require attribute populate", Fixture)
+{
+    f.addOldConfig({"a"}, {}).addNewConfig({"a"}, {"a"}).init();
+    EXPECT_TRUE(f.assertAttributes({"a"}));
+}
+
+TEST_F("require that initializer can setup populate of several attributes", Fixture)
+{
+    f.addOldConfig({"a", "b", "c", "d"}, {"a", "b"}).
+            addNewConfig({"a", "b", "c", "d"}, {"a", "b", "c", "d"}).init();
+    EXPECT_TRUE(f.assertAttributes({"c", "d"}));
+}
+
+TEST_F("require that new field does NOT require document field populate", Fixture)
+{
+    f.addOldConfig({}, {}).addNewConfig({"a"}, {"a"}).init();
+    EXPECT_TRUE(f.assertFields({}));
+}
+
+TEST_F("require that removed field does NOT require document field populate", Fixture)
+{
+    f.addOldConfig({"a"}, {"a"}).addNewConfig({}, {}).init();
+    EXPECT_TRUE(f.assertFields({}));
+}
+
+TEST_F("require that removed attribute aspect does require document field populate", Fixture)
+{
+    f.addOldConfig({"a"}, {"a"}).addNewConfig({"a"}, {}).init();
+    EXPECT_TRUE(f.assertFields({"a"}));
+}
+
+TEST_F("require that removed attribute aspect (when also index field) does NOT require document field populate",
+        Fixture)
+{
+    f.addOldConfig({"a"}, {"a"}).addNewConfig({"a"}, {});
+    f._oldCfg.addIndexField("a");
+    f._newCfg.addIndexField("a");
+    f.init();
+    EXPECT_TRUE(f.assertFields({}));
+}
+
+TEST_F("require that initializer can setup populate of several document fields", Fixture)
+{
+    f.addOldConfig({"a", "b", "c", "d"}, {"a", "b", "c", "d"}).
+            addNewConfig({"a", "b", "c", "d"}, {"a", "b"}).init();
+    EXPECT_TRUE(f.assertFields({"c", "d"}));
+}
+
+TEST_F("require that initializer can setup both attribute and document field populate", Fixture)
+{
+    f.addOldConfig({"a", "b"}, {"a"}).
+            addNewConfig({"a", "b"}, {"b"}).init();
+    EXPECT_TRUE(f.assertAttributes({"b"}));
+    EXPECT_TRUE(f.assertFields({"a"}));
+}
+
+TEST_F("require that tensor fields are not populated from attribute", Fixture)
+{
+    f.addOldConfig({"a", "b", "c", "d", "tensor"},
+                   {"a", "b", "c", "d", "tensor"}).
+        addNewConfig({"a", "b", "c", "d", "tensor"}, {"a", "b"}).init();
+    EXPECT_TRUE(f.assertFields({"c", "d"}));
+}
+
+TEST_F("require that predicate fields are not populated from attribute", Fixture)
+{
+    f.addOldConfig({"a", "b", "c", "d", "predicate"},
+                   {"a", "b", "c", "d", "predicate"}).
+        addNewConfig({"a", "b", "c", "d", "predicate"}, {"a", "b"}).init();
+    EXPECT_TRUE(f.assertFields({"c", "d"}));
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/.gitignore b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/.gitignore
new file mode 100644
index 00000000000..50e203b78e8
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/.gitignore
@@ -0,0 +1 @@
+searchcore_document_reprocessing_handler_test_app
diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/CMakeLists.txt b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/CMakeLists.txt
new file mode 100644
index 00000000000..170e381c99c
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_document_reprocessing_handler_test_app
+    SOURCES
+    document_reprocessing_handler_test.cpp
+    DEPENDS
+    searchcore_reprocessing
+)
+vespa_add_test(NAME searchcore_document_reprocessing_handler_test_app COMMAND searchcore_document_reprocessing_handler_test_app)
diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/DESC b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/DESC
new file mode 100644
index 00000000000..663b2304d37
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/DESC
@@ -0,0 +1,2 @@
+Test for document reprocessing handler. Take a look at document_reprocessing_handler_test.cpp for details.
+
diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/FILES b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/FILES
new file mode 100644
index 00000000000..2301fc01844
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/FILES
@@ -0,0 +1 @@
+document_reprocessing_handler_test.cpp
diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp
new file mode 100644
index 00000000000..f22762a56bb
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp
@@ -0,0 +1,124 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("document_reprocessing_handler_test");
+
+#include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h>
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+using namespace proton;
+using namespace search::index;
+
+template <typename ReprocessingType, typename DocumentType>
+struct MyProcessor : public ReprocessingType
+{
+    typedef std::shared_ptr<MyProcessor<ReprocessingType, DocumentType> > SP;
+    uint32_t _lid;
+    DocumentId _docId;
+
+    MyProcessor() : _lid(0), _docId() {}
+    virtual void handleExisting(uint32_t lid, DocumentType doc) {
+        _lid = lid;
+        _docId = doc.getId();
+    }
+};
+
+typedef MyProcessor<IReprocessingReader, const Document &> MyReader;
+typedef MyProcessor<IReprocessingRewriter, Document &> MyRewriter;
+
+const vespalib::string DOC_ID = "id:test:searchdocument::0";
+
+struct FixtureBase
+{
+    DocumentReprocessingHandler _handler;
+    DocBuilder _docBuilder;
+    FixtureBase(uint32_t docIdLimit)
+        : _handler(docIdLimit),
+          _docBuilder(Schema())
+    {
+    }
+    Document::UP createDoc() {
+        return _docBuilder.startDocument(DOC_ID).endDocument();
+    }
+};
+
+struct ReaderFixture : public FixtureBase
+{
+    MyReader::SP _reader1;
+    MyReader::SP _reader2;
+    ReaderFixture()
+        : ReaderFixture(std::numeric_limits<uint32_t>::max())
+    {
+    }
+    ReaderFixture(uint32_t docIdLimit)
+        : FixtureBase(docIdLimit),
+          _reader1(new MyReader()),
+          _reader2(new MyReader())
+    {
+        _handler.addReader(_reader1);
+        _handler.addReader(_reader2);
+    }
+};
+
+struct RewriterFixture : public FixtureBase
+{
+    MyRewriter::SP _rewriter1;
+    MyRewriter::SP _rewriter2;
+    RewriterFixture()
+        : RewriterFixture(std::numeric_limits<uint32_t>::max())
+    {
+    }
+    RewriterFixture(uint32_t docIdLimit)
+        : FixtureBase(docIdLimit),
+          _rewriter1(new MyRewriter()),
+          _rewriter2(new MyRewriter())
+    {
+        _handler.addRewriter(_rewriter1);
+        _handler.addRewriter(_rewriter2);
+    }
+};
+
+TEST_F("require that handler propagates visit of existing document to readers", ReaderFixture)
+{
+    f._handler.visit(23u, *f.createDoc());
+    EXPECT_EQUAL(23u, f._reader1->_lid);
+    EXPECT_EQUAL(DOC_ID, f._reader1->_docId.toString());
+    EXPECT_EQUAL(23u, f._reader2->_lid);
+    EXPECT_EQUAL(DOC_ID, f._reader2->_docId.toString());
+}
+
+TEST_F("require that handler propagates visit of existing document to rewriters", RewriterFixture)
+{
+    f._handler.getRewriteVisitor().visit(23u, *f.createDoc());
+    EXPECT_EQUAL(23u, f._rewriter1->_lid);
+    EXPECT_EQUAL(DOC_ID, f._rewriter1->_docId.toString());
+    EXPECT_EQUAL(23u, f._rewriter2->_lid);
+    EXPECT_EQUAL(DOC_ID, f._rewriter2->_docId.toString());
+}
+
+TEST_F("require that handler skips out of range visit to readers",
+       ReaderFixture(10))
+{
+    f._handler.visit(23u, *f.createDoc());
+    EXPECT_EQUAL(0u, f._reader1->_lid);
+    EXPECT_EQUAL(DocumentId().toString(), f._reader1->_docId.toString());
+    EXPECT_EQUAL(0u, f._reader2->_lid);
+    EXPECT_EQUAL(DocumentId().toString(), f._reader2->_docId.toString());
+}
+
+TEST_F("require that handler skips out of range visit to rewriters",
+       RewriterFixture(10))
+{
+    f._handler.getRewriteVisitor().visit(23u, *f.createDoc());
+    EXPECT_EQUAL(0u, f._rewriter1->_lid);
+    EXPECT_EQUAL(DocumentId().toString(), f._rewriter1->_docId.toString());
+    EXPECT_EQUAL(0u, f._rewriter2->_lid);
+    EXPECT_EQUAL(DocumentId().toString(), f._rewriter2->_docId.toString());
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/.gitignore b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/.gitignore
new file mode 100644
index 00000000000..ecb260d2c0e
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/.gitignore
@@ -0,0 +1 @@
+searchcore_reprocessing_runner_test_app
diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/CMakeLists.txt b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/CMakeLists.txt
new file mode 100644
index 00000000000..f5eff73f9f0
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_reprocessing_runner_test_app
+    SOURCES
+    reprocessing_runner_test.cpp
+    DEPENDS
+    searchcore_reprocessing
+)
+vespa_add_test(NAME searchcore_reprocessing_runner_test_app COMMAND searchcore_reprocessing_runner_test_app)
diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/DESC b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/DESC
new file mode 100644
index 00000000000..ffa0db7ae9e
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/DESC
@@ -0,0 +1 @@
+Test reprocessing runner.
diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/FILES b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/FILES
new file mode 100644
index 00000000000..091769d58cb
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/FILES
@@ -0,0 +1 @@
+reprocessing_runner_test.cpp
diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp
new file mode 100644
index 00000000000..c4c462ecfa1
--- /dev/null
+++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("reprocessing_runner_test");
+
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h>
+#include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace proton;
+
+struct Fixture
+{
+    ReprocessingRunner _runner;
+    Fixture()
+        : _runner()
+    {
+    }
+};
+
+typedef ReprocessingRunner::ReprocessingTasks TaskList;
+
+struct MyTask : public IReprocessingTask
+{
+    ReprocessingRunner &_runner;
+    double _initProgress;
+    double _middleProgress;
+    double _finalProgress;
+    double _myProgress;
+    double _weight;
+
+    MyTask(ReprocessingRunner &runner,
+           double initProgress,
+           double middleProgress,
+           double finalProgress,
+           double weight)
+        : _runner(runner),
+          _initProgress(initProgress),
+          _middleProgress(middleProgress),
+          _finalProgress(finalProgress),
+          _myProgress(0.0),
+          _weight(weight)
+    {
+    }
+
+    virtual void
+    run()
+    {
+        ASSERT_EQUAL(_initProgress, _runner.getProgress());
+        _myProgress = 0.5;
+        ASSERT_EQUAL(_middleProgress, _runner.getProgress());
+        _myProgress = 1.0;
+        ASSERT_EQUAL(_finalProgress, _runner.getProgress());
+    }
+
+    virtual Progress
+    getProgress(void) const
+    {
+        return Progress(_myProgress, _weight);
+    }
+
+    static std::shared_ptr<MyTask>
+    create(ReprocessingRunner &runner,
+           double initProgress,
+           double middleProgress,
+           double finalProgress,
+           double weight)
+    {
+        return std::make_shared<MyTask>(runner,
+                                        initProgress,
+                                        middleProgress,
+                                        finalProgress,
+                                        weight);
+    }
+};
+
+TEST_F("require that progress is calculated when tasks are executed", Fixture)
+{
+    TaskList tasks;
+    EXPECT_EQUAL(0.0, f._runner.getProgress());
+    tasks.push_back(MyTask::create(f._runner,
+                                   0.0,
+                                   0.1,
+                                   0.2,
+                                   1.0));
+    tasks.push_back(MyTask::create(f._runner,
+                                   0.2,
+                                   0.6,
+                                   1.0,
+                                   4.0));
+    f._runner.addTasks(tasks);
+    tasks.clear();
+    EXPECT_EQUAL(0.0, f._runner.getProgress());
+    f._runner.run();
+    EXPECT_EQUAL(1.0, f._runner.getProgress());
+}
+
+
+TEST_F("require that runner can be reset", Fixture)
+{
+    TaskList tasks;
+    EXPECT_EQUAL(0.0, f._runner.getProgress());
+    tasks.push_back(MyTask::create(f._runner,
+                                   0.0,
+                                   0.5,
+                                   1.0,
+                                   1.0));
+    f._runner.addTasks(tasks);
+    tasks.clear();
+    EXPECT_EQUAL(0.0, f._runner.getProgress());
+    f._runner.run();
+    EXPECT_EQUAL(1.0, f._runner.getProgress());
+    f._runner.reset();
+    EXPECT_EQUAL(0.0, f._runner.getProgress());
+    tasks.push_back(MyTask::create(f._runner,
+                                   0.0,
+                                   0.5,
+                                   1.0,
+                                   1.0));
+    f._runner.addTasks(tasks);
+    tasks.clear();
+    EXPECT_EQUAL(0.0, f._runner.getProgress());
+    f._runner.reset();
+    EXPECT_EQUAL(0.0, f._runner.getProgress());
+    tasks.push_back(MyTask::create(f._runner,
+                                   0.0,
+                                   0.5,
+                                   1.0,
+                                   4.0));
+    f._runner.addTasks(tasks);
+    tasks.clear();
+    EXPECT_EQUAL(0.0, f._runner.getProgress());
+    f._runner.run();
+    EXPECT_EQUAL(1.0, f._runner.getProgress());
+}
+
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/server/.gitignore b/searchcore/src/tests/proton/server/.gitignore
new file mode 100644
index 00000000000..dc96b15f5fe
--- /dev/null
+++ b/searchcore/src/tests/proton/server/.gitignore
@@ -0,0 +1,9 @@
+*_test
+.depend
+Makefile
+test_data
+searchcore_attribute_metrics_test_app
+searchcore_documentretriever_test_app
+searchcore_feeddebugger_test_app
+searchcore_feedstates_test_app
+searchcore_memoryconfigstore_test_app
diff --git a/searchcore/src/tests/proton/server/CMakeLists.txt b/searchcore/src/tests/proton/server/CMakeLists.txt
new file mode 100644
index 00000000000..3ae89e7393d
--- /dev/null
+++ b/searchcore/src/tests/proton/server/CMakeLists.txt
@@ -0,0 +1,52 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_attribute_metrics_test_app
+    SOURCES
+    attribute_metrics_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_proton_metrics
+)
+vespa_add_test(NAME searchcore_attribute_metrics_test_app COMMAND searchcore_attribute_metrics_test_app)
+vespa_add_executable(searchcore_documentretriever_test_app
+    SOURCES
+    documentretriever_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_fconfig
+    searchcore_attribute
+    searchcore_feedoperation
+    searchcore_documentmetastore
+    searchcore_bucketdb
+    searchcore_pcommon
+    searchcore_persistenceengine
+)
+vespa_add_test(NAME searchcore_documentretriever_test_app COMMAND searchcore_documentretriever_test_app)
+vespa_add_executable(searchcore_feeddebugger_test_app
+    SOURCES
+    feeddebugger_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_feeddebugger_test_app COMMAND searchcore_feeddebugger_test_app)
+vespa_add_executable(searchcore_feedstates_test_app
+    SOURCES
+    feedstates_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_bucketdb
+    searchcore_persistenceengine
+    searchcore_feedoperation
+    searchcore_pcommon
+    searchcore_proton_metrics
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_feedstates_test_app COMMAND searchcore_feedstates_test_app)
+vespa_add_executable(searchcore_memoryconfigstore_test_app
+    SOURCES
+    memoryconfigstore_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_memoryconfigstore_test_app COMMAND searchcore_memoryconfigstore_test_app)
diff --git a/searchcore/src/tests/proton/server/attribute_metrics_test.cpp b/searchcore/src/tests/proton/server/attribute_metrics_test.cpp
new file mode 100644
index 00000000000..18f35d9bf5e
--- /dev/null
+++ b/searchcore/src/tests/proton/server/attribute_metrics_test.cpp
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_metrics_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/metrics/attribute_metrics.h>
+
+using namespace proton;
+
+class Test : public vespalib::TestApp
+{
+public:
+    int Main();
+};
+
+int
+Test::Main()
+{
+    TEST_INIT("attribute_metrics_test");
+    {
+        AttributeMetrics attrMetrics(0);
+        EXPECT_EQUAL(0u, attrMetrics.list.release().size());
+        {
+            AttributeMetrics::List::Entry::LP e1 = attrMetrics.list.add("foo");
+            AttributeMetrics::List::Entry::LP e2 = attrMetrics.list.add("bar");
+            AttributeMetrics::List::Entry::LP e3 = attrMetrics.list.add("foo");
+            EXPECT_TRUE(e1.get() != 0);
+            EXPECT_TRUE(e2.get() != 0);
+            EXPECT_TRUE(e3.get() == 0);
+        }
+        {
+            const AttributeMetrics &constMetrics = attrMetrics;
+            AttributeMetrics::List::Entry::LP e1 = constMetrics.list.get("foo");
+            AttributeMetrics::List::Entry::LP e2 = constMetrics.list.get("bar");
+            AttributeMetrics::List::Entry::LP e3 = constMetrics.list.get("baz");
+            EXPECT_TRUE(e1.get() != 0);
+            EXPECT_TRUE(e2.get() != 0);
+            EXPECT_TRUE(e3.get() == 0);
+        }
+        EXPECT_EQUAL(2u, attrMetrics.list.release().size());
+        {
+            const AttributeMetrics &constMetrics = attrMetrics;
+            AttributeMetrics::List::Entry::LP e1 = constMetrics.list.get("foo");
+            AttributeMetrics::List::Entry::LP e2 = constMetrics.list.get("bar");
+            AttributeMetrics::List::Entry::LP e3 = constMetrics.list.get("baz");
+            EXPECT_TRUE(e1.get() == 0);
+            EXPECT_TRUE(e2.get() == 0);
+            EXPECT_TRUE(e3.get() == 0);
+        }
+        EXPECT_EQUAL(0u, attrMetrics.list.release().size());
+    }
+    TEST_DONE();
+}
+
+TEST_APPHOOK(Test);
diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore b/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore
new file mode 100644
index 00000000000..b085eedc970
--- /dev/null
+++ b/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore
@@ -0,0 +1 @@
+searchcore_data_directory_upgrader_test_app
diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt b/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt
new file mode 100644
index 00000000000..9c8048a0a69
--- /dev/null
+++ b/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_data_directory_upgrader_test_app
+    SOURCES
+    data_directory_upgrader_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_flushengine
+)
+vespa_add_test(NAME searchcore_data_directory_upgrader_test_app COMMAND searchcore_data_directory_upgrader_test_app)
diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/DESC b/searchcore/src/tests/proton/server/data_directory_upgrader/DESC
new file mode 100644
index 00000000000..d0ca4c99210
--- /dev/null
+++ b/searchcore/src/tests/proton/server/data_directory_upgrader/DESC
@@ -0,0 +1 @@
+data_directory_upgrader test. Take a look at data_directory_upgrader_test.cpp for details.
diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/FILES b/searchcore/src/tests/proton/server/data_directory_upgrader/FILES
new file mode 100644
index 00000000000..d1aee9bddfa
--- /dev/null
+++ b/searchcore/src/tests/proton/server/data_directory_upgrader/FILES
@@ -0,0 +1 @@
+data_directory_upgrader_test.cpp
diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp b/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp
new file mode 100644
index 00000000000..7b6cf9143ee
--- /dev/null
+++ b/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp
@@ -0,0 +1,200 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("data_directory_upgrader_test");
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/searchcore/proton/server/data_directory_upgrader.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <iostream>
+
+using namespace proton;
+using namespace vespalib;
+
+typedef DataDirectoryUpgrader::RowColDir RowColDir;
+typedef DataDirectoryUpgrader::ScanResult ScanResult;
+typedef DataDirectoryUpgrader::UpgradeResult UpgradeResult;
+
+const string SCAN_DIR = "mytest";
+const string DEST_DIR = SCAN_DIR + "/n1";
+
+void
+assertDirs(const DirectoryList &exp, const DirectoryList &act)
+{
+    ASSERT_EQUAL(exp.size(), act.size());
+    for (size_t i = 0; i < exp.size(); ++i) {
+        EXPECT_EQUAL(exp[i], act[i]);
+    }
+}
+
+void
+assertDirs(const DirectoryList &rowColDirs, bool destDirExisting, const ScanResult &act)
+{
+    ASSERT_EQUAL(rowColDirs.size(), act.getRowColDirs().size());
+    for (size_t i = 0; i < rowColDirs.size(); ++i) {
+        EXPECT_EQUAL(rowColDirs[i], act.getRowColDirs()[i].dir());
+    }
+    EXPECT_EQUAL(destDirExisting, act.isDestDirExisting());
+}
+
+void
+assertDataFile(const vespalib::string &dir)
+{
+    FileInfo::UP file = stat(dir + "/data.txt");
+    ASSERT_TRUE(file.get() != NULL);
+    EXPECT_TRUE(file->_plainfile);
+}
+
+vespalib::string
+readFile(const vespalib::string &fileName)
+{
+    File file(fileName);
+    file.open(File::READONLY);
+    FileInfo info = file.stat();
+    char buf[512];
+    size_t bytesRead = file.read(&buf, info._size, 0);
+    return vespalib::string(buf, bytesRead);
+}
+
+void
+assertUpgradeFile(const vespalib::string &exp, const vespalib::string &dir)
+{
+    EXPECT_EQUAL(exp, readFile(dir + "/data-directory-upgrade-source.txt"));
+}
+
+void
+assertDowngradeScript(const vespalib::string &exp, const vespalib::string &dir)
+{
+    EXPECT_EQUAL(exp, readFile(dir + "/data-directory-downgrade.sh"));
+}
+
+struct BaseFixture
+{
+    DataDirectoryUpgrader _upg;
+    BaseFixture(const DirectoryList &dirs, bool createDestDir = false) : _upg(SCAN_DIR, DEST_DIR) {
+        mkdir(SCAN_DIR);
+        if (createDestDir) {
+            mkdir(DEST_DIR);
+        }
+        for (const string &dir : dirs) {
+            mkdir(SCAN_DIR + "/" + dir);
+            File f(SCAN_DIR + "/" + dir + "/data.txt");
+            f.open(File::CREATE);
+            f.close();
+        }
+    }
+    virtual ~BaseFixture() {
+        rmdir(SCAN_DIR, true);
+    }
+    DirectoryList getDirs(const vespalib::string &subDir = "") const {
+        DirectoryList l = listDirectory(SCAN_DIR + "/" + subDir);
+        std::sort(l.begin(), l.end());
+        return l;
+    }  
+};
+
+struct EmptyFixture : public BaseFixture
+{
+    EmptyFixture() : BaseFixture({}) {}
+};
+
+struct SingleFixture : public BaseFixture
+{
+    SingleFixture() : BaseFixture({"r0/c0"}) {}
+};
+
+struct DoubleFixture : public BaseFixture
+{
+    DoubleFixture() : BaseFixture({"r0/c0", "r1/c1"}) {}
+};
+
+struct UnrelatedFixture : public BaseFixture
+{
+    UnrelatedFixture() : BaseFixture({"r0/cY", "rX/c1", "r0"}) {}
+};
+
+struct ExistingDestinationFixture : public BaseFixture
+{
+    ExistingDestinationFixture() : BaseFixture({"r0/c0"}, true) {}
+};
+
+TEST_F("require that single row/column directory is discovered", SingleFixture)
+{
+    ScanResult res = f._upg.scan();
+    assertDirs({"r0/c0"}, false, res);
+}
+
+TEST_F("require that multiple row/column directories are discovered", DoubleFixture)
+{
+    ScanResult res = f._upg.scan();
+    assertDirs({"r0/c0", "r1/c1"}, false, res);
+}
+
+TEST_F("require that unrelated directories are not discovered", UnrelatedFixture)
+{
+    ScanResult res = f._upg.scan();
+    assertDirs({}, false, res);
+}
+
+TEST_F("require that existing destination directory is discovered", ExistingDestinationFixture)
+{
+    ScanResult res = f._upg.scan();
+    assertDirs({"r0/c0"}, true, res);
+}
+
+TEST("require that no-existing scan directory is handled")
+{
+    DataDirectoryUpgrader upg(SCAN_DIR, DEST_DIR);
+    ScanResult res = upg.scan();
+    assertDirs({}, false, res);
+}
+
+TEST_F("require that empty directory is left untouched", EmptyFixture)
+{
+    UpgradeResult res = f._upg.upgrade(f._upg.scan());
+    EXPECT_EQUAL(DataDirectoryUpgrader::IGNORE, res.getStatus());
+    EXPECT_EQUAL("No directory to upgrade", res.getDesc());
+    DirectoryList dirs = f.getDirs();
+    assertDirs({}, dirs);
+}
+
+TEST_F("require that existing destination directory is left untouched", ExistingDestinationFixture)
+{
+    UpgradeResult res = f._upg.upgrade(f._upg.scan());
+    EXPECT_EQUAL(DataDirectoryUpgrader::IGNORE, res.getStatus());
+    EXPECT_EQUAL("Destination directory 'mytest/n1' is already existing", res.getDesc());
+    DirectoryList dirs = f.getDirs();
+    assertDirs({"n1", "r0"}, dirs);
+}
+
+TEST_F("require that single directory is upgraded", SingleFixture)
+{
+    UpgradeResult res = f._upg.upgrade(f._upg.scan());
+    EXPECT_EQUAL(DataDirectoryUpgrader::COMPLETE, res.getStatus());
+    EXPECT_EQUAL("Moved data from 'mytest/r0/c0' to 'mytest/n1'", res.getDesc());
+    DirectoryList dirs = f.getDirs();
+    std::sort(dirs.begin(), dirs.end());
+    assertDirs({"n1"}, dirs);
+    assertDataFile(DEST_DIR);
+    assertUpgradeFile("mytest/r0/c0", DEST_DIR);
+    assertDowngradeScript("#!/bin/sh\n\n"
+                          "mkdir mytest/r0 || exit 1\n"
+                          "chown yahoo mytest/r0\n"
+                          "mv mytest/n1 mytest/r0/c0\n"
+                          "rm mytest/r0/c0/data-directory-upgrade-source.txt\n"
+                          "rm mytest/r0/c0/data-directory-downgrade.sh\n", DEST_DIR);
+}
+
+TEST_F("require that multiple directories are left untouched", DoubleFixture)
+{
+    UpgradeResult res = f._upg.upgrade(f._upg.scan());
+    EXPECT_EQUAL(DataDirectoryUpgrader::ERROR, res.getStatus());
+    EXPECT_EQUAL("Can only upgrade a single directory, was asked to upgrade 2 ('r0/c0', 'r1/c1')", res.getDesc());
+    DirectoryList dirs = f.getDirs();
+    std::sort(dirs.begin(), dirs.end());
+    assertDirs({"r0", "r1"}, dirs);
+    assertDataFile(SCAN_DIR + "/r0/c0");
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/.gitignore b/searchcore/src/tests/proton/server/disk_mem_usage_filter/.gitignore
new file mode 100644
index 00000000000..ec8610c93cf
--- /dev/null
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/.gitignore
@@ -0,0 +1 @@
+searchcore_disk_mem_usage_filter_test_app
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt b/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt
new file mode 100644
index 00000000000..1d9b0234d76
--- /dev/null
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_disk_mem_usage_filter_test_app
+    SOURCES
+    disk_mem_usage_filter_test.cpp
+    DEPENDS
+    searchcore_server
+)
+vespa_add_target_system_dependency(searchcore_disk_mem_usage_filter_test_app boost boost_system-mt-d)
+vespa_add_target_system_dependency(searchcore_disk_mem_usage_filter_test_app boost boost_filesystem-mt-d)
+vespa_add_test(NAME searchcore_disk_mem_usage_filter_test_app COMMAND searchcore_disk_mem_usage_filter_test_app)
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/DESC b/searchcore/src/tests/proton/server/disk_mem_usage_filter/DESC
new file mode 100644
index 00000000000..fca9e3b7656
--- /dev/null
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/DESC
@@ -0,0 +1 @@
+DiskMemUsageFilter test. Take a look at disk_mem_usage_filter_test.cpp for details.
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/FILES b/searchcore/src/tests/proton/server/disk_mem_usage_filter/FILES
new file mode 100644
index 00000000000..b6cdfc4bffc
--- /dev/null
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/FILES
@@ -0,0 +1 @@
+disk_mem_usage_filter_test.cpp
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
new file mode 100644
index 00000000000..70e559e2d23
--- /dev/null
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("disk_mem_usage_filter_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/server/disk_mem_usage_filter.h>
+
+using proton::DiskMemUsageFilter;
+
+namespace fs = boost::filesystem;
+
+namespace
+{
+
+struct Fixture
+{
+    DiskMemUsageFilter _filter;
+    using State = DiskMemUsageFilter::State;
+    using Config = DiskMemUsageFilter::Config;
+
+    Fixture()
+        : _filter(64 * 1024 * 1024)
+    {
+        _filter.setDiskStats({.capacity = 100, .free = 100, .available=100});
+        _filter.setMemoryStats(vespalib::ProcessMemoryStats(10000000,
+                                                            10000001,
+                                                            10000002,
+                                                            10000003));
+    }
+
+    void testWrite(const vespalib::string &exp) {
+        if (exp.empty()) {
+            EXPECT_TRUE(_filter.acceptWriteOperation());
+            State state = _filter.getAcceptState();
+            EXPECT_TRUE(state.acceptWriteOperation());
+            EXPECT_EQUAL(exp, state.message());
+        } else {
+            EXPECT_FALSE(_filter.acceptWriteOperation());
+            State state = _filter.getAcceptState();
+            EXPECT_FALSE(state.acceptWriteOperation());
+            EXPECT_EQUAL(exp, state.message());
+        }
+    }
+
+    void triggerDiskLimit() {
+        _filter.setDiskStats({.capacity = 100, .free = 20, .available=10});
+    }
+
+    void triggerMemoryLimit()
+    {
+        _filter.setMemoryStats(vespalib::ProcessMemoryStats(58720259,
+                                                            58720258,
+                                                            58720257,
+                                                            58720256));
+    }
+};
+
+}
+
+TEST_F("Check that default filter allows write", Fixture)
+{
+    f.testWrite("");
+}
+
+
+TEST_F("Check that disk limit can be reached", Fixture)
+{
+    f._filter.setConfig(Fixture::Config(1.0, 0.8));
+    f.triggerDiskLimit();
+    f.testWrite("diskLimitReached: { "
+                "action: \"add more content nodes\", "
+                "reason: \""
+                "disk used (0.9) > disk limit (0.8)"
+                "\", "
+                "capacity: 100, free: 20, available: 10, diskLimit: 0.8}");
+}
+
+TEST_F("Check that memory limit can be reached", Fixture)
+{
+    f._filter.setConfig(Fixture::Config(0.8, 1.0));
+    f.triggerMemoryLimit();
+    f.testWrite("memoryLimitReached: { "
+                "action: \"add more content nodes\", "
+                "reason: \""
+                "memory used (0.875) > memory limit (0.8)"
+                "\", "
+                "mapped: { virt: 58720259, rss: 58720258}, "
+                "anonymous: { virt: 58720257, rss: 58720256}, "
+                "physicalMemory: 67108864, memoryLimit : 0.8}");
+}
+
+TEST_F("Check that both disk limit and memory limit can be reached", Fixture)
+{
+    f._filter.setConfig(Fixture::Config(0.8, 0.8));
+    f.triggerMemoryLimit();
+    f.triggerDiskLimit();
+    f.testWrite("memoryLimitReached: { "
+                "action: \"add more content nodes\", "
+                "reason: \""
+                "memory used (0.875) > memory limit (0.8)"
+                "\", "
+                "mapped: { virt: 58720259, rss: 58720258}, "
+                "anonymous: { virt: 58720257, rss: 58720256}, "
+                "physicalMemory: 67108864, memoryLimit : 0.8}, "
+                "diskLimitReached: { "
+                "action: \"add more content nodes\", "
+                "reason: \""
+                "disk used (0.9) > disk limit (0.8)"
+                "\", "
+                "capacity: 100, free: 20, available: 10, diskLimit: 0.8}");
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp
new file mode 100644
index 00000000000..99ef5879682
--- /dev/null
+++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp
@@ -0,0 +1,455 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for documentretriever.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("documentretriever_test");
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/persistence/spi/bucket.h>
+#include <vespa/persistence/spi/result.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
+#include <vespa/searchcore/proton/server/documentretriever.h>
+#include <vespa/searchcore/proton/test/dummy_document_store.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/attribute/attributemanager.h>
+#include <vespa/searchlib/attribute/floatbase.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/predicate_attribute.h>
+#include <vespa/searchlib/attribute/stringbase.h>
+#include <vespa/searchlib/docstore/cachestats.h>
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::ArrayFieldValue;
+using document::BucketId;
+using document::DataType;
+using document::Document;
+using document::DocumentId;
+using document::DocumentType;
+using document::DocumentTypeRepo;
+using document::DoubleFieldValue;
+using document::GlobalId;
+using document::IntFieldValue;
+using document::LongFieldValue;
+using document::PositionDataType;
+using document::PredicateFieldValue;
+using document::StringFieldValue;
+using document::StructFieldValue;
+using document::WeightedSetFieldValue;
+using search::AttributeFactory;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::CacheStats;
+using search::DocumentIdT;
+using search::DocumentMetaData;
+using search::FloatingPointAttribute;
+using search::IDocumentStore;
+using search::IntegerAttribute;
+using search::PredicateAttribute;
+using search::StringAttribute;
+using search::attribute::BasicType;
+using search::attribute::CollectionType;
+using search::attribute::Config;
+using search::attribute::IAttributeVector;
+using search::index::Schema;
+using storage::spi::Bucket;
+using storage::spi::GetResult;
+using storage::spi::PartitionId;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+using vespalib::string;
+using namespace document::config_builder;
+
+using namespace proton;
+
+namespace {
+
+const string doc_type_name = "type_name";
+const char static_field[] = "static field";
+const char dyn_field_i[] = "dynamic int field";
+const char dyn_field_d[] = "dynamic double field";
+const char dyn_field_s[] = "dynamic string field";
+const char dyn_field_n[] = "dynamic null field"; // not in document, not in attribute
+const char dyn_field_nai[] = "dynamic null attr int field"; // in document, not in attribute
+const char dyn_field_nas[] = "dynamic null attr string field"; // in document, not in attribute
+const char position_field[] = "position_field";
+const char zcurve_field[] = "position_field_zcurve";
+const char dyn_field_p[] = "dynamic predicate field";
+const char dyn_arr_field_i[] = "dynamic int array field";
+const char dyn_arr_field_d[] = "dynamic double array field";
+const char dyn_arr_field_s[] = "dynamic string array field";
+const char dyn_arr_field_n[] = "dynamic null array field";
+const char dyn_wset_field_i[] = "dynamic int wset field";
+const char dyn_wset_field_d[] = "dynamic double wset field";
+const char dyn_wset_field_s[] = "dynamic string wset field";
+const char dyn_wset_field_n[] = "dynamic null wset field";
+const DocumentId doc_id("doc:test:1");
+const int32_t static_value = 4;
+const int32_t dyn_value_i = 17;
+const double dyn_value_d = 42.42;
+const char dyn_value_s[] = "Batman & Robin";
+const char static_value_s[] = "Dynamic duo";
+const PredicateFieldValue static_value_p;
+const int32_t dyn_weight = 21;
+const int64_t static_zcurve_value = 1118035438880ll;
+const int64_t dynamic_zcurve_value = 6145423666930817152ll;
+
+struct MyDocumentStore : proton::test::DummyDocumentStore {
+    virtual Document::UP read(DocumentIdT lid,
+                              const DocumentTypeRepo &r) const override {
+        if (lid == 0) {
+            return Document::UP();
+        }
+        const DocumentType *doc_type = r.getDocumentType(doc_type_name);
+        Document::UP doc(new Document(*doc_type, doc_id));
+        ASSERT_TRUE(doc.get());
+        doc->set(static_field, static_value);
+        doc->set(dyn_field_i, static_value);
+        doc->set(dyn_field_s, static_value_s);
+        doc->set(dyn_field_nai, static_value);
+        doc->set(dyn_field_nas, static_value_s);
+        doc->set(zcurve_field, static_zcurve_value);
+        doc->setValue(dyn_field_p, static_value_p);
+        FieldValue::UP fv = PositionDataType::getInstance().createFieldValue();
+        StructFieldValue &pos = static_cast<StructFieldValue &>(*fv);
+        pos.set(PositionDataType::FIELD_X, 42);
+        pos.set(PositionDataType::FIELD_Y, 21);
+        doc->setValue(doc->getField(position_field), *fv);
+
+        return doc;
+    }
+    
+    virtual uint64_t
+    initFlush(uint64_t syncToken) override
+    {
+        return syncToken;
+    }
+};
+
+document::DocumenttypesConfig getRepoConfig() {
+    const int32_t doc_type_id = 787121340;
+
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, doc_type_name,
+                     Struct(doc_type_name + ".header"),
+                     Struct(doc_type_name + ".body")
+                     .addField(static_field, DataType::T_INT)
+                     .addField(dyn_field_i, DataType::T_INT)
+                     .addField(dyn_field_d, DataType::T_DOUBLE)
+                     .addField(dyn_field_s, DataType::T_STRING)
+                     .addField(dyn_field_n, DataType::T_FLOAT)
+                     .addField(dyn_field_nai, DataType::T_INT)
+                     .addField(dyn_field_nas, DataType::T_STRING)
+                     .addField(dyn_field_p, DataType::T_PREDICATE)
+                     .addField(dyn_arr_field_i, Array(DataType::T_INT))
+                     .addField(dyn_arr_field_d, Array(DataType::T_DOUBLE))
+                     .addField(dyn_arr_field_s, Array(DataType::T_STRING))
+                     .addField(dyn_arr_field_n, Array(DataType::T_FLOAT))
+                     .addField(dyn_wset_field_i, Wset(DataType::T_INT))
+                     .addField(dyn_wset_field_d, Wset(DataType::T_DOUBLE))
+                     .addField(dyn_wset_field_s, Wset(DataType::T_STRING))
+                     .addField(dyn_wset_field_n, Wset(DataType::T_FLOAT))
+                     .addField(position_field,
+                               PositionDataType::getInstance().getId())
+                     .addField(zcurve_field, DataType::T_LONG));
+     return builder.config();
+}
+
+BasicType
+convertDataType(Schema::DataType t)
+{
+    switch (t) {
+    case Schema::INT32:
+        return BasicType::INT32;
+    case Schema::INT64:
+        return BasicType::INT64;
+    case Schema::FLOAT:
+        return BasicType::FLOAT;
+    case Schema::DOUBLE:
+        return BasicType::DOUBLE;
+    case Schema::STRING:
+        return BasicType::STRING;
+    case Schema::BOOLEANTREE:
+        return BasicType::PREDICATE;
+    default:
+        throw std::runtime_error(make_string("Data type %u not handled", (uint32_t)t));
+    }
+}
+
+CollectionType
+convertCollectionType(Schema::CollectionType ct)
+{
+    switch (ct) {
+    case Schema::SINGLE:
+        return CollectionType::SINGLE;
+    case Schema::ARRAY:
+        return CollectionType::ARRAY;
+    case Schema::WEIGHTEDSET:
+        return CollectionType::WSET;
+    default:
+        throw std::runtime_error(make_string("Collection type %u not handled", (uint32_t)ct));
+    }
+}
+
+search::attribute::Config
+convertConfig(Schema::DataType t, Schema::CollectionType ct)
+{
+    return search::attribute::Config(convertDataType(t), convertCollectionType(ct));
+}
+
+struct Fixture {
+    DocumentTypeRepo repo;
+    DocumentMetaStoreContext meta_store;
+    const GlobalId &gid;
+    BucketId bucket_id;
+    Timestamp timestamp;
+    DocumentMetaStore::DocId lid;
+    MyDocumentStore doc_store;
+    search::AttributeManager attr_manager;
+    Schema schema;
+    DocTypeName _dtName;
+    DocumentRetriever retriever;
+
+    template <typename T>
+    T *addAttribute(const char *name,
+                      Schema::DataType t, Schema::CollectionType ct) {
+        AttributeVector::SP attrPtr = AttributeFactory::createAttribute(name, convertConfig(t, ct));
+        T *attr = dynamic_cast<T *>(attrPtr.get());
+        AttributeVector::DocId id;
+        attr_manager.add(attrPtr);
+        attr->addReservedDoc();
+        attr->addDoc(id);
+        attr->clearDoc(id);
+        EXPECT_EQUAL(id, lid);
+        schema.addAttributeField(Schema::Field(name, t, ct));
+        attr->commit();
+        return attr;
+    }
+
+    template <typename T, typename U>
+    void addAttribute(const char *name, U val,
+                      Schema::DataType t, Schema::CollectionType ct) {
+        T *attr = addAttribute<T>(name, t, ct);
+        if (ct == Schema::SINGLE) {
+            attr->update(lid, val);
+        } else {
+            attr->append(lid, val + 1, dyn_weight);
+            attr->append(lid, val, dyn_weight);
+        }
+        attr->commit();
+    }
+
+    Fixture()
+        : repo(getRepoConfig()),
+          meta_store(std::make_shared<BucketDBOwner>()),
+          gid(doc_id.getGlobalId()),
+          bucket_id(gid.convertToBucketId()),
+          timestamp(21),
+          lid(),
+          doc_store(),
+          attr_manager(),
+          schema(),
+          _dtName(doc_type_name),
+          retriever(_dtName,
+                    repo, schema, meta_store, attr_manager, doc_store)
+    {
+        typedef DocumentMetaStore::Result Result;
+        meta_store.constructFreeList();
+        Result inspect = meta_store.get().inspect(gid);
+        Result putRes(meta_store.get().put(gid, bucket_id, timestamp, inspect.getLid()));
+        lid = putRes.getLid();
+        ASSERT_TRUE(putRes.ok());
+        Schema::CollectionType ct = Schema::SINGLE;
+        addAttribute<IntegerAttribute>(
+                dyn_field_i, dyn_value_i, Schema::INT32, ct);
+        addAttribute<FloatingPointAttribute>(
+                dyn_field_d, dyn_value_d, Schema::DOUBLE, ct);
+        addAttribute<StringAttribute>(
+                dyn_field_s, dyn_value_s, Schema::STRING, ct);
+        addAttribute<FloatingPointAttribute>(
+                dyn_field_n, Schema::FLOAT, ct);
+        addAttribute<IntegerAttribute>(
+                dyn_field_nai, Schema::INT32, ct);
+        addAttribute<StringAttribute>(
+                dyn_field_nas, Schema::STRING, ct);
+        addAttribute<IntegerAttribute>(
+                zcurve_field, dynamic_zcurve_value, Schema::INT64, ct);
+        PredicateAttribute *attr = addAttribute<PredicateAttribute>(
+                dyn_field_p, Schema::BOOLEANTREE, ct);
+        attr->getIndex().indexEmptyDocument(lid);
+        attr->commit();
+        ct = Schema::ARRAY;
+        addAttribute<IntegerAttribute>(
+                dyn_arr_field_i, dyn_value_i, Schema::INT32, ct);
+        addAttribute<FloatingPointAttribute>(
+                dyn_arr_field_d, dyn_value_d, Schema::DOUBLE, ct);
+        addAttribute<StringAttribute>(
+                dyn_arr_field_s, dyn_value_s, Schema::STRING, ct);
+        addAttribute<FloatingPointAttribute>(
+                dyn_arr_field_n, Schema::FLOAT, ct);
+        ct = Schema::WEIGHTEDSET;
+        addAttribute<IntegerAttribute>(
+                dyn_wset_field_i, dyn_value_i, Schema::INT32, ct);
+        addAttribute<FloatingPointAttribute>(
+                dyn_wset_field_d, dyn_value_d, Schema::DOUBLE, ct);
+        addAttribute<StringAttribute>(
+                dyn_wset_field_s, dyn_value_s, Schema::STRING, ct);
+        addAttribute<FloatingPointAttribute>(
+                dyn_wset_field_n, Schema::FLOAT, ct);
+    }
+};
+
+TEST_F("require that document retriever can retrieve document meta data",
+       Fixture) {
+    DocumentMetaData meta_data = f.retriever.getDocumentMetaData(doc_id);
+    EXPECT_EQUAL(f.lid, meta_data.lid);
+    EXPECT_EQUAL(f.timestamp, meta_data.timestamp);
+}
+
+TEST_F("require that document retriever can retrieve bucket meta data",
+       Fixture) {
+    DocumentMetaData::Vector result;
+    f.retriever.getBucketMetaData(Bucket(f.bucket_id, PartitionId(0)), result);
+    ASSERT_EQUAL(1u, result.size());
+    EXPECT_EQUAL(f.lid, result[0].lid);
+    EXPECT_EQUAL(f.timestamp, result[0].timestamp);
+    result.clear();
+    f.retriever.getBucketMetaData(Bucket(BucketId(f.bucket_id.getId() + 1),
+                                         PartitionId(0)), result);
+    EXPECT_EQUAL(0u, result.size());
+}
+
+TEST_F("require that document retriever can retrieve document", Fixture) {
+    DocumentMetaData meta_data = f.retriever.getDocumentMetaData(doc_id);
+    Document::UP doc = f.retriever.getDocument(meta_data.lid);
+    ASSERT_TRUE(doc.get());
+    EXPECT_EQUAL(doc_id, doc->getId());
+}
+
+template <typename T>
+bool checkFieldValue(FieldValue::UP field_value, typename T::value_type v) {
+    ASSERT_TRUE(field_value.get());
+    T *t_value = dynamic_cast<T *>(field_value.get());
+    ASSERT_TRUE(t_value);
+    return EXPECT_EQUAL(v, t_value->getValue());
+}
+
+template <typename T>
+void checkArray(FieldValue::UP array, typename T::value_type v) {
+    ASSERT_TRUE(array.get());
+    ArrayFieldValue *array_val = dynamic_cast<ArrayFieldValue *>(array.get());
+    ASSERT_TRUE(array_val);
+    ASSERT_EQUAL(2u, array_val->size());
+    T *t_value = dynamic_cast<T *>(&(*array_val)[0]);
+    ASSERT_TRUE(t_value);
+    t_value = dynamic_cast<T *>(&(*array_val)[1]);
+    ASSERT_TRUE(t_value);
+    EXPECT_EQUAL(v, t_value->getValue());
+}
+
+template <typename T>
+void checkWset(FieldValue::UP wset, T v) {
+    ASSERT_TRUE(wset.get());
+    WeightedSetFieldValue *wset_val =
+        dynamic_cast<WeightedSetFieldValue *>(wset.get());
+    ASSERT_TRUE(wset_val);
+    ASSERT_EQUAL(2u, wset_val->size());
+    EXPECT_EQUAL(dyn_weight, wset_val->get(v));
+    EXPECT_EQUAL(dyn_weight, wset_val->get(v + 1));
+}
+
+TEST_F("require that attributes are patched into stored document", Fixture) {
+    DocumentMetaData meta_data = f.retriever.getDocumentMetaData(doc_id);
+    Document::UP doc = f.retriever.getDocument(meta_data.lid);
+    ASSERT_TRUE(doc.get());
+
+    FieldValue::UP value = doc->getValue(static_field);
+    ASSERT_TRUE(value.get());
+    IntFieldValue *int_value = dynamic_cast<IntFieldValue *>(value.get());
+    ASSERT_TRUE(int_value);
+    EXPECT_EQUAL(static_value, int_value->getValue());
+
+    EXPECT_TRUE(checkFieldValue<IntFieldValue>(doc->getValue(static_field), static_value));
+    EXPECT_TRUE(checkFieldValue<IntFieldValue>(doc->getValue(dyn_field_i), dyn_value_i));
+    EXPECT_TRUE(checkFieldValue<DoubleFieldValue>(doc->getValue(dyn_field_d), dyn_value_d));
+    EXPECT_TRUE(checkFieldValue<StringFieldValue>(doc->getValue(dyn_field_s), dyn_value_s));
+    EXPECT_FALSE(doc->getValue(dyn_field_n));
+    EXPECT_FALSE(doc->getValue(dyn_field_nai));
+    EXPECT_FALSE(doc->getValue(dyn_field_nas));
+
+    checkArray<IntFieldValue>(doc->getValue(dyn_arr_field_i), dyn_value_i);
+    checkArray<DoubleFieldValue>(doc->getValue(dyn_arr_field_d), dyn_value_d);
+    checkArray<StringFieldValue>(doc->getValue(dyn_arr_field_s), dyn_value_s);
+    EXPECT_FALSE(doc->getValue(dyn_arr_field_n));
+
+    checkWset(doc->getValue(dyn_wset_field_i), dyn_value_i);
+    checkWset(doc->getValue(dyn_wset_field_d), dyn_value_d);
+    checkWset(doc->getValue(dyn_wset_field_s), dyn_value_s);
+    EXPECT_FALSE(doc->getValue(dyn_wset_field_n));
+}
+
+TEST_F("require that attributes are patched into stored document unless also index field", Fixture) {
+    f.schema.addIndexField(Schema::IndexField(dyn_field_s, Schema::STRING));
+    DocumentMetaData meta_data = f.retriever.getDocumentMetaData(doc_id);
+    Document::UP doc = f.retriever.getDocument(meta_data.lid);
+    ASSERT_TRUE(doc.get());
+    checkFieldValue<StringFieldValue>(doc->getValue(dyn_field_s), static_value_s);
+}
+
+TEST_F("require that position fields are regenerated from zcurves", Fixture) {
+    DocumentMetaData meta_data = f.retriever.getDocumentMetaData(doc_id);
+    Document::UP doc = f.retriever.getDocument(meta_data.lid);
+    ASSERT_TRUE(doc.get());
+
+    FieldValue::UP value = doc->getValue(position_field);
+    ASSERT_TRUE(value.get());
+    StructFieldValue *position = dynamic_cast<StructFieldValue *>(value.get());
+    ASSERT_TRUE(position);
+    FieldValue::UP x = position->getValue(PositionDataType::FIELD_X);
+    FieldValue::UP y = position->getValue(PositionDataType::FIELD_Y);
+    EXPECT_EQUAL(-123096000, static_cast<IntFieldValue&>(*x).getValue());
+    EXPECT_EQUAL(49401000, static_cast<IntFieldValue&>(*y).getValue());
+
+    checkFieldValue<LongFieldValue>(doc->getValue(zcurve_field),
+                                    dynamic_zcurve_value);
+}
+
+TEST_F("require that non-existing lid returns null pointer", Fixture) {
+    Document::UP doc = f.retriever.getDocument(0);
+    ASSERT_FALSE(doc.get());
+}
+
+TEST_F("require that predicate attributes can be retrieved", Fixture) {
+    DocumentMetaData meta_data = f.retriever.getDocumentMetaData(doc_id);
+    Document::UP doc = f.retriever.getDocument(meta_data.lid);
+    ASSERT_TRUE(doc.get());
+
+    FieldValue::UP value = doc->getValue(dyn_field_p);
+    ASSERT_TRUE(value.get());
+    PredicateFieldValue *predicate_value =
+        dynamic_cast<PredicateFieldValue *>(value.get());
+    ASSERT_TRUE(predicate_value);
+}
+
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/server/feeddebugger_test.cpp b/searchcore/src/tests/proton/server/feeddebugger_test.cpp
new file mode 100644
index 00000000000..dfb1e7aa5ef
--- /dev/null
+++ b/searchcore/src/tests/proton/server/feeddebugger_test.cpp
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for feeddebugger.
+
+#include <vespa/log/log.h>
+LOG_SETUP("feeddebugger_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/searchcore/proton/common/feeddebugger.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using document::DocumentId;
+using std::string;
+using namespace proton;
+
+namespace {
+
+const char lid_env_name[] = "VESPA_PROTON_DEBUG_FEED_LID_LIST";
+const char docid_env_name[] = "VESPA_PROTON_DEBUG_FEED_DOCID_LIST";
+
+class EnvSaver {
+    const char *_name;
+    string _value;
+    bool _is_set;
+
+public:
+    EnvSaver(const char *name) : _name(name) {
+        char *val = getenv(_name);
+        _is_set = val;
+        if (val) {
+            _value = val;
+        }
+    }
+    ~EnvSaver() {
+        if (_is_set) {
+            setenv(_name, _value.c_str(), true);
+        } else {
+            unsetenv(_name);
+        }
+    }
+};
+
+TEST("require that when environment variable is not set, debugging is off") {
+    EnvSaver save_lid_env(lid_env_name);
+    EnvSaver save_docid_env(docid_env_name);
+    FeedDebugger debugger;
+    EXPECT_FALSE(debugger.isDebugging());
+}
+
+TEST("require that setting an environment variable turns on lid-specific"
+     " debugging.") {
+    EnvSaver save_lid_env(lid_env_name);
+    EnvSaver save_docid_env(docid_env_name);
+    setenv(lid_env_name, "1,3,5", true);
+
+    FeedDebugger debugger;
+    EXPECT_TRUE(debugger.isDebugging());
+    EXPECT_EQUAL(ns_log::Logger::info, debugger.getDebugLevel(1, 0));
+    EXPECT_EQUAL(ns_log::Logger::spam, debugger.getDebugLevel(2, 0));
+    EXPECT_EQUAL(ns_log::Logger::info, debugger.getDebugLevel(3, 0));
+    EXPECT_EQUAL(ns_log::Logger::spam, debugger.getDebugLevel(4, 0));
+    EXPECT_EQUAL(ns_log::Logger::info, debugger.getDebugLevel(5, 0));
+}
+
+TEST("require that setting an environment variable turns on docid-specific"
+     " debugging.") {
+    EnvSaver save_lid_env(lid_env_name);
+    EnvSaver save_docid_env(docid_env_name);
+    setenv(docid_env_name, "doc:test:foo,doc:test:bar,doc:test:baz", true);
+
+    FeedDebugger debugger;
+    EXPECT_TRUE(debugger.isDebugging());
+    EXPECT_EQUAL(ns_log::Logger::info,
+                 debugger.getDebugLevel(1, DocumentId("doc:test:foo")));
+    EXPECT_EQUAL(ns_log::Logger::info,
+                 debugger.getDebugLevel(1, DocumentId("doc:test:bar")));
+    EXPECT_EQUAL(ns_log::Logger::info,
+                 debugger.getDebugLevel(1, DocumentId("doc:test:baz")));
+    EXPECT_EQUAL(ns_log::Logger::spam,
+                 debugger.getDebugLevel(1, DocumentId("doc:test:qux")));
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp
new file mode 100644
index 00000000000..1d38fe6806a
--- /dev/null
+++ b/searchcore/src/tests/proton/server/feedstates_test.cpp
@@ -0,0 +1,136 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for feedstates.
+
+#include <vespa/log/log.h>
+LOG_SETUP("feedstates_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/testdocrepo.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/server/feedstates.h>
+#include <vespa/searchcore/proton/server/memoryconfigstore.h>
+#include <vespa/searchcore/proton/test/dummy_feed_view.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/buffer.h>
+#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
+
+using document::BucketId;
+using document::DocumentId;
+using document::DocumentTypeRepo;
+using document::TestDocRepo;
+using search::transactionlog::Packet;
+using search::SerialNum;
+using storage::spi::Timestamp;
+using vespalib::ConstBufferRef;
+using vespalib::nbostream;
+using namespace proton;
+
+namespace {
+
+struct MyFeedView : public test::DummyFeedView {
+    TestDocRepo repo;
+    DocumentTypeRepo::SP repo_sp;
+    int remove_handled;
+
+    MyFeedView() : repo_sp(repo.getTypeRepoSp()), remove_handled(0) {}
+
+    virtual const DocumentTypeRepo::SP &getDocumentTypeRepo() const
+    { return repo_sp; }
+    virtual void handleRemove(FeedToken *, const RemoveOperation &)
+    { ++remove_handled; }
+};
+
+struct MyReplayConfig : IReplayConfig {
+    virtual void replayConfig(SerialNum) {}
+    virtual void replayWipeHistory(SerialNum, fastos::TimeStamp) {}
+};
+
+struct InstantExecutor : vespalib::Executor {
+    virtual Task::UP execute(Task::UP task) {
+        task->run();
+        return Task::UP();
+    }
+};
+
+struct Fixture
+{
+    MyFeedView feed_view1;
+    MyFeedView feed_view2;
+    IFeedView *feed_view_ptr;
+    MyReplayConfig replay_config;
+    MemoryConfigStore config_store;
+    BucketDBOwner _bucketDB;
+    bucketdb::BucketDBHandler _bucketDBHandler;
+    ReplayTransactionLogState state;
+
+    Fixture()
+        : feed_view1(),
+          feed_view2(),
+          feed_view_ptr(&feed_view1),
+          replay_config(),
+          config_store(),
+          _bucketDB(),
+          _bucketDBHandler(_bucketDB),
+          state("doctypename", feed_view_ptr, _bucketDBHandler, replay_config,
+                  config_store)
+    {
+    }
+};
+
+struct RemoveOperationContext
+{
+    DocumentId doc_id;
+    RemoveOperation op;
+    nbostream str;
+    std::unique_ptr<Packet> packet;
+
+    RemoveOperationContext(search::SerialNum serial)
+        : doc_id("doc:foo:bar"),
+          op(BucketFactory::getBucketId(doc_id), Timestamp(10), doc_id),
+          str(),
+          packet()
+    {
+        op.serialize(str);
+        ConstBufferRef buf(str.c_str(), str.wp());
+        packet.reset(new Packet());
+        packet->add(Packet::Entry(serial, FeedOperation::REMOVE, buf));
+    }
+};
+
+TEST_F("require that active FeedView can change during replay", Fixture)
+{
+    RemoveOperationContext opCtx(10);
+    PacketWrapper::SP wrap(new PacketWrapper(*opCtx.packet, NULL));
+    InstantExecutor executor;
+
+    EXPECT_EQUAL(0, f.feed_view1.remove_handled);
+    EXPECT_EQUAL(0, f.feed_view2.remove_handled);
+    f.state.receive(wrap, executor);
+    EXPECT_EQUAL(1, f.feed_view1.remove_handled);
+    EXPECT_EQUAL(0, f.feed_view2.remove_handled);
+    f.feed_view_ptr = &f.feed_view2;
+    f.state.receive(wrap, executor);
+    EXPECT_EQUAL(1, f.feed_view1.remove_handled);
+    EXPECT_EQUAL(1, f.feed_view2.remove_handled);
+}
+
+TEST_F("require that replay progress is tracked", Fixture)
+{
+    RemoveOperationContext opCtx(10);
+    TlsReplayProgress progress("test", 5, 15);
+    PacketWrapper::SP wrap(new PacketWrapper(*opCtx.packet, &progress));
+    InstantExecutor executor;
+
+    f.state.receive(wrap, executor);
+    EXPECT_EQUAL(10u, progress.getCurrent());
+    EXPECT_EQUAL(0.5, progress.getProgress());
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/server/health_adapter/.gitignore b/searchcore/src/tests/proton/server/health_adapter/.gitignore
new file mode 100644
index 00000000000..c82499f49b7
--- /dev/null
+++ b/searchcore/src/tests/proton/server/health_adapter/.gitignore
@@ -0,0 +1 @@
+searchcore_health_adapter_test_app
diff --git a/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt b/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt
new file mode 100644
index 00000000000..2fee205f636
--- /dev/null
+++ b/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_health_adapter_test_app
+    SOURCES
+    health_adapter_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_health_adapter_test_app COMMAND searchcore_health_adapter_test_app)
diff --git a/searchcore/src/tests/proton/server/health_adapter/FILES b/searchcore/src/tests/proton/server/health_adapter/FILES
new file mode 100644
index 00000000000..6faa8f6155f
--- /dev/null
+++ b/searchcore/src/tests/proton/server/health_adapter/FILES
@@ -0,0 +1 @@
+health_adapter_test.cpp
diff --git a/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp b/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp
new file mode 100644
index 00000000000..1957d8fbf33
--- /dev/null
+++ b/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchcore/proton/server/health_adapter.h>
+#include <vespa/searchcore/proton/common/statusreport.h>
+
+using namespace proton;
+
+struct MyStatusProducer : public StatusProducer {
+    StatusReport::List list;
+    void add(const std::string &comp, StatusReport::State state,
+             const std::string &msg)
+    {
+        list.push_back(StatusReport::SP(new StatusReport(StatusReport::Params(comp).
+                state(state).message(msg))));
+    }
+    virtual StatusReport::List getStatusReports() const {
+        return list;
+    }
+};
+
+TEST_FF("require that empty status list passes health check", MyStatusProducer(), HealthAdapter(f1)) {
+    EXPECT_TRUE(f2.getHealth().ok);
+    EXPECT_EQUAL(std::string("All OK"), f2.getHealth().msg);
+}
+
+TEST_FF("require that UP components passes health check", MyStatusProducer(), HealthAdapter(f1)) {
+    f1.add("c1", StatusReport::UPOK, "xxx");
+    f1.add("c2", StatusReport::UPOK, "yyy");
+    f1.add("c3", StatusReport::UPOK, "zzz");
+    EXPECT_TRUE(f2.getHealth().ok);
+    EXPECT_EQUAL(std::string("All OK"), f2.getHealth().msg);
+}
+
+TEST_FF("require that PARTIAL component fails health check", MyStatusProducer(), HealthAdapter(f1)) {
+    f1.add("c1", StatusReport::UPOK, "xxx");
+    f1.add("c2", StatusReport::PARTIAL, "yyy");
+    f1.add("c3", StatusReport::UPOK, "zzz");
+    EXPECT_FALSE(f2.getHealth().ok);
+    EXPECT_EQUAL(std::string("c2: yyy"), f2.getHealth().msg);
+}
+
+TEST_FF("require that DOWN component fails health check", MyStatusProducer(), HealthAdapter(f1)) {
+    f1.add("c1", StatusReport::UPOK, "xxx");
+    f1.add("c2", StatusReport::DOWN, "yyy");
+    f1.add("c3", StatusReport::UPOK, "zzz");
+    EXPECT_FALSE(f2.getHealth().ok);
+    EXPECT_EQUAL(std::string("c2: yyy"), f2.getHealth().msg);
+}
+
+TEST_FF("require that multiple failure messages are concatenated", MyStatusProducer(), HealthAdapter(f1)) {
+    f1.add("c1", StatusReport::PARTIAL, "xxx");
+    f1.add("c2", StatusReport::UPOK, "yyy");
+    f1.add("c3", StatusReport::DOWN, "zzz");
+    EXPECT_FALSE(f2.getHealth().ok);
+    EXPECT_EQUAL(std::string("c1: xxx, c3: zzz"), f2.getHealth().msg);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp
new file mode 100644
index 00000000000..301633404bc
--- /dev/null
+++ b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp
@@ -0,0 +1,211 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for memoryconfigstore.
+
+#include <vespa/log/log.h>
+LOG_SETUP("memoryconfigstore_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/server/memoryconfigstore.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/common/schemautil.h>
+
+using search::index::Schema;
+using search::SerialNum;
+using namespace proton;
+
+namespace {
+
+DocumentDBConfig::SP
+getConfig(int64_t generation, const Schema::SP &schema)
+{
+    return DocumentDBConfig::SP(
+            new DocumentDBConfig(
+                    generation,
+                    DocumentDBConfig::RankProfilesConfigSP(),
+                    DocumentDBConfig::IndexschemaConfigSP(),
+                    DocumentDBConfig::AttributesConfigSP(),
+                    DocumentDBConfig::SummaryConfigSP(),
+                    DocumentDBConfig::SummarymapConfigSP(),
+                    DocumentDBConfig::JuniperrcConfigSP(),
+                    DocumentDBConfig::DocumenttypesConfigSP(),
+                    document::DocumentTypeRepo::SP(),
+                    search::TuneFileDocumentDB::SP(),
+                    schema,
+                    DocumentDBMaintenanceConfig::SP(),
+                    "client",
+                    "test"));
+}
+
+
+DocumentDBConfig::SP
+getConfig(int64_t generation)
+{
+    return getConfig(generation, Schema::SP());
+}
+
+
+Schema::SP
+getSchema(int step)
+{
+    Schema::SP schema(new Schema);
+    schema->addIndexField(Schema::IndexField("foo1", Schema::STRING));
+    if (step < 2) {
+        schema->addIndexField(Schema::IndexField("foo2", Schema::STRING));
+    }
+    if (step < 1) {
+        schema->addIndexField(Schema::IndexField("foo3", Schema::STRING));
+    }
+    return schema;
+}
+
+TEST("require that configs can be stored and loaded") {
+    MemoryConfigStore config_store;
+    SerialNum serial(12);
+    config_store.saveConfig(*getConfig(10), Schema(), serial);
+    DocumentDBConfig::SP config;
+    Schema::SP history;
+    config_store.loadConfig(*getConfig(14), serial, config, history);
+    ASSERT_TRUE(config.get());
+    ASSERT_TRUE(history.get());
+    EXPECT_EQUAL(10, config->getGeneration());
+}
+
+TEST("require that best serial number is the most recent one") {
+    MemoryConfigStore config_store;
+    EXPECT_EQUAL(0u, config_store.getBestSerialNum());
+    config_store.saveConfig(*getConfig(10), Schema(), 5);
+    EXPECT_EQUAL(5u, config_store.getBestSerialNum());
+    config_store.saveConfig(*getConfig(10), Schema(), 2);
+    EXPECT_EQUAL(5u, config_store.getBestSerialNum());
+}
+
+TEST("require that oldest serial number is the first one or 0") {
+    MemoryConfigStore config_store;
+    EXPECT_EQUAL(0u, config_store.getOldestSerialNum());
+    config_store.saveConfig(*getConfig(10), Schema(), 5);
+    EXPECT_EQUAL(5u, config_store.getOldestSerialNum());
+    config_store.saveConfig(*getConfig(10), Schema(), 2);
+    EXPECT_EQUAL(2u, config_store.getOldestSerialNum());
+}
+
+TEST("require that existing serial numbers are valid") {
+    MemoryConfigStore config_store;
+    EXPECT_FALSE(config_store.hasValidSerial(5));
+    config_store.saveConfig(*getConfig(10), Schema(), 5);
+    EXPECT_TRUE(config_store.hasValidSerial(5));
+}
+
+TEST("require that prev valid serial number is the last one before the arg") {
+    MemoryConfigStore config_store;
+    EXPECT_EQUAL(0u, config_store.getPrevValidSerial(10));
+    config_store.saveConfig(*getConfig(10), Schema(), 5);
+    EXPECT_EQUAL(5u, config_store.getPrevValidSerial(10));
+    EXPECT_EQUAL(0u, config_store.getPrevValidSerial(5));
+    EXPECT_EQUAL(0u, config_store.getPrevValidSerial(4));
+    config_store.saveConfig(*getConfig(10), Schema(), 2);
+    EXPECT_EQUAL(0u, config_store.getPrevValidSerial(1));
+    EXPECT_EQUAL(0u, config_store.getPrevValidSerial(2));
+    EXPECT_EQUAL(2u, config_store.getPrevValidSerial(4));
+    EXPECT_EQUAL(2u, config_store.getPrevValidSerial(5));
+    EXPECT_EQUAL(5u, config_store.getPrevValidSerial(10));
+}
+
+TEST("require that prune removes old configs") {
+    MemoryConfigStore config_store;
+    config_store.saveConfig(*getConfig(10), Schema(), 5);
+    config_store.saveConfig(*getConfig(10), Schema(), 6);
+    EXPECT_TRUE(config_store.hasValidSerial(5));
+    config_store.prune(5);
+    EXPECT_FALSE(config_store.hasValidSerial(5));
+    EXPECT_TRUE(config_store.hasValidSerial(6));
+    config_store.prune(10);
+    EXPECT_FALSE(config_store.hasValidSerial(6));
+}
+
+TEST("require that wipe history clears previous history schema "
+     "and adds new, identical entry for current serial num") {
+    MemoryConfigStore config_store;
+    Schema::SP history(new Schema);
+    history->addIndexField(Schema::IndexField("foo", Schema::STRING));
+    config_store.saveConfig(*getConfig(10), *history, 5);
+    DocumentDBConfig::SP config;
+    config_store.loadConfig(*getConfig(14), 5, config, history);
+    EXPECT_EQUAL(1u, history->getNumIndexFields());
+    config_store.saveWipeHistoryConfig(6, 0);
+    EXPECT_TRUE(config_store.hasValidSerial(6));
+    config_store.loadConfig(*getConfig(14), 5, config, history);
+    EXPECT_EQUAL(1u, history->getNumIndexFields());
+    config_store.loadConfig(*getConfig(14), 6, config, history);
+    ASSERT_TRUE(config.get());
+    ASSERT_TRUE(history.get());
+    EXPECT_EQUAL(0u, history->getNumIndexFields());
+}
+
+
+TEST("require that wipe history clears only portions of history")
+{
+    MemoryConfigStore config_store;
+    Schema::SP schema(getSchema(0));
+    Schema::SP history(new Schema);
+    DocumentDBConfig::SP config(getConfig(5, schema));
+    config_store.saveConfig(*config, *history, 5);
+    Schema::SP oldSchema(schema);
+    schema = getSchema(1);
+    history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history,
+                                            100);
+    config_store.saveConfig(*config, *history, 10);
+    oldSchema = schema;
+    schema = getSchema(2);
+    history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history,
+                                            200);
+    config_store.saveConfig(*config, *history, 15);
+    config_store.saveWipeHistoryConfig(20, 50);
+    config_store.saveWipeHistoryConfig(25, 100);
+    config_store.saveWipeHistoryConfig(30, 150);
+    config_store.saveWipeHistoryConfig(35, 200);
+    config_store.saveWipeHistoryConfig(40, 250);
+    DocumentDBConfig::SP oldconfig(config);
+    config_store.loadConfig(*oldconfig, 20, config, history);
+    EXPECT_EQUAL(2u, history->getNumIndexFields());
+    oldconfig = config;
+    config_store.loadConfig(*oldconfig, 25, config, history);
+    EXPECT_EQUAL(2u, history->getNumIndexFields());
+    oldconfig = config;
+    config_store.loadConfig(*oldconfig, 30, config, history);
+    EXPECT_EQUAL(1u, history->getNumIndexFields());
+    oldconfig = config;
+    config_store.loadConfig(*oldconfig, 35, config, history);
+    EXPECT_EQUAL(1u, history->getNumIndexFields());
+    oldconfig = config;
+    config_store.loadConfig(*oldconfig, 40, config, history);
+    EXPECT_EQUAL(0u, history->getNumIndexFields());
+}
+
+TEST("require that wipe history does nothing if serial num exists") {
+    MemoryConfigStore config_store;
+    Schema::SP history(new Schema);
+    history->addIndexField(Schema::IndexField("foo", Schema::STRING));
+    config_store.saveConfig(*getConfig(10), *history, 5);
+    DocumentDBConfig::SP config;
+    config_store.saveWipeHistoryConfig(5, 0);
+    config_store.loadConfig(*getConfig(14), 5, config, history);
+    EXPECT_EQUAL(1u, history->getNumIndexFields());
+}
+
+TEST("require that MemoryConfigStores preserves state of "
+     "MemoryConfigStore between instantiations") {
+    MemoryConfigStores config_stores;
+    const std::string name("foo");
+    ConfigStore::UP config_store = config_stores.getConfigStore(name);
+    config_store->saveConfig(*getConfig(10), Schema(), 5);
+    EXPECT_TRUE(config_store->hasValidSerial(5));
+    config_store.reset();
+    config_store = config_stores.getConfigStore(name);
+    EXPECT_TRUE(config_store->hasValidSerial(5));
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/server/memoryflush/.gitignore b/searchcore/src/tests/proton/server/memoryflush/.gitignore
new file mode 100644
index 00000000000..e7a2a22798a
--- /dev/null
+++ b/searchcore/src/tests/proton/server/memoryflush/.gitignore
@@ -0,0 +1 @@
+searchcore_memoryflush_test_app
diff --git a/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt b/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt
new file mode 100644
index 00000000000..51ea36dc077
--- /dev/null
+++ b/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_memoryflush_test_app
+    SOURCES
+    memoryflush_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_flushengine
+)
+vespa_add_test(NAME searchcore_memoryflush_test_app COMMAND searchcore_memoryflush_test_app)
diff --git a/searchcore/src/tests/proton/server/memoryflush/DESC b/searchcore/src/tests/proton/server/memoryflush/DESC
new file mode 100644
index 00000000000..69bfba597c4
--- /dev/null
+++ b/searchcore/src/tests/proton/server/memoryflush/DESC
@@ -0,0 +1 @@
+memoryflush test. Take a look at memoryflush_test.cpp for details.
diff --git a/searchcore/src/tests/proton/server/memoryflush/FILES b/searchcore/src/tests/proton/server/memoryflush/FILES
new file mode 100644
index 00000000000..94ca0c97eb0
--- /dev/null
+++ b/searchcore/src/tests/proton/server/memoryflush/FILES
@@ -0,0 +1 @@
+memoryflush_test.cpp
diff --git a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp
new file mode 100644
index 00000000000..2f4083228f9
--- /dev/null
+++ b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp
@@ -0,0 +1,361 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("memoryflush_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/flushengine/flushcontext.h>
+#include <vespa/searchcore/proton/flushengine/iflushhandler.h>
+#include <vespa/searchcore/proton/flushengine/tls_stats_map.h>
+#include <vespa/searchcore/proton/test/dummy_flush_target.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <vespa/searchcore/proton/server/memoryflush.h>
+
+using fastos::TimeStamp;
+using search::SerialNum;
+using namespace proton;
+using namespace searchcorespi;
+
+namespace
+{
+
+static constexpr uint64_t gibi = UINT64_C(1024) * UINT64_C(1024) * UINT64_C(1024);
+
+}
+
+typedef IFlushTarget::MemoryGain MemoryGain;
+typedef IFlushTarget::DiskGain DiskGain;
+
+class MyFlushHandler : public IFlushHandler {
+public:
+    MyFlushHandler(const vespalib::string &name) : IFlushHandler(name) {}
+    // Implements IFlushHandler
+    virtual std::vector<IFlushTarget::SP> getFlushTargets() {
+        return std::vector<IFlushTarget::SP>();
+    }
+    virtual SerialNum getCurrentSerialNumber() const { return 0; }
+    virtual void flushDone(SerialNum oldestSerial) { (void) oldestSerial; }
+
+    virtual void
+    syncTls(search::SerialNum syncTo)
+    {
+        (void) syncTo;
+    }
+};
+
+class MyFlushTarget : public test::DummyFlushTarget {
+private:
+    MemoryGain _memoryGain;
+    DiskGain   _diskGain;
+    SerialNum  _flushedSerial;
+    TimeStamp  _lastFlushTime;
+    bool       _urgentFlush;
+public:
+    MyFlushTarget(const vespalib::string &name, MemoryGain memoryGain,
+                  DiskGain diskGain, SerialNum flushedSerial,
+                  TimeStamp lastFlushTime, bool urgentFlush) :
+        test::DummyFlushTarget(name),
+        _memoryGain(memoryGain),
+        _diskGain(diskGain),
+        _flushedSerial(flushedSerial),
+        _lastFlushTime(lastFlushTime),
+        _urgentFlush(urgentFlush)
+    {
+    }
+    // Implements IFlushTarget
+    virtual MemoryGain getApproxMemoryGain() const override { return _memoryGain; }
+    virtual DiskGain getApproxDiskGain() const override { return _diskGain; }
+    virtual SerialNum getFlushedSerialNum() const override { return _flushedSerial; }
+    virtual TimeStamp getLastFlushTime() const override { return _lastFlushTime; }
+    virtual bool needUrgentFlush() const override { return _urgentFlush; }
+};
+
+struct StringList : public std::vector<vespalib::string> {
+    StringList() : std::vector<vespalib::string>() {}
+    StringList &add(const vespalib::string &str) {
+        push_back(str);
+        return *this;
+    }
+};
+
+class ContextBuilder {
+private:
+    FlushContext::List _list;
+    IFlushHandler::SP  _handler;
+    flushengine::TlsStatsMap::Map _map;
+    void
+    fixupMap(const vespalib::string &name, SerialNum lastSerial)
+    {
+        flushengine::TlsStats oldStats = _map[name];
+        if (oldStats.getLastSerial() < lastSerial) {
+            _map[name] =
+                flushengine::TlsStats(oldStats.getNumBytes(),
+                                      oldStats.getFirstSerial(),
+                                      lastSerial);
+        }
+    }
+public:
+    ContextBuilder() : _list(), _handler(new MyFlushHandler("myhandler")) {}
+    void addTls(const vespalib::string &name,
+                const flushengine::TlsStats &tlsStats) {
+        _map[name] = tlsStats;
+    }
+    ContextBuilder &add(const FlushContext::SP &context) {
+        _list.push_back(context);
+        fixupMap(_handler->getName(), context->getLastSerial());
+        return *this;
+    }
+    ContextBuilder &add(const IFlushTarget::SP &target, SerialNum lastSerial = 0) {
+        FlushContext::SP ctx(new FlushContext(_handler, target, 0, lastSerial));
+        return add(ctx);
+    }
+    const FlushContext::List &list() const { return _list; }
+    flushengine::TlsStatsMap tlsStats() const {
+        flushengine::TlsStatsMap::Map map(_map);
+        return flushengine::TlsStatsMap(std::move(map));
+    }
+};
+
+MyFlushTarget::SP
+createTargetM(const vespalib::string &name, MemoryGain memoryGain)
+{
+    return MyFlushTarget::SP(new MyFlushTarget(name, memoryGain, DiskGain(),
+                                               SerialNum(), TimeStamp(), false));
+}
+
+MyFlushTarget::SP
+createTargetD(const vespalib::string &name, DiskGain diskGain, SerialNum serial = 0)
+{
+    return MyFlushTarget::SP(new MyFlushTarget(name, MemoryGain(), diskGain,
+                                               serial, TimeStamp(), false));
+}
+
+MyFlushTarget::SP
+createTargetS(const vespalib::string &name, SerialNum serial, TimeStamp timeStamp = TimeStamp())
+{
+    return MyFlushTarget::SP(new MyFlushTarget(name, MemoryGain(), DiskGain(),
+                                               serial, timeStamp, false));
+}
+
+MyFlushTarget::SP
+createTargetT(const vespalib::string &name, TimeStamp lastFlushTime, SerialNum serial = 0)
+{
+    return MyFlushTarget::SP(new MyFlushTarget(name, MemoryGain(), DiskGain(),
+                                               serial, lastFlushTime, false));
+}
+
+MyFlushTarget::SP
+createTargetF(const vespalib::string &name, bool urgentFlush)
+{
+    return MyFlushTarget::SP(new MyFlushTarget(name, MemoryGain(), DiskGain(),
+                                               SerialNum(), TimeStamp(), urgentFlush));
+}
+
+bool
+assertOrder(const StringList &exp, const FlushContext::List &act)
+{
+    if (!EXPECT_EQUAL(exp.size(), act.size())) return false;
+    for (size_t i = 0; i < exp.size(); ++i) {
+        if (!EXPECT_EQUAL(exp[i], act[i]->getTarget()->getName())) return false;
+    }
+    return true;
+}
+
+void
+requireThatWeCanOrderByMemoryGain()
+{
+    ContextBuilder cb;
+    cb.add(createTargetM("t2", MemoryGain(10, 0)))
+      .add(createTargetM("t1", MemoryGain(5, 0)))
+      .add(createTargetM("t4", MemoryGain(20, 0)))
+      .add(createTargetM("t3", MemoryGain(15, 0)));
+    { // target t4 has memoryGain >= maxMemoryGain
+        MemoryFlush flush({1000, 20 * gibi, 1.0, 20, 1.0, 1000, TimeStamp(TimeStamp::MINUTE)});
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+    { // trigger totalMemoryGain >= globalMaxMemory
+        MemoryFlush flush({50, 20 * gibi, 1.0, 1000, 1.0, 1000, TimeStamp(TimeStamp::MINUTE)});
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+}
+
+int64_t milli = 1000000;
+
+void
+requireThatWeCanOrderByDiskGainWithLargeValues()
+{
+    ContextBuilder cb;
+    int64_t before = 100 * milli;
+    cb.add(createTargetD("t2", DiskGain(before, 70 * milli)))  // gain 30M
+      .add(createTargetD("t1", DiskGain(before, 75 * milli)))  // gain 25M
+      .add(createTargetD("t4", DiskGain(before, 45 * milli)))  // gain 55M
+      .add(createTargetD("t3", DiskGain(before, 50 * milli))); // gain 50M
+    { // target t4 has diskGain > bloatValue
+        // t4 gain: 55M / 100M = 0.55 -> bloat factor 0.54 to trigger
+        MemoryFlush flush({1000, 20 * gibi, 10.0, 1000, 0.54, 1000, TimeStamp(TimeStamp::MINUTE)});
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+    { // trigger totalDiskGain > totalBloatValue
+        // total gain: 160M / 4 * 100M = 0.4 -> bloat factor 0.39 to trigger
+        MemoryFlush flush({1000, 20 * gibi, 0.39, 1000, 10.0, 1000, TimeStamp(TimeStamp::MINUTE)});
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+}
+
+void
+requireThatWeCanOrderByDiskGainWithSmallValues()
+{
+    ContextBuilder cb;
+    cb.add(createTargetD("t2", DiskGain(100, 70)))  // gain 30
+      .add(createTargetD("t1", DiskGain(100, 75)))  // gain 25
+      .add(createTargetD("t4", DiskGain(100, 45)))  // gain 55
+      .add(createTargetD("t3", DiskGain(100, 50))); // gain 50
+    // total disk bloat value calculation uses min 100M disk size
+    // target bloat value calculation uses min 10M disk size
+    { // target t4 has diskGain > bloatValue
+        // t4 gain: 55 / 10M = 0.0000055 -> bloat factor 0.0000054 to trigger
+        MemoryFlush flush({1000, 20 * gibi, 10.0, 1000, 0.0000054, 1000, TimeStamp(TimeStamp::MINUTE)});
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+    { // trigger totalDiskGain > totalBloatValue
+        // total gain: 160 / 100M = 0.0000016 -> bloat factor 0.0000015 to trigger
+        MemoryFlush flush({1000, 20 * gibi, 0.0000015, 1000, 10.0, 1000, TimeStamp(TimeStamp::MINUTE)});
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+}
+
+void
+requireThatWeCanOrderBySerialNum()
+{
+    SerialNum lastSerial = 99;
+    ContextBuilder cb;
+    cb.add(createTargetS("t2", 89), lastSerial)
+      .add(createTargetS("t1", 94), lastSerial)
+      .add(createTargetS("t4", 98), lastSerial + 19)
+      .add(createTargetS("t3", 84), lastSerial);
+    { // target t4 has serialDiff >= maxSerialGain
+        MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 20, TimeStamp(TimeStamp::MINUTE)});
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+}
+
+void
+requireThatWeCanOrderByAge()
+{
+    TimeStamp now(fastos::ClockSystem::now());
+    TimeStamp start(now.val() - 20 * TimeStamp::SEC);
+    ContextBuilder cb;
+    cb.add(createTargetT("t2", TimeStamp(now.val() - 10 * TimeStamp::SEC)))
+      .add(createTargetT("t1", TimeStamp(now.val() - 5 * TimeStamp::SEC)))
+      .add(createTargetT("t4", TimeStamp()))
+      .add(createTargetT("t3", TimeStamp(now.val() - 15 * TimeStamp::SEC)));
+
+    { // all targets have timeDiff >= maxTimeGain
+        MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 1000, TimeStamp(2 * TimeStamp::SEC)}, start);
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+    { // no targets have timeDiff >= maxTimeGain
+        MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 1000, TimeStamp(30 * TimeStamp::SEC)}, start);
+        EXPECT_TRUE(assertOrder(StringList(), flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+}
+
+void
+requireThatWeCanOrderByTlsSize()
+{
+    TimeStamp now(fastos::ClockSystem::now());
+    TimeStamp start(now.val() - 20 * TimeStamp::SEC);
+    flushengine::TlsStatsMap::Map tlsMap;
+    ContextBuilder cb;
+    IFlushHandler::SP handler1(std::make_shared<MyFlushHandler>("handler1"));
+    IFlushHandler::SP handler2(std::make_shared<MyFlushHandler>("handler2"));
+    cb.addTls("handler1", {20 * gibi, 1001, 2000 });
+    cb.addTls("handler2", { 5 * gibi, 1001, 2000 });
+    cb.add(std::make_shared<FlushContext>
+           (handler1,
+            createTargetT("t2", TimeStamp(now.val() - 10 * TimeStamp::SEC),
+                          1900),
+            2000, 2000)).
+        add(std::make_shared<FlushContext>
+            (handler2,
+             createTargetT("t1", TimeStamp(now.val() - 5 * TimeStamp::SEC),
+                           1000),
+             2000, 2000)).
+        add(std::make_shared<FlushContext>
+            (handler1,
+             createTargetT("t4", TimeStamp(),
+                           1000),
+             2000, 2000)).
+        add(std::make_shared<FlushContext>
+            (handler2,
+             createTargetT("t3", TimeStamp(now.val() - 15 * TimeStamp::SEC),
+                           1900),
+             2000, 2000));
+    { // sum of tls sizes above limit, trigger sort order based on tls size
+        MemoryFlush flush({1000, 3 * gibi, 1.0, 1000, 1.0, 2000, TimeStamp(2 * TimeStamp::SEC)}, start);
+        EXPECT_TRUE(assertOrder(StringList().add("t4").add("t1").add("t2").add("t3"),
+                                flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+    { // sum of tls sizes below limit
+        MemoryFlush flush({1000, 30 * gibi, 1.0, 1000, 1.0, 2000, TimeStamp(30 * TimeStamp::SEC)}, start);
+        EXPECT_TRUE(assertOrder(StringList(), flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+}
+
+void
+requireThatOrderTypeIsPreserved()
+{
+    TimeStamp now(fastos::ClockSystem::now());
+    TimeStamp ts1(now.val() - 30 * TimeStamp::SEC);
+    TimeStamp ts2(now.val() - 20 * TimeStamp::SEC);
+    TimeStamp ts3(now.val() - 10 * TimeStamp::SEC);
+    TimeStamp maxTimeGain(15 * TimeStamp::SEC);
+    { // MAXAGE VS MAXSERIAL
+        ContextBuilder cb;
+        cb.add(createTargetT("t2", ts2, 5), 14)
+          .add(createTargetS("t1", 4, ts3), 14);
+        MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 10, maxTimeGain}, ts1);
+        EXPECT_TRUE(assertOrder(StringList().add("t1").add("t2"), flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+    { // MAXSERIAL VS DISKBLOAT
+        ContextBuilder cb;
+        cb.add(createTargetS("t2", 4))
+          .add(createTargetD("t1", DiskGain(100 * milli, 80 * milli), 5));
+        MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 0.19, 10, TimeStamp(30 * TimeStamp::SEC)});
+        EXPECT_TRUE(assertOrder(StringList().add("t1").add("t2"), flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+    { // DISKBLOAT VS MEMORY
+        ContextBuilder cb;
+        cb.add(createTargetD("t2", DiskGain(100 * milli, 80 * milli)))
+          .add(createTargetM("t1", MemoryGain(100, 80)));
+        MemoryFlush flush({1000, 20 * gibi, 1.0, 20, 0.19, 1000, TimeStamp(30 * TimeStamp::SEC)});
+        EXPECT_TRUE(assertOrder(StringList().add("t1").add("t2"), flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+    { // urgent flush
+        ContextBuilder cb;
+        cb.add(createTargetF("t2", false))
+          .add(createTargetF("t1", true));
+        MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 1000, TimeStamp(30 * TimeStamp::SEC)});
+        EXPECT_TRUE(assertOrder(StringList().add("t1").add("t2"), flush.getFlushTargets(cb.list(), cb.tlsStats())));
+    }
+}
+
+TEST_MAIN()
+{
+    TEST_DO(requireThatWeCanOrderByMemoryGain());
+    TEST_DO(requireThatWeCanOrderByDiskGainWithLargeValues());
+    TEST_DO(requireThatWeCanOrderByDiskGainWithSmallValues());
+    TEST_DO(requireThatWeCanOrderBySerialNum());
+    TEST_DO(requireThatWeCanOrderByAge());
+    TEST_DO(requireThatWeCanOrderByTlsSize());
+    TEST_DO(requireThatOrderTypeIsPreserved());
+}
+
+
diff --git a/searchcore/src/tests/proton/server/visibility_handler/.gitignore b/searchcore/src/tests/proton/server/visibility_handler/.gitignore
new file mode 100644
index 00000000000..3666e0c37c3
--- /dev/null
+++ b/searchcore/src/tests/proton/server/visibility_handler/.gitignore
@@ -0,0 +1 @@
+searchcore_visibility_handler_test_app
diff --git a/searchcore/src/tests/proton/server/visibility_handler/CMakeLists.txt b/searchcore/src/tests/proton/server/visibility_handler/CMakeLists.txt
new file mode 100644
index 00000000000..f86504c84dc
--- /dev/null
+++ b/searchcore/src/tests/proton/server/visibility_handler/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_visibility_handler_test_app
+    SOURCES
+    visibility_handler_test.cpp
+    DEPENDS
+    searchcore_server
+)
+vespa_add_target_system_dependency(searchcore_visibility_handler_test_app boost boost_system-mt-d)
+vespa_add_target_system_dependency(searchcore_visibility_handler_test_app boost boost_filesystem-mt-d)
+vespa_add_test(NAME searchcore_visibility_handler_test_app COMMAND searchcore_visibility_handler_test_app)
diff --git a/searchcore/src/tests/proton/server/visibility_handler/DESC b/searchcore/src/tests/proton/server/visibility_handler/DESC
new file mode 100644
index 00000000000..588cd8b923e
--- /dev/null
+++ b/searchcore/src/tests/proton/server/visibility_handler/DESC
@@ -0,0 +1 @@
+visibility_handler test. Take a look at visibility_handler_test.cpp for details.
diff --git a/searchcore/src/tests/proton/server/visibility_handler/FILES b/searchcore/src/tests/proton/server/visibility_handler/FILES
new file mode 100644
index 00000000000..8dea2c9d408
--- /dev/null
+++ b/searchcore/src/tests/proton/server/visibility_handler/FILES
@@ -0,0 +1 @@
+visibility_handler_test.cpp
diff --git a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp
new file mode 100644
index 00000000000..2b6b5bccee7
--- /dev/null
+++ b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp
@@ -0,0 +1,188 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("visibility_handler_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/server/visibilityhandler.h>
+#include <vespa/searchcore/proton/test/dummy_feed_view.h>
+#include <vespa/searchcore/proton/test/threading_service_observer.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+#include <vespa/searchlib/common/lambdatask.h>
+
+using search::SerialNum;
+using proton::IGetSerialNum;
+using proton::test::DummyFeedView;
+using proton::ExecutorThreadingService;
+using proton::test::ThreadingServiceObserver;
+using proton::IFeedView;
+using proton::VisibilityHandler;
+using search::makeLambdaTask;
+using fastos::TimeStamp;
+
+namespace {
+
+class MyGetSerialNum : public IGetSerialNum
+{
+    SerialNum _serialNum;
+public:
+    MyGetSerialNum()
+        : _serialNum(0u)
+    {
+    }
+    virtual SerialNum getSerialNum() const override { return _serialNum; }
+    void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; }
+};
+
+
+
+class MyFeedView : public DummyFeedView
+{
+    uint32_t _forceCommitCount;
+    SerialNum _committedSerialNum;
+
+
+public:
+    MyFeedView()
+        : _forceCommitCount(0u),
+          _committedSerialNum(0u)
+    {
+    }
+
+    void forceCommit(SerialNum serialNum) override
+    {
+        EXPECT_TRUE(serialNum >= _committedSerialNum);
+        _committedSerialNum = serialNum;
+        ++_forceCommitCount;
+    }
+
+    uint32_t getForceCommitCount() const { return _forceCommitCount; }
+    SerialNum getCommittedSerialNum() const { return _committedSerialNum; }
+};
+
+
+class Fixture
+{
+public:
+    MyGetSerialNum _getSerialNum;
+    ExecutorThreadingService _writeServiceReal;
+    ThreadingServiceObserver _writeService;
+    std::shared_ptr<MyFeedView> _feedViewReal;
+    vespalib::VarHolder<IFeedView::SP> _feedView;
+    VisibilityHandler _visibilityHandler;
+
+
+    Fixture()
+        : _getSerialNum(),
+          _writeServiceReal(),
+          _writeService(_writeServiceReal),
+          _feedViewReal(std::make_shared<MyFeedView>()),
+          _feedView(_feedViewReal),
+          _visibilityHandler(_getSerialNum, _writeService, _feedView)
+    {
+    }
+
+    void
+    checkCommitPostCondition(uint32_t expForceCommitCount,
+                             SerialNum expCommittedSerialNum,
+                             uint32_t expMasterExecuteCnt,
+                             uint32_t expAttributeFieldWriterSyncCnt)
+    {
+        EXPECT_EQUAL(expForceCommitCount, _feedViewReal->getForceCommitCount());
+        EXPECT_EQUAL(expCommittedSerialNum,
+                     _feedViewReal->getCommittedSerialNum());
+        EXPECT_EQUAL(expMasterExecuteCnt,
+                     _writeService.masterObserver().getExecuteCnt());
+        EXPECT_EQUAL(expAttributeFieldWriterSyncCnt,
+                     _writeService.attributeFieldWriterObserver().getSyncCnt());
+    }
+
+    void
+    testCommit(double visibilityDelay, bool internal,
+               uint32_t expForceCommitCount, SerialNum expCommittedSerialNum,
+               uint32_t expMasterExecuteCnt,
+               uint32_t expAttributeFieldWriterSyncCnt)
+    {
+        _getSerialNum.setSerialNum(10u);
+        _visibilityHandler.setVisibilityDelay(TimeStamp::Seconds(visibilityDelay));
+        if (internal) {
+            VisibilityHandler *visibilityHandler = &_visibilityHandler;
+            auto task = makeLambdaTask([=]() { visibilityHandler->commit(); });
+            _writeService.master().execute(std::move(task));
+        } else {
+            _visibilityHandler.commit();
+        }
+        _writeService.master().sync();
+        checkCommitPostCondition(expForceCommitCount,
+                                 expCommittedSerialNum,
+                                 expMasterExecuteCnt,
+                                 expAttributeFieldWriterSyncCnt);
+    }
+
+    void
+    testCommitAndWait(double visibilityDelay, bool internal,
+                      uint32_t expForceCommitCount,
+                      SerialNum expCommittedSerialNum,
+                      uint32_t expMasterExecuteCnt,
+                      uint32_t expAttributeFieldWriterSyncCnt)
+    {
+        _getSerialNum.setSerialNum(10u);
+        _visibilityHandler.setVisibilityDelay(TimeStamp::Seconds(visibilityDelay));
+        if (internal) {
+            VisibilityHandler *visibilityHandler = &_visibilityHandler;
+            auto task =
+                makeLambdaTask([=]() { visibilityHandler->commitAndWait(); });
+            _writeService.master().execute(std::move(task));
+            _writeService.master().sync();
+        } else {
+            _visibilityHandler.commitAndWait();
+        }
+        checkCommitPostCondition(expForceCommitCount,
+                                 expCommittedSerialNum,
+                                 expMasterExecuteCnt,
+                                 expAttributeFieldWriterSyncCnt);
+    }
+};
+
+}
+
+TEST_F("Check external commit with zero visibility delay", Fixture)
+{
+    f.testCommit(0.0, false, 0u, 0u, 0u, 0u);
+}
+
+TEST_F("Check external commit with nonzero visibility delay", Fixture)
+{
+    f.testCommit(1.0, false, 1u, 10u, 1u, 0u);
+}
+
+TEST_F("Check internal commit with zero visibility delay", Fixture)
+{
+    f.testCommit(0.0, true, 0u, 0u, 1u, 0u);
+}
+
+TEST_F("Check internal commit with nonzero visibility delay", Fixture)
+{
+    f.testCommit(1.0, true, 1u, 10u, 1u, 0u);
+}
+
+TEST_F("Check external commitAndWait with zero visibility delay", Fixture)
+{
+    f.testCommitAndWait(0.0, false, 0u, 0u, 0u, 1u);
+}
+
+TEST_F("Check external commitAndWait with nonzero visibility delay", Fixture)
+{
+    f.testCommitAndWait(1.0, false, 1u, 10u, 1u, 1u);
+}
+
+TEST_F("Check internal commitAndWait with zero visibility delay", Fixture)
+{
+    f.testCommitAndWait(0.0, true, 0u, 0u, 1u, 1u);
+}
+
+TEST_F("Check internal commitAndWait with nonzero visibility delay", Fixture)
+{
+    f.testCommitAndWait(1.0, true, 1u, 10u, 1u, 1u);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/statusreport/.gitignore b/searchcore/src/tests/proton/statusreport/.gitignore
new file mode 100644
index 00000000000..68753df292a
--- /dev/null
+++ b/searchcore/src/tests/proton/statusreport/.gitignore
@@ -0,0 +1 @@
+searchcore_statusreport_test_app
diff --git a/searchcore/src/tests/proton/statusreport/CMakeLists.txt b/searchcore/src/tests/proton/statusreport/CMakeLists.txt
new file mode 100644
index 00000000000..fa11b343d0d
--- /dev/null
+++ b/searchcore/src/tests/proton/statusreport/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_statusreport_test_app
+    SOURCES
+    statusreport.cpp
+    DEPENDS
+)
+vespa_add_test(NAME searchcore_statusreport_test_app COMMAND searchcore_statusreport_test_app)
diff --git a/searchcore/src/tests/proton/statusreport/DESC b/searchcore/src/tests/proton/statusreport/DESC
new file mode 100644
index 00000000000..36c1c49cc80
--- /dev/null
+++ b/searchcore/src/tests/proton/statusreport/DESC
@@ -0,0 +1 @@
+statusreport test. Take a look at statusreport.cpp for details.
diff --git a/searchcore/src/tests/proton/statusreport/FILES b/searchcore/src/tests/proton/statusreport/FILES
new file mode 100644
index 00000000000..fe27097df03
--- /dev/null
+++ b/searchcore/src/tests/proton/statusreport/FILES
@@ -0,0 +1 @@
+statusreport.cpp
diff --git a/searchcore/src/tests/proton/statusreport/statusreport.cpp b/searchcore/src/tests/proton/statusreport/statusreport.cpp
new file mode 100644
index 00000000000..81dfe05fa2c
--- /dev/null
+++ b/searchcore/src/tests/proton/statusreport/statusreport.cpp
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("statusreport_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/common/statusreport.h>
+
+namespace proton {
+
+TEST("require that default status report works")
+{
+    StatusReport sr(StatusReport::Params("foo"));
+
+    EXPECT_EQUAL("foo", sr.getComponent());
+    EXPECT_EQUAL(StatusReport::DOWN, sr.getState());
+    EXPECT_EQUAL("", sr.getInternalState());
+    EXPECT_EQUAL("", sr.getInternalConfigState());
+    EXPECT_FALSE(sr.hasProgress());
+    EXPECT_EQUAL("", sr.getMessage());
+    EXPECT_EQUAL("state=", sr.getInternalStatesStr());
+}
+
+TEST("require that custom status report works")
+{
+    StatusReport sr(StatusReport::Params("foo").
+            state(StatusReport::UPOK).
+            internalState("mystate").
+            internalConfigState("myconfigstate").
+            progress(65).
+            message("mymessage"));
+
+    EXPECT_EQUAL("foo", sr.getComponent());
+    EXPECT_EQUAL(StatusReport::UPOK, sr.getState());
+    EXPECT_EQUAL("mystate", sr.getInternalState());
+    EXPECT_EQUAL("myconfigstate", sr.getInternalConfigState());
+    EXPECT_TRUE(sr.hasProgress());
+    EXPECT_EQUAL(65, sr.getProgress());
+    EXPECT_EQUAL("mymessage", sr.getMessage());
+    EXPECT_EQUAL("state=mystate configstate=myconfigstate", sr.getInternalStatesStr());
+}
+
+}  // namespace proton
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/summaryengine/.gitignore b/searchcore/src/tests/proton/summaryengine/.gitignore
new file mode 100644
index 00000000000..3a635a51def
--- /dev/null
+++ b/searchcore/src/tests/proton/summaryengine/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+summaryengine_test
+searchcore_summaryengine_test_app
diff --git a/searchcore/src/tests/proton/summaryengine/CMakeLists.txt b/searchcore/src/tests/proton/summaryengine/CMakeLists.txt
new file mode 100644
index 00000000000..af3b09db1de
--- /dev/null
+++ b/searchcore/src/tests/proton/summaryengine/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_summaryengine_test_app
+    SOURCES
+    summaryengine.cpp
+    DEPENDS
+    searchcore_summaryengine
+    searchcore_pcommon
+)
+vespa_add_test(NAME searchcore_summaryengine_test_app COMMAND searchcore_summaryengine_test_app)
diff --git a/searchcore/src/tests/proton/summaryengine/DESC b/searchcore/src/tests/proton/summaryengine/DESC
new file mode 100644
index 00000000000..be0687ecb07
--- /dev/null
+++ b/searchcore/src/tests/proton/summaryengine/DESC
@@ -0,0 +1 @@
+summaryengine test. Take a look at summaryengine.cpp for details.
diff --git a/searchcore/src/tests/proton/summaryengine/FILES b/searchcore/src/tests/proton/summaryengine/FILES
new file mode 100644
index 00000000000..cef9a6e88bf
--- /dev/null
+++ b/searchcore/src/tests/proton/summaryengine/FILES
@@ -0,0 +1 @@
+summaryengine.cpp
diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
new file mode 100644
index 00000000000..c0692ecd7ec
--- /dev/null
+++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
@@ -0,0 +1,434 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("summaryengine_test");
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/summaryengine/summaryengine.h>
+#include <vespa/searchcore/proton/summaryengine/docsum_by_slime.h>
+#include <vespa/searchlib/engine/docsumapi.h>
+#include <vespa/searchlib/util/rawbuf.h>
+#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/data/databuffer.h>
+#include <vespa/document/util/compressor.h>
+
+using namespace search::engine;
+using namespace document;
+using namespace vespalib::slime;
+using vespalib::stringref;
+using vespalib::ConstBufferRef;
+using vespalib::DataBuffer;
+
+namespace proton {
+
+namespace {
+stringref MYREPLY("myreply");
+Memory DOCSUMS("docsums");
+Memory DOCSUM("docsum");
+}
+
+class MySearchHandler : public ISearchHandler {
+    std::string _name;
+    stringref _reply;
+public:
+    MySearchHandler(const std::string & name = "my",
+                    const stringref & reply = MYREPLY) :
+        _name(name), _reply(reply) {}
+    virtual DocsumReply::UP getDocsums(const DocsumRequest & request) {
+        return (request.useRootSlime())
+            ? std::make_unique<DocsumReply>(createSlimeReply(request.hits.size()))
+            : createOldDocSum(request);
+    }
+    vespalib::Slime::UP createSlimeReply(size_t count) { 
+        vespalib::Slime::UP response(std::make_unique<vespalib::Slime>());
+        Cursor & root = response->setObject();
+        Cursor & array = root.setArray(DOCSUMS);
+        const Symbol docsumSym = response->insert(DOCSUM);
+        for (size_t i=0; i < count; i++) {
+            Cursor & docSumC = array.addObject();
+            ObjectSymbolInserter inserter(docSumC, docsumSym);
+            inserter.insertObject().setLong("long", 982);
+        }
+        return response;
+    }
+    DocsumReply::UP createOldDocSum(const DocsumRequest & request) { 
+        DocsumReply::UP retval(new DocsumReply());
+        for (size_t i=0; i < request.hits.size(); i++) {
+            const DocsumRequest::Hit & h = request.hits[i];
+            DocsumReply::Docsum docsum;
+            docsum.docid = 10 + i;
+            docsum.gid = h.gid;
+            docsum.setData(_reply.c_str(), _reply.size());
+            retval->docsums.push_back(docsum);
+        }
+        return retval;
+    }
+
+    virtual search::engine::SearchReply::UP match(
+            const ISearchHandler::SP &,
+            const search::engine::SearchRequest &,
+            vespalib::ThreadBundle &) const {
+        return SearchReply::UP(new SearchReply);
+    }
+};
+
+class MyDocsumClient : public DocsumClient {
+private:
+    vespalib::Monitor _monitor;
+    DocsumReply::UP   _reply;
+
+public:
+    void getDocsumsDone(DocsumReply::UP reply) {
+        vespalib::MonitorGuard guard(_monitor);
+        _reply = std::move(reply);
+        guard.broadcast();
+    }
+
+    DocsumReply::UP getReply(uint32_t millis) {
+        vespalib::MonitorGuard guard(_monitor);
+        vespalib::TimedWaiter waiter(guard, millis);
+        while (_reply.get() == NULL && waiter.hasTime()) {
+            waiter.wait();
+        }
+        return std::move(_reply);
+    }
+};
+
+class Test : public vespalib::TestApp {
+private:
+    bool assertDocsumReply(SummaryEngine & engine,
+                           const std::string & searchDocType,
+                           const stringref & expReply);
+
+    void requireThatGetDocsumsExecute();
+    void requireThatHandlersAreStored();
+    void requireThatCorrectHandlerIsUsed();
+    void requireThatSlimeRequestIsConvertedCorrectly();
+    void requireThatSlimeInterfaceWorksFine();
+    void requireThatRPCInterfaceWorks();
+public:
+    int Main();
+};
+
+DocsumRequest::UP
+createRequest(size_t num=1)
+{
+    DocsumRequest::UP r(new DocsumRequest());
+    if (num == 1) {
+        r->hits.emplace_back(GlobalId("aaaaaaaaaaaa"));
+    } else {
+        for (size_t i=0; i < num; i++) {
+            vespalib::string s = vespalib::make_string("aaaaaaaaaaa%c", char('a' + i%26));
+            r->hits.push_back(GlobalId(s.c_str()));
+        }
+    }
+    return r;
+}
+
+void
+Test::requireThatGetDocsumsExecute()
+{
+    int numSummaryThreads = 2;
+    SummaryEngine engine(numSummaryThreads);
+    ISearchHandler::SP handler(new MySearchHandler);
+    DocTypeName dtnvfoo("foo");
+    engine.putSearchHandler(dtnvfoo, handler);
+
+    MyDocsumClient client;
+    { // async call when engine running
+        DocsumRequest::Source request(createRequest());
+        DocsumReply::UP reply = engine.getDocsums(std::move(request), client);
+        EXPECT_TRUE(reply.get() == NULL);
+        reply = client.getReply(10000);
+        EXPECT_TRUE(reply.get() != NULL);
+        EXPECT_EQUAL(1u, reply->docsums.size());
+        EXPECT_EQUAL(10u, reply->docsums[0].docid);
+        EXPECT_EQUAL(GlobalId("aaaaaaaaaaaa"), reply->docsums[0].gid);
+        EXPECT_EQUAL("myreply", std::string(reply->docsums[0].data.c_str(), reply->docsums[0].data.size()));
+    }
+    engine.close();
+    { // sync call when engine closed
+        DocsumRequest::Source request(createRequest());
+        DocsumReply::UP reply = engine.getDocsums(std::move(request), client);
+        EXPECT_TRUE(reply.get() != NULL);
+    }
+}
+
+void
+Test::requireThatHandlersAreStored()
+{
+    DocTypeName dtnvfoo("foo");
+    DocTypeName dtnvbar("bar");
+    int numSummaryThreads = 2;
+    SummaryEngine engine(numSummaryThreads);
+    ISearchHandler::SP h1(new MySearchHandler("foo"));
+    ISearchHandler::SP h2(new MySearchHandler("bar"));
+    ISearchHandler::SP h3(new MySearchHandler("baz"));
+    // not found
+    EXPECT_TRUE(engine.getSearchHandler(dtnvfoo).get() == NULL);
+    EXPECT_TRUE(engine.removeSearchHandler(dtnvfoo).get() == NULL);
+    // put & get
+    EXPECT_TRUE(engine.putSearchHandler(dtnvfoo, h1).get() == NULL);
+    EXPECT_EQUAL(engine.getSearchHandler(dtnvfoo).get(), h1.get());
+    EXPECT_TRUE(engine.putSearchHandler(dtnvbar, h2).get() == NULL);
+    EXPECT_EQUAL(engine.getSearchHandler(dtnvbar).get(), h2.get());
+    // replace
+    EXPECT_TRUE(engine.putSearchHandler(dtnvfoo, h3).get() == h1.get());
+    EXPECT_EQUAL(engine.getSearchHandler(dtnvfoo).get(), h3.get());
+    // remove
+    EXPECT_EQUAL(engine.removeSearchHandler(dtnvfoo).get(), h3.get());
+    EXPECT_TRUE(engine.getSearchHandler(dtnvfoo).get() == NULL);
+}
+
+bool
+Test::assertDocsumReply(SummaryEngine & engine, const std::string & searchDocType, const stringref & expReply)
+{
+    DocsumRequest::UP request(createRequest());
+    request->propertiesMap.lookupCreate(search::MapNames::MATCH).add("documentdb.searchdoctype", searchDocType);
+    MyDocsumClient client;
+    engine.getDocsums(DocsumRequest::Source(std::move(request)), client);
+    DocsumReply::UP reply = client.getReply(10000);
+    return EXPECT_EQUAL(vespalib::stringref(expReply), vespalib::stringref(reply->docsums[0].data.c_str(), reply->docsums[0].data.size()));
+}
+
+void
+Test::requireThatCorrectHandlerIsUsed()
+{
+    DocTypeName dtnvfoo("foo");
+    DocTypeName dtnvbar("bar");
+    DocTypeName dtnvbaz("baz");
+    SummaryEngine engine(1);
+    ISearchHandler::SP h1(new MySearchHandler("foo", "foo reply"));
+    ISearchHandler::SP h2(new MySearchHandler("bar", "bar reply"));
+    ISearchHandler::SP h3(new MySearchHandler("baz", "baz reply"));
+    engine.putSearchHandler(dtnvfoo, h1);
+    engine.putSearchHandler(dtnvbar, h2);
+    engine.putSearchHandler(dtnvbaz, h3);
+
+    EXPECT_TRUE(assertDocsumReply(engine, "foo", "foo reply"));
+    EXPECT_TRUE(assertDocsumReply(engine, "bar", "bar reply"));
+    EXPECT_TRUE(assertDocsumReply(engine, "baz", "baz reply"));
+    EXPECT_TRUE(assertDocsumReply(engine, "not", "bar reply")); // uses the first (sorted on name)
+}
+
+using vespalib::Slime;
+
+const char *GID1 = "abcdefghijkl";
+const char *GID2 = "bcdefghijklm";
+
+void
+verify(vespalib::stringref exp, const Slime & slime)
+{
+    Memory expMemory(exp);
+    vespalib::Slime expSlime;
+    size_t used = vespalib::slime::JsonFormat::decode(expMemory, expSlime);
+    EXPECT_EQUAL(used, expMemory.size);
+    SimpleBuffer output;
+    vespalib::slime::JsonFormat::encode(slime, output, true);
+    Slime reSlimed;
+    used = vespalib::slime::JsonFormat::decode(output.get(), reSlimed);
+    EXPECT_EQUAL(used, output.get().size);
+    EXPECT_EQUAL(expSlime, reSlimed);
+}
+
+Slime
+createSlimeRequestLarger(size_t num)
+{
+    Slime r;
+    Cursor & root = r.setObject();
+    root.setString("class", "your-summary");
+    Cursor & array = root.setArray("gids");
+    for (size_t i(0); i < num; i++) {
+        array.addData(Memory(GID1, 12));
+        array.addData(Memory(GID2, 12));
+    }
+    return std::move(r);
+}
+
+Slime
+createSlimeRequest()
+{
+    return createSlimeRequestLarger(1);
+}
+
+void
+Test::requireThatSlimeRequestIsConvertedCorrectly()
+{
+    vespalib::Slime slimeRequest = createSlimeRequest();
+    TEST_DO(verify("{"
+                   "    class: 'your-summary',"
+                   "    gids: ["
+                   "        '0x6162636465666768696A6B6C',"
+                   "        '0x62636465666768696A6B6C6D'"
+                   "    ]"
+                   "}", slimeRequest));
+    DocsumRequest::UP r = DocsumBySlime::slimeToRequest(slimeRequest.get());
+    EXPECT_EQUAL("your-summary", r->resultClassName);
+    EXPECT_EQUAL(2u, r->hits.size());
+    EXPECT_EQUAL(GlobalId(GID1), r->hits[0].gid);
+    EXPECT_EQUAL(GlobalId(GID2), r->hits[1].gid);
+}
+
+void
+createSummary(search::RawBuf & buf)
+{
+    vespalib::Slime summary;
+    summary.setObject().setLong("long", 982);
+    uint32_t magic = search::fs4transport::SLIME_MAGIC_ID;
+    buf.append(&magic, sizeof(magic));
+    search::SlimeOutputRawBufAdapter adapter(buf);
+    BinaryFormat::encode(summary, adapter);
+}
+
+class BaseServer
+{
+protected:
+    BaseServer() :
+        buf(100)
+    {
+        createSummary(buf);
+    }
+protected:
+    search::RawBuf      buf;
+};
+class Server : public BaseServer {
+public:
+    Server() :
+        BaseServer(),
+        engine(2),
+        handler(new MySearchHandler("slime", stringref(buf.GetDrainPos(), buf.GetUsedLen()))),
+        docsumBySlime(engine),
+        docsumByRPC(docsumBySlime)
+    {
+        DocTypeName dtnvfoo("foo");
+        engine.putSearchHandler(dtnvfoo, handler);
+    };
+private:
+    SummaryEngine       engine;
+    ISearchHandler::SP  handler;
+public:
+    DocsumBySlime       docsumBySlime;
+    DocsumByRPC         docsumByRPC;
+};
+
+vespalib::string
+getAnswer(size_t num)
+{
+    vespalib::string s =
+           "{"
+           "    docsums: [";
+    for (size_t i(1); i < num*2; i++) {
+        s +=
+           "        {"
+           "            docsum: {"
+           "                long: 982"
+           "            }"
+           "        },";
+    }
+    s +=
+           "        {"
+           "            docsum: {"
+           "                long: 982"
+           "            }"
+           "        }"
+           "    ]";
+    s +=   "}";
+    return s;
+}
+
+void
+Test::requireThatSlimeInterfaceWorksFine()
+{
+    Server server;
+    vespalib::Slime slimeRequest = createSlimeRequest();
+    vespalib::Slime::UP response = server.docsumBySlime.getDocsums(slimeRequest.get());
+    TEST_DO(verify("{"
+                   "    docsums: ["
+                   "        {"
+                   "            docsum: {"
+                   "                long: 982"
+                   "            }"
+                   "        },"
+                   "        {"
+                   "            docsum: {"
+                   "                long: 982"
+                   "            }"
+                   "        }"
+                   "    ]"
+                   "}", *response));
+}
+
+void
+verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t orgSize, size_t compressedSize, FRT_RPCRequest * request)
+{
+    FRT_Values &ret = *request->GetReturn();
+    EXPECT_EQUAL(encoding, ret[0]._intval8);
+    EXPECT_EQUAL(orgSize, ret[1]._intval32);
+    EXPECT_EQUAL(compressedSize, ret[2]._data._len);
+
+    DataBuffer uncompressed;
+    ConstBufferRef blob(ret[2]._data._buf, ret[2]._data._len);
+    document::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false);
+    EXPECT_EQUAL(orgSize, uncompressed.getDataLen());
+
+    vespalib::Slime summaries;
+    BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), summaries);
+    TEST_DO(verify(getAnswer(count), summaries));
+}
+
+void
+verifyRPC(size_t count,
+          document::CompressionConfig::Type requestCompression, size_t requestSize, size_t requestBlobSize,
+          document::CompressionConfig::Type replyCompression, size_t replySize, size_t replyBlobSize)
+{
+    Server server;
+    vespalib::Slime slimeRequest = createSlimeRequestLarger(count);
+    SimpleBuffer buf; 
+    BinaryFormat::encode(slimeRequest, buf);
+    EXPECT_EQUAL(requestSize, buf.get().size);
+
+    CompressionConfig config(requestCompression, 9, 100);
+    DataBuffer compressed(const_cast<char *>(buf.get().data), buf.get().size);
+    CompressionConfig::Type type = document::compress(config, ConstBufferRef(buf.get().data, buf.get().size), compressed, true);
+    EXPECT_EQUAL(type, requestCompression);
+
+    FRT_RPCRequest * request = new FRT_RPCRequest();
+    FRT_Values &arg = *request->GetParams();
+    arg.AddInt8(type);
+    arg.AddInt32(buf.get().size);
+    arg.AddData(compressed.getData(), compressed.getDataLen());
+    EXPECT_EQUAL(requestBlobSize, compressed.getDataLen());
+
+    server.docsumByRPC.getDocsums(*request);
+    verifyReply(count, replyCompression, replySize, replyBlobSize, request);
+
+    request->SubRef();
+}
+
+void
+Test::requireThatRPCInterfaceWorks()
+{
+    verifyRPC(1, document::CompressionConfig::NONE, 55, 55, document::CompressionConfig::NONE, 38, 38);
+    verifyRPC(100, document::CompressionConfig::NONE, 2631, 2631, document::CompressionConfig::LZ4, 1426, 46);
+    verifyRPC(100, document::CompressionConfig::LZ4, 2631, 69, document::CompressionConfig::LZ4, 1426, 46);
+}
+
+int
+Test::Main()
+{
+    TEST_INIT("summaryengine_test");
+
+    requireThatGetDocsumsExecute();
+    requireThatHandlersAreStored();
+    requireThatCorrectHandlerIsUsed();
+    requireThatSlimeRequestIsConvertedCorrectly();
+    requireThatSlimeInterfaceWorksFine();
+    requireThatRPCInterfaceWorks();
+
+    TEST_DONE();
+}
+
+}
+
+TEST_APPHOOK(proton::Test);
+
diff --git a/searchcore/src/tests/proton/verify_ranksetup/.cvsignore b/searchcore/src/tests/proton/verify_ranksetup/.cvsignore
new file mode 100644
index 00000000000..a4848db32f8
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/.cvsignore
@@ -0,0 +1,3 @@
+.depend
+Makefile
+verify_ranksetup_test
diff --git a/searchcore/src/tests/proton/verify_ranksetup/.gitignore b/searchcore/src/tests/proton/verify_ranksetup/.gitignore
new file mode 100644
index 00000000000..1142087d03d
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/.gitignore
@@ -0,0 +1,5 @@
+*_test
+.depend
+Makefile
+generated
+searchcore_verify_ranksetup_test_app
diff --git a/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt b/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt
new file mode 100644
index 00000000000..2d74c323c1a
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_verify_ranksetup_test_app
+    SOURCES
+    verify_ranksetup_test.cpp
+    DEPENDS
+)
+vespa_add_test(NAME searchcore_verify_ranksetup_test_app COMMAND sh verify_ranksetup_test.sh)
diff --git a/searchcore/src/tests/proton/verify_ranksetup/DESC b/searchcore/src/tests/proton/verify_ranksetup/DESC
new file mode 100644
index 00000000000..700e2b1c2f9
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/DESC
@@ -0,0 +1 @@
+verify_ranksetup test. Take a look at verify_ranksetup.cpp for details.
diff --git a/searchcore/src/tests/proton/verify_ranksetup/FILES b/searchcore/src/tests/proton/verify_ranksetup/FILES
new file mode 100644
index 00000000000..9c4a2ef3776
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/FILES
@@ -0,0 +1 @@
+verify_ranksetup.cpp
diff --git a/searchcore/src/tests/proton/verify_ranksetup/invalid_attr_name/.gitignore b/searchcore/src/tests/proton/verify_ranksetup/invalid_attr_name/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/invalid_attr_name/.gitignore
diff --git a/searchcore/src/tests/proton/verify_ranksetup/invalid_feature_name/.gitignore b/searchcore/src/tests/proton/verify_ranksetup/invalid_feature_name/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/invalid_feature_name/.gitignore
diff --git a/searchcore/src/tests/proton/verify_ranksetup/unsupported_collection_type/.gitignore b/searchcore/src/tests/proton/verify_ranksetup/unsupported_collection_type/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/unsupported_collection_type/.gitignore
diff --git a/searchcore/src/tests/proton/verify_ranksetup/valid/.gitignore b/searchcore/src/tests/proton/verify_ranksetup/valid/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/valid/.gitignore
diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp
new file mode 100644
index 00000000000..3db23380675
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp
@@ -0,0 +1,250 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/util/slaveproc.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchlib/fef/indexproperties.h>
+#include <string>
+#include <vector>
+#include <map>
+#include <initializer_list>
+
+const char *prog = "../../../apps/verify_ranksetup/verify_ranksetup-bin";
+const std::string gen_dir("generated");
+
+const char *valid_feature = "value(0)";
+const char *invalid_feature = "invalid_feature_name and format";
+
+using search::index::Schema;
+using namespace search::fef::indexproperties;
+
+struct Writer {
+    FILE *file;
+    Writer(const std::string &file_name) {
+        file = fopen(file_name.c_str(), "w");
+        ASSERT_TRUE(file != 0);
+    }
+    void fmt(const char *format, ...) const
+#ifdef __GNUC__
+        __attribute__ ((format (printf,2,3)))
+#endif
+    {
+        va_list ap;
+        va_start(ap, format);
+        vfprintf(file, format, ap);
+        va_end(ap);
+    }
+    ~Writer() { fclose(file); }
+};
+
+void verify_dir() {
+    std::string pwd(getenv("PWD"));
+    ASSERT_NOT_EQUAL(pwd.find("searchcore/src/tests/proton/verify_ranksetup"), pwd.npos);
+}
+
+//-----------------------------------------------------------------------------
+
+struct Model {
+    std::map<std::string,std::pair<std::string,std::string> > indexes;
+    std::map<std::string,std::pair<std::string,std::string> > attributes;
+    std::map<std::string,std::string>                         properties;
+    std::vector<bool>                                         extra_profiles;
+    Model() : indexes(), attributes(), properties(), extra_profiles() {
+        verify_dir();
+    }
+    void index(const std::string &name, Schema::DataType data_type,
+               Schema::CollectionType collection_type)
+    {
+        indexes[name].first = Schema::getTypeName(data_type);
+        indexes[name].second = Schema::getTypeName(collection_type);
+    }
+    void attribute(const std::string &name, Schema::DataType data_type,
+                   Schema::CollectionType collection_type)
+    {
+        attributes[name].first = Schema::getTypeName(data_type);
+        attributes[name].second = Schema::getTypeName(collection_type);
+    }
+    void property(const std::string &name, const std::string &val) {
+        properties[name] = val;
+    }
+    void first_phase(const std::string &feature) {
+        property(rank::FirstPhase::NAME, feature);
+    }
+    void second_phase(const std::string &feature) {
+        property(rank::SecondPhase::NAME, feature);
+    }
+    void summary_feature(const std::string &feature) {
+        property(summary::Feature::NAME, feature);
+    }
+    void dump_feature(const std::string &feature) {
+        property(dump::Feature::NAME, feature);
+    }
+    void good_profile() {
+        extra_profiles.push_back(true);
+    }
+    void bad_profile() {
+        extra_profiles.push_back(false);
+    }
+    void write_attributes(const Writer &out) {
+        out.fmt("attribute[%zu]\n", attributes.size());
+        std::map<std::string,std::pair<std::string,std::string> >::const_iterator pos = attributes.begin();
+        for (size_t i = 0; pos != attributes.end(); ++pos, ++i) {
+            out.fmt("attribute[%zu].name \"%s\"\n", i, pos->first.c_str());
+            out.fmt("attribute[%zu].datatype %s\n", i, pos->second.first.c_str());
+            out.fmt("attribute[%zu].collectiontype %s\n", i, pos->second.second.c_str());
+        }
+    }
+    void write_indexschema(const Writer &out) {
+        out.fmt("indexfield[%zu]\n", indexes.size());
+        std::map<std::string,std::pair<std::string,std::string> >::const_iterator pos = indexes.begin();
+        for (size_t i = 0; pos != indexes.end(); ++pos, ++i) {
+            out.fmt("indexfield[%zu].name \"%s\"\n", i, pos->first.c_str());
+            out.fmt("indexfield[%zu].datatype %s\n", i, pos->second.first.c_str());
+            out.fmt("indexfield[%zu].collectiontype %s\n", i, pos->second.second.c_str());
+        }
+    }
+    void write_rank_profiles(const Writer &out) {
+        out.fmt("rankprofile[%zu]\n", extra_profiles.size() + 1);
+        out.fmt("rankprofile[0].name \"default\"\n");
+        std::map<std::string,std::string>::const_iterator pos = properties.begin();
+        for (size_t i = 0; pos != properties.end(); ++pos, ++i) {
+            out.fmt("rankprofile[0].fef.property[%zu]\n", properties.size());
+            out.fmt("rankprofile[0].fef.property[%zu].name \"%s\"\n", i, pos->first.c_str());
+            out.fmt("rankprofile[0].fef.property[%zu].value \"%s\"\n", i, pos->second.c_str());
+        }
+        for (size_t i = 1; i < (extra_profiles.size() + 1); ++i) {
+            out.fmt("rankprofile[%zu].name \"extra_%zu\"\n", i, i);
+            out.fmt("rankprofile[%zu].fef.property[%zu].name \"%s\"\n", i, i, rank::FirstPhase::NAME.c_str());
+            out.fmt("rankprofile[%zu].fef.property[%zu].value \"%s\"\n", i, i, extra_profiles[i-1]?valid_feature:invalid_feature);
+        }
+    }
+    void generate() {
+        write_attributes(Writer(gen_dir + "/attributes.cfg"));
+        write_indexschema(Writer(gen_dir + "/indexschema.cfg"));
+        write_rank_profiles(Writer(gen_dir + "/rank-profiles.cfg"));
+    }
+    bool verify() {
+        generate();
+        return vespalib::SlaveProc::run(vespalib::make_string("%s dir:%s", prog, gen_dir.c_str()).c_str());
+    }
+    void verify_valid(std::initializer_list<std::string> features) {
+        for (const std::string &f: features) {
+            first_phase(f);
+            if (!EXPECT_TRUE(verify())) {
+                fprintf(stderr, "--> feature '%s' was invalid (should be valid)\n", f.c_str());
+            }
+        }
+    }
+    void verify_invalid(std::initializer_list<std::string> features) {
+        for (const std::string &f: features) {
+            first_phase(f);
+            if (!EXPECT_FALSE(verify())) {
+                fprintf(stderr, "--> feature '%s' was valid (should be invalid)\n", f.c_str());
+            }
+        }
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+struct EmptyModel : Model {};
+
+struct SimpleModel : Model {
+    SimpleModel() : Model() {
+        index("title", Schema::STRING, Schema::SINGLE);
+        index("list", Schema::STRING, Schema::ARRAY);
+        index("keywords", Schema::STRING, Schema::WEIGHTEDSET);
+        attribute("date", Schema::INT32, Schema::SINGLE);
+    }
+};
+
+struct ShadowModel : Model {
+    ShadowModel() : Model() {
+        index("both", Schema::STRING, Schema::SINGLE);
+        attribute("both", Schema::STRING, Schema::SINGLE);
+    }
+};
+
+TEST_F("print usage", Model()) {
+    EXPECT_FALSE(vespalib::SlaveProc::run(vespalib::make_string("%s", prog).c_str()));
+}
+
+TEST_F("setup output directory", Model()) {
+    ASSERT_TRUE(vespalib::SlaveProc::run(vespalib::make_string("rm -rf %s", gen_dir.c_str()).c_str()));
+    ASSERT_TRUE(vespalib::SlaveProc::run(vespalib::make_string("mkdir %s", gen_dir.c_str()).c_str()));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_F("require that empty setup passes validation", EmptyModel()) {
+    EXPECT_TRUE(f.verify());
+}
+
+TEST_F("require that we can verify multiple rank profiles", SimpleModel()) {
+    f.first_phase(valid_feature);
+    f.good_profile();
+    EXPECT_TRUE(f.verify());
+    f.bad_profile();
+    EXPECT_FALSE(f.verify());
+}
+
+TEST_F("require that first phase can break validation", SimpleModel()) {
+    f.first_phase(invalid_feature);
+    EXPECT_FALSE(f.verify());
+}
+
+TEST_F("require that second phase can break validation", SimpleModel()) {
+    f.second_phase(invalid_feature);
+    EXPECT_FALSE(f.verify());
+}
+
+TEST_F("require that summary features can break validation", SimpleModel()) {
+    f.summary_feature(invalid_feature);
+    EXPECT_FALSE(f.verify());
+}
+
+TEST_F("require that dump features can break validation", SimpleModel()) {
+    f.dump_feature(invalid_feature);
+    EXPECT_FALSE(f.verify());
+}
+
+TEST_F("require that fieldMatch feature requires single value field", SimpleModel()) {
+    f.first_phase("fieldMatch(keywords)");
+    EXPECT_FALSE(f.verify());
+    f.first_phase("fieldMatch(list)");
+    EXPECT_FALSE(f.verify());
+    f.first_phase("fieldMatch(title)");
+    EXPECT_TRUE(f.verify());
+}
+
+TEST_F("require that age feature requires attribute parameter", SimpleModel()) {
+    f.first_phase("age(unknown)");
+    EXPECT_FALSE(f.verify());
+    f.first_phase("age(title)");
+    EXPECT_FALSE(f.verify());
+    f.first_phase("age(date)");
+    EXPECT_TRUE(f.verify());
+}
+
+TEST_F("require that nativeRank can be used on any valid field", SimpleModel()) {
+    f.verify_invalid({"nativeRank(unknown)"});
+    f.verify_valid({"nativeRank", "nativeRank(title)", "nativeRank(date)", "nativeRank(title,date)"});
+}
+
+TEST_F("require that nativeAttributeMatch requires attribute parameter", SimpleModel()) {
+    f.verify_invalid({"nativeAttributeMatch(unknown)", "nativeAttributeMatch(title)", "nativeAttributeMatch(title,date)"});
+    f.verify_valid({"nativeAttributeMatch", "nativeAttributeMatch(date)"});
+}
+
+TEST_F("require that shadowed attributes can be used", ShadowModel()) {
+    f.first_phase("attribute(both)");
+    EXPECT_TRUE(f.verify());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_F("cleanup files", Model()) {
+    ASSERT_TRUE(vespalib::SlaveProc::run(vespalib::make_string("rm -rf %s", gen_dir.c_str()).c_str()));
+}
+
+TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.sh b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.sh
new file mode 100755
index 00000000000..d03b6309ec9
--- /dev/null
+++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export PWD=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+$VALGRIND ./searchcore_verify_ranksetup_test_app
diff --git a/searchcore/src/tests/slime/convert_document_to_slime/.gitignore b/searchcore/src/tests/slime/convert_document_to_slime/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/tests/slime/convert_document_to_slime/.gitignore
diff --git a/searchcore/src/versiontag.mak b/searchcore/src/versiontag.mak
new file mode 100644
index 00000000000..388e8461844
--- /dev/null
+++ b/searchcore/src/versiontag.mak
@@ -0,0 +1,34 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+VTAG_DATE=$(shell date +%Y.%m.%d-%H.%M.%S)
+VTAG_SYSTEM=$(shell uname -s)
+VTAG_SYSTEM_REV=$(shell uname -r)
+VTAG_BUILDER=$(shell (whoami) 2>/dev/null||logname)@$(shell uname -n)
+ifneq (X$(SPECIFIED_VTAG),XDISABLE)
+  ifeq (X$(UNAME), XWin32)
+    VTAG=-DV_TAG='\"$(SPECIFIED_VTAG)\"'
+  else
+    VTAG=-DV_TAG='"$(SPECIFIED_VTAG)"'
+  endif
+else
+  ifeq (X$(UNAME), XWin32)
+    VTAG=
+  else
+    VTAG_TAG=$(shell cat $(TOP)/CVS/Tag 2>/dev/null | sed "s/^.//" 2>/dev/null)
+    ifeq (X$(VTAG_TAG),X)
+      VTAG_TAG=CURRENT
+    endif
+    ifeq ($(findstring _RELEASE, $(VTAG_TAG)),_RELEASE)
+      VTAG_SYSTEM=$(shell uname -s)
+      VTAG=-DV_TAG='"$(VTAG_TAG)-$(VTAG_SYSTEM)"'
+    else
+      VTAG_DATE=$(shell date +%Y.%m.%d-%H:%M:%S)
+      VTAG_SYSTEM=$(shell (whoami) 2>/dev/null||logname)@$(shell uname -n)-$(shell uname -s)-$(shell uname -r)
+      VTAG=-DV_TAG='"$(VTAG_TAG)-$(VTAG_SYSTEM)-$(VTAG_DATE)"'
+    endif
+  endif
+endif
+VTAG+= -DV_TAG_DATE='"$(VTAG_DATE)"'
+VTAG+= -DV_TAG_SYSTEM='"$(VTAG_SYSTEM)"'
+VTAG+= -DV_TAG_SYSTEM_REV='"$(VTAG_SYSTEM_REV)"'
+VTAG+= -DV_TAG_BUILDER='"$(VTAG_BUILDER)"'
+
diff --git a/searchcore/src/vespa/searchcore/.gitignore b/searchcore/src/vespa/searchcore/.gitignore
new file mode 100644
index 00000000000..05d624d9ed8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/.gitignore
@@ -0,0 +1,5 @@
+*.lib
+.depend
+Makefile
+config.h
+lib*.a
diff --git a/searchcore/src/vespa/searchcore/common/.gitignore b/searchcore/src/vespa/searchcore/common/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/common/.gitignore
diff --git a/searchcore/src/vespa/searchcore/common/OWNERS b/searchcore/src/vespa/searchcore/common/OWNERS
new file mode 100644
index 00000000000..1037590124e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/common/OWNERS
@@ -0,0 +1 @@
+balder
diff --git a/searchcore/src/vespa/searchcore/config/.gitignore b/searchcore/src/vespa/searchcore/config/.gitignore
new file mode 100644
index 00000000000..d58390943e2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/config/.gitignore
@@ -0,0 +1,4 @@
+.depend
+Makefile
+config-*.cpp
+config-*.h
diff --git a/searchcore/src/vespa/searchcore/config/CMakeLists.txt b/searchcore/src/vespa/searchcore/config/CMakeLists.txt
new file mode 100644
index 00000000000..babd73aff77
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/config/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_fconfig STATIC
+    SOURCES
+    DEPENDS
+)
+vespa_generate_config(searchcore_fconfig partitions.def)
+install(FILES partitions.def DESTINATION var/db/vespa/config_server/serverdb/classes)
+vespa_generate_config(searchcore_fconfig fdispatchrc.def)
+install(FILES fdispatchrc.def DESTINATION var/db/vespa/config_server/serverdb/classes)
+vespa_generate_config(searchcore_fconfig proton.def)
+install(FILES proton.def DESTINATION var/db/vespa/config_server/serverdb/classes)
diff --git a/searchcore/src/vespa/searchcore/config/OWNERS b/searchcore/src/vespa/searchcore/config/OWNERS
new file mode 100644
index 00000000000..7ae1acb1be9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/config/OWNERS
@@ -0,0 +1 @@
+geirst
diff --git a/searchcore/src/vespa/searchcore/config/fdispatchrc.def b/searchcore/src/vespa/searchcore/config/fdispatchrc.def
new file mode 100644
index 00000000000..0f64be20ae0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/config/fdispatchrc.def
@@ -0,0 +1,87 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=vespa.config.search.core
+
+## Default bias used when calculating docsum slowness limit. The
+## formula used is: factor * Average Docsum Time + bias. If a node
+## uses more time than this limit, it is considered slow. This config
+## value may be overridden per dataset by using the slowdocsumlimitbias
+## keyword in the partitions file.
+defaultslowdocsumlimitbias      double       default=1.0 restart
+
+## Default factor used when calculating docsum slowness limit. The
+## formula used is: factor * Average Docsum Time + bias. If a node
+## uses more time than this limit, it is considered slow.  This config
+## value may be overridden per dataset by using the slowdocsumlimitfactor
+## keyword in the partitions file.
+defaultslowdocsumlimitfactor    double       default=2.0 restart
+
+## Default bias used when calculating query slowness limit. The
+## formula used is: factor * Average Search Time + bias. If a node
+## uses more time than this limit, it is considered slow.  This config
+## value may be overridden per dataset by using the slowquerylimitbias
+## keyword in the partitions file.
+defaultslowquerylimitbias       double       default=1.0 restart
+
+## Default factor used when calculating query slowness limit. The
+## formula used is: factor * Average Search Time + bias. If a node
+## uses more time than this limit, it is considered slow.  This config
+## value may be overridden per dataset by using the slowquerylimitfactor
+## keyword in the partitions file.
+defaultslowquerylimitfactor     double       default=2.0 restart
+
+## The port where FNET Remote Tools RPC service should be made available.
+## If 0, fdispatch will not offer RPC service.
+frtport                         int         default=0 restart
+
+## Port to use for yamas health reporting
+healthport int default=0 restart
+
+## The maximum time between successful reads on a socket before timeout. 
+maxsocksilent                   double       default=5.0 restart
+
+## The maximum number of threads used. 0 means no limit.
+maxthreads                      int         default=150 restart
+
+## The number of transport threads used when talking to search nodes.
+transportthreads                int         default=1 restart
+
+## specifies the partition we provide upwards in a multi-level dispatch system.
+partition                       int         default=0 restart
+
+## specifies the port number for the persistent internal transport
+## protocol provided for a multi-level dispatch system. If this value
+## is 0, MLD service is not provided, and this process cannot be used
+## by other fdispatch processes.
+ptport                          int         default=0 restart
+
+## The name of the upwards transport to be used.  If empty, use
+## the default transport.
+## This config setting is unused and should be removed.
+transport                       string      default=""
+
+## If present, the TCP_NODELAY option is set on the persistent
+## transport connections. This causes non-full packets to be sent even
+## though previously sent data has not yet been acknowledged (e.g. due
+## to the delayed ack feature present on various tcp stacks).
+transportnodelay                bool   	    default=true restart
+
+## Decides if a Q is used when sending data. Q will increase throughput.
+transportdirectwrite            bool        default=false restart
+
+## Minimum size of packets to compress (0 means no compression)
+##
+packetcompresslimit int default = 1024 restart
+
+## Compression level for packets
+##
+## Default value is 3
+packetcompresslevel int default = 3 restart
+
+## Compression type for packets
+##
+## Default is LZ4
+packetcompresstype enum {NONE, LZ4} default=LZ4 restart
+
+## Specifies at which level this dispatcher is in a multi-level dispatch system.
+## The top-level dispatcher is at level 0.
+dispatchlevel int default=0 restart
diff --git a/searchcore/src/vespa/searchcore/config/partitions.def b/searchcore/src/vespa/searchcore/config/partitions.def
new file mode 100644
index 00000000000..257ce7a41d3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/config/partitions.def
@@ -0,0 +1,205 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=vespa.config.search.core
+
+## each dataset must be identified by a unique id, preferably a small integer.
+## note that the array index in the dataset[] array is never significant.
+dataset[].id       int
+
+## Define the cost of using a dataset. fdispatch will try to
+## spread queries not specifying dataset over all datasets having a
+## nonzero refcost to minimize the maximum active refcost.
+dataset[].refcost			int default=0
+
+## Defines the number of bits used to encode the partition number
+## internally on a dispatch node. The value must be in the range [1,8].
+## The default value is 6, allowing the numparts parameter to be up to 63.
+dataset[].partbits			int default=8
+
+## Defines the number of bits used to encode the row number internally
+## on a dispatch node. A nonzero value is needed to allow slightly
+## different engines to be on the same partition.
+dataset[].rowbits			int default=0
+
+## Defines the number of partitions below this fdispatch process.
+## The fsearch and fdispatch processes contacted must provide a partition
+## number in the range [ firstpart, firstpart + partitions - 1 ]
+## The legal range for numparts is limited by the partbits parameter.
+## For PLAIN datasets numparts must be positive.
+dataset[].numparts			int default=0
+
+## Defines the lowest partition number accepted by this fdispatch
+## process.
+dataset[].firstpart			int default=0
+
+## Minimum number of partitions available for the dataset to be
+## considered up. If any dataset is considered down then the HTTP
+## interface on fdispatch is temporarily closed. A dispatch node below
+## can represent multiple partitions, e.g. a dispatch node having 5
+## dispatch nodes below that each has 20 search nodes below can have
+## 100 partitions. Setting minpartitions to 95 will then mean that at
+## most 5 search nodes can be down before the top level dispatch node
+## takes down it's HTTP interface.
+dataset[].minpartitions			int default=0
+
+## Minimum number of good engines in a partition before the partition is
+## used for queries.
+dataset[].mpp			int default=1
+
+## Maximum number of nodes that can be down in a row while still using this row
+## for queries when using the FIXEDROW query distribution.
+dataset[].maxnodesdownperfixedrow int default=0
+
+## Use simple roundrobin or random.
+dataset[].useroundrobinforfixedrow bool default=true
+
+## specifies where a fdispatch or fsearch process can be contacted.
+## must be in the format hostname:port/id where /id is optional.
+## Normally you have at least as many engine array members as
+## the number of partitions specified. To reduce the impact of
+## packet loss, multiple instances of the same host and port but
+## with different ids can be used.
+dataset[].engine[].name_and_port			string
+
+## If a non-negative partition number is specified then the engine
+## is hardwired to that partition number and considered to be down if
+## the monitoring partition ID doesn't match (but see overridepartids).
+dataset[].engine[].partid				int default=0
+
+## If rowbits is nonzero you can specify a rowid for each engine.
+## Then engines with different rowid and the same partition number
+## don't need to be completely identical. If the rowid can't be
+## encoded within the number of rowbits then the behaviour is undefined.
+dataset[].engine[].rowid				int default=0
+
+## If the engine is a dispatch node, the subdatasetid specifies
+## which of the engine's datasets should be used.
+dataset[].engine[].subdatasetid				int default=0
+
+## The refcost is used for load balancing between different engines
+## in the same partition (column) in the same manner as the refcost
+## parameter for datasets. A refcost of 0 means that the engine won't
+## be selected as part of load balancing.
+dataset[].engine[].refcost				int default=1
+
+## When set to true, overrides of the partition id reported by the node.
+## This only affects the engine if the partid is specified.
+dataset[].engine[].overridepartids			bool default=true
+
+## Maximum number of hits that will be requested from a single node
+## in this dataset. If not set, there is no limit. Using this option
+## may help reduce network traffic when searching in datasets with big
+## fan-out, but it will also result in incorrect and incomplete results;
+## don't use it if you don't (really) mean it.
+dataset[].maxhitspernode		int default=2147483647
+
+## When using this dataset as an estimation dataset
+## only use estparts number of partitions instead of the full numparts.
+## The partitions are randomly chosen and the estimate multiplied with
+## numparts/estparts.  If estparts=0 the entire dataset is used.
+dataset[].estparts			int default=0
+
+## When using this dataset as an estimation dataset only consider
+## the partitions with partition number below estpartcutoff.
+## estpartcutoff=0 means no cutoff. Avoid using this parameter.
+dataset[].estpartcutoff			int default=0
+
+## Minimum active requests before considering estimates.
+dataset[].minactive			int default=500
+
+## Maximum active requests before queueing.
+dataset[].maxactive			int default=500
+
+## Maximum active requests before cutoff.
+dataset[].cutoffactive			int default=1000
+
+## Minimum estimated active requests before queueing.
+dataset[].minestactive			int default=500
+
+## Maximum estimated active requests before early drop.
+dataset[].maxestactive			int default=1000
+
+## Maximum estimated active requests (100% early drop).
+dataset[].cutoffestactive		int default=1000
+
+## Maximum query queue drain rate.
+dataset[].queuedrainrate		double default=400
+
+## Maximum queued queries drained at once.
+dataset[].queuedrainmax			double default=40
+
+## Factor used when calculating query slowness limit. The formula
+## used is: factor * Average Search Time + bias. If a node uses more
+## time than this limit, it is considered slow. The default value is
+## defined by the defaultslowquerylimitfactor config value in the
+## fdispatchrc config.
+dataset[].slowquerylimitfactor		double default=0.0
+
+## Bias used when calculating query slowness limit.
+dataset[].slowquerylimitbias		double default=100.0
+
+## Factor used when calculating docsum slowness limit. The formula
+## used is: factor * Average Docsum Time + bias. If a node uses more
+## time than this limit, it is considered slow. The default value is
+## defined by the defaultslowdocsumlimitfactor config value in the
+## fdispatchrc config.
+dataset[].slowdocsumlimitfactor		double default=0
+
+## Bias used when calculating docsum slowness limit.
+dataset[].slowdocsumlimitbias		double default=100.0
+
+## The number of seconds between sending monitor requests to the
+## attached search nodes.
+dataset[].monitorinterval                 double       default=1.0
+
+## The maximum number of seconds to wait for the resultset after
+## minimal search coverage is reached.
+dataset[].higher_coverage_maxsearchwait   double       default=1.0
+
+## The minimum number of seconds to wait for the resultset while
+## full search coverage is still not reached.
+dataset[].higher_coverage_minsearchwait   double       default=0.0
+
+## If less time than the base wait has elapsed then boost the timeouts
+## above with the difference between the base wait and the elapsed
+## time.  This compensates somewhat for quick responses (e.g.  cached
+## response on most nodes) without impacting the timeouts for
+## responses that are not so quick.
+dataset[].higher_coverage_basesearchwait double default=0.1
+
+## The minimum search coverage, as a percentage.
+dataset[].minimal_searchcoverage   double       default=100.0
+
+## The maximum number of seconds to wait for document summaries
+## after minimum docsum coverage is reached.
+dataset[].higher_coverage_maxdocsumwait   double       default=0.3
+
+## The minimum number of seconds to wait for document summaries
+## while full docsum coverage is still not reached.
+dataset[].higher_coverage_mindocsumwait   double       default=0.1
+
+## If less time than the base wait has elapsed then boost the timeouts
+## above with the difference between the base wait and the elapsed
+## time.  This compensates somewhat for quick responses (e.g.  cached
+## response on most nodes) without impacting the timeouts for
+## responses that are not so quick.
+dataset[].higher_coverage_basedocsumwait double default=0.1
+
+## The minimum docsum coverage, as a percentage.
+dataset[].minimal_docsumcoverage   double       default=100.0
+
+## If random, use standard load balancing.
+## if deterministic, use deterministic query forwarding
+## If auto, use deterministic when the frequence distribution of 
+## the queries are relatively well balanced, and fall back to 
+## standard load balancing when not.
+dataset[].querydistribution enum { RANDOM, AUTOMATIC, FIXEDROW } default=AUTOMATIC
+
+## Minimum coverage for allowing a group to be considered for serving.
+dataset[].min_group_coverage double default=100
+
+## Required activedocs coverage for a group, as a percentage
+dataset[].min_activedocs_coverage double default=97.0
+
+## Decay rate used when loadbalancing between groups.
+## Lower number will react faster to changes in cluster.
+dataset[].latency_decay_rate double default=10000
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
new file mode 100644
index 00000000000..b55f29ff810
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -0,0 +1,340 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=vespa.config.search.core
+
+## Base directory. The default is ignored as it is assigned by the model
+basedir       string default="tmp" restart
+
+## specifies the port number for the persistent internal transport
+## protocol provided for a multi-level dispatch system.
+ptport        int default=8003 restart
+
+## Port to use for the rpcserver.
+rpcport       int default=8004 restart
+
+## Port used for Slime Message Passing (srmp protocol)
+slime_messaging_port int default=8005 restart
+
+## Connect spec for rtc.
+rtcspec       string default="tcp/localhost:8004" restart
+
+## Port to use for the web server
+httpport       int default=0 restart
+
+## Port to use for the persistence provider rpc server
+persistenceprovider.port       int default=8006 restart
+
+## Number of threads used to drive the internal persistence SPI
+persistenceprovider.threads    int default=16 restart
+
+## Cluster name
+clustername	string default="" restart
+
+## Partition number
+partition     int default=0 restart
+
+## Distribution key
+distributionkey int default=-1
+
+## Num searcher threads
+numsearcherthreads int default=64 restart
+
+## Number of threads used per search
+numthreadspersearch int default=1 restart
+
+## Num summary threads
+numsummarythreads int default=16 restart
+
+## Stop on io errors ?
+stoponioerrors bool default=false restart
+
+## Maximum number of concurrent flushes outstanding.
+flush.maxconcurrent int default=2 restart
+
+## Number of seconds between checking for stuff to flush when the system is idling.
+flush.idleinterval double default=10.0 restart
+
+## Which flushstrategy to use.
+flush.strategy enum {SIMPLE, MEMORY} default=MEMORY restart
+
+## Total number of bytes allowed before forcing flush.
+flush.memory.maxmemory long default=4294967296 restart
+
+## Maximum total disk bloat factor before forcing flush.
+flush.memory.diskbloatfactor double default=0.2 restart
+
+## Max disk usage (in bytes) for all transaction logs before forcing flush.
+flush.memory.maxtlssize long default=21474836480 restart
+
+## Number of bytes allowed per component before forcing memory prioritization.
+flush.memory.each.maxmemory long default=1073741824 restart
+
+## Maximum disk bloat factor per component before forcing flush.
+flush.memory.each.diskbloatfactor double default=0.2 restart
+
+## Age of unflushed content before forcing age prioritization.
+## Unit is seconds with 1 day being the default.
+flush.memory.maxage.time double default=86400.0 restart
+
+## Max diff in serial number allowed before that takes precedence.
+flush.memory.maxage.serial long default=1000000 restart
+
+## The cost of doing replay when replaying the transaction log.
+##
+## The number of bytes to replay * replaycost gives an estimate of the
+## total cost of replaying the transaction log.
+##
+## The prepare for restart flush strategy will choose a set of components to flush
+## such that the cost of flushing these + the cost of replaying the transaction log
+## is as low as possible.
+flush.preparerestart.replaycost double default=4.0 restart
+
+## The cost of doing writes when flushing components to disk.
+##
+## The number of bytes to write (for a set of flushed components) * writecost
+## gives an estimate of the total cost of flushing this set of components.
+##
+## The prepare for restart flush strategy will choose a set of components to flush
+## such that the cost of flushing these + the cost of replaying the transaction log
+## is as low as possible.
+flush.preparerestart.writecost double default=1.0 restart
+
+## Control io options during write both under dump and fusion.
+indexing.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart
+
+## Control io options during read both under dump and fusion.
+indexing.read.io enum {NORMAL, DIRECTIO} default=DIRECTIO restart
+
+## Control number of threads used for indexing
+indexing.threads int default=1 restart
+
+## How long a freshly loaded index shall be warmed up
+## before being used for serving
+index.warmup.time double default=0.0 restart
+
+## How many flushed indexes there can be befor fusion is forced.
+## Setting to 1 will force an immediate fusion.
+index.maxflushed int default=2 restart
+
+## How much memory is set aside for caching.
+## Now only used for caching of dictionary lookups.
+index.cache.size long default=0 restart
+
+## Control io options during flushing of attributes.
+attribute.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart
+
+## Control options for io during search.
+## Dictionary is always MMAP.
+search.io enum {NORMAL, DIRECTIO, MMAP } default=MMAP restart
+
+## Multiple optional options for use with mmap
+search.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart
+
+## Advise to give to os when mapping memory.
+search.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart
+
+## Max number of threads allowed to handle large queries concurrently
+## Postitive number means there is a limit, 0 or negative mean no limit.
+search.memory.limiter.maxthreads int default=0
+
+## Minimum coverage of corpus to postprocess before applying above concurrency limit.
+search.memory.limiter.mincoverage double default=1.0
+
+## Minimum number of hits to postprocess before applying above concurrency limit.
+## Both must be covered before applying limiter.
+search.memory.limiter.minhits int default=1000000
+
+## Control of grouping session manager entries
+grouping.sessionmanager.maxentries int default=500 restart
+
+## Control of pruning interval to remove sessions that have timed out
+grouping.sessionmanager.pruning.interval double default=1.0
+
+## Minimum initial size for any per document tables.
+grow.initial int default=1024 restart
+## Grow factor in percent for any per document tables.
+grow.factor int default=50 restart
+## Constant added when growing any per document tables.
+grow.add int default=1 restart
+## The number of documents to amortize memory spike cost over
+grow.numdocs int default=10000 restart
+
+## Control cache size in bytes.
+summary.cache.maxbytes long default=0 restart
+
+## Control number of cache entries preallocated.
+## Default is no preallocation
+## Can be set to a higher number to avoid resizing.
+summary.cache.initialentries long default=0 restart
+
+## Control compression type of the summary while in the cache.
+summary.cache.compression.type enum {NONE, LZ4} default=LZ4 restart
+
+## Control compression level of the summary while in cache.
+summary.cache.compression.level int default=9 restart
+
+## Control compression type of the summary
+## NB So far only stragey=LOG honours it.
+summary.log.chunk.compression.type enum {NONE, LZ4} default=LZ4 restart
+
+## Control compression level of the summary
+summary.log.chunk.compression.level int default=9 restart
+
+## Max size in bytes per chunk.
+summary.log.chunk.maxbytes int default=65536 restart
+
+## Max number of documents in each chunk.
+summary.log.chunk.maxentries int default=256 restart
+
+## Skip crc32 check on read.
+summary.log.chunk.skipcrconread bool default=false restart
+
+## Control how compation is done, write to the front or to new const file.
+summary.log.compact2activefile bool default=true restart
+
+## Max size per summary file.
+summary.log.maxfilesize long default=1000000000 restart
+
+## Max disk bloat factor. This will trigger compacting.
+summary.log.maxdiskbloatfactor double default=0.1 restart
+
+## Max bucket spread within a single summary file. This will trigger bucket order compacting.
+## Only used when summary.compact2buckets is true.
+summary.log.maxbucketspread double default=2.5 restart
+
+## If a file goes below this ratio compared to allowed max size it will be joined to the front.
+## Value in the range [0.0, 1.0]
+summary.log.minfilesizefactor double default=0.2 restart
+
+## Number of threads used for compressing incomming documents/compacting.
+summary.log.numthreads int default=8 restart
+
+## Control io options during flush of stored documents.
+summary.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart
+
+## Control io options during read of stored documents.
+summary.read.io enum {NORMAL, DIRECTIO, MMAP } default=MMAP restart
+
+## Multiple optional options for use with mmap
+summary.read.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart
+
+## Advise to give to os when mapping memory.
+summary.read.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart
+
+## Enable compact for bucket oriented access.
+summary.compact2buckets bool default=false restart
+
+## The name of the input document type
+documentdb[].inputdoctypename string restart
+## The configid used to subscribe to config for this database.
+documentdb[].configid string restart
+## How many seconds is allowed from document is received to it is visible in the index.
+documentdb[].visibilitydelay double default=0.0
+
+## The interval of when periodic tasks should be run
+periodic.interval double default=3600.0
+
+## Connect spec for transactionlog server.
+tlsspec       string default="tcp/localhost:13700" restart
+
+## ConfigId for transactionlogserver
+tlsconfigid   string default="" restart
+
+## Slobrok configid
+slobrokconfigid         string default="" restart
+
+## Slobrok configid
+routingconfigid         string default="" restart
+
+## Interval between pruning of old removed documents.
+##
+## Default value is 6 hours (21600 seconds).
+pruneremoveddocumentsinterval double default=21600.0
+
+## Age of removed document before it can be pruned.
+##
+## Default value is 2 weeks (1209600 seconds).
+pruneremoveddocumentsage double default=1209600.0
+
+## Interval between wiping of old removed document fields (in seconds).
+##
+## Default value is 6 hours (21600 seconds).
+wipeoldremovedfieldsinterval double default=21600.0
+
+## Age of removed document fields before they can be wiped (in seconds).
+##
+## Default value is 2 weeks (1209600 seconds).
+wipeoldremovedfieldsage double default=1209600.0
+
+## Timeout of recovery rpc calls in rowcolumn mode
+##
+## Default value is 60 seconds
+recoverymanager.rpctimeout double default=60.0 restart
+
+## Minimum size of packets to compress (0 means no compression)
+##
+packetcompresslimit int default=1024
+
+## Compression level for packets
+##
+## Default value is 3
+packetcompresslevel int default=3
+
+## Compression type for packets
+##
+## Default is LZ4
+packetcompresstype enum {NONE, LZ4} default=LZ4
+
+## Interval between considering if lid space compaction should be done (in seconds).
+##
+## Default value is 1 hour (3600 seconds).
+lidspacecompaction.interval double default=3600.0
+
+## The allowed lid bloat (in docs) before considering lid space compaction.
+##
+## When considering compaction the lid bloat is calculated as (docIdLimit - numDocs).
+## The lid bloat must be >= allowedlidbloat before considering compaction.
+lidspacecompaction.allowedlidbloat int default=1000
+
+## The allowed lid bloat factor (relative) before considering lid space compaction.
+##
+## When considering compaction the lid bloat factor is calculated as (docIdLimit - numDocs)/docIdLimit.
+## The lid bloat factor must be >= allowedlidbloatfactor before considering compaction.
+lidspacecompaction.allowedlidbloatfactor double default=0.01
+
+## This is the maximum value visibilitydelay you can have.
+## A to higher value here will cost more memory while not improving too much.
+maxvisibilitydelay double default=1.0
+
+## You can set this to a number above zero for visit to shortcut expensive serialize size computation.
+## This value will be provided instead.
+## negative number will compute it accurately.
+## Default will be flipped shortly. (Once image personal index has gone live.)
+visit.defaultserializedsize long default=-1
+
+## This will ignore the maxbytes limit advised from above.
+## default will be flipped shortly. (Once image personal index has gone live.)
+visit.ignoremaxbytes bool default=false
+
+## Number of initializer threads used for loading structures from disk at proton startup.
+## The threads are shared between document databases when value is larger than 0.
+## When set to 0 (default) we use 1 separate thread per document database.
+initialize.threads int default = 0
+
+## Portion of enumstore address space that can be used before put and update
+## portion of feed is blocked.
+writefilter.attribute.enumstorelimit double default = 1.0
+
+## Portion of attribute multivalue mapping address space that can be used
+## before put and update portion of feed is blocked.
+writefilter.attribute.multivaluelimit double default = 1.0
+
+## Portion of physical memory that can be resident memory in anonymous mapping
+## by the proton process before put and update portion of feed is blocked.
+writefilter.memorylimit double default = 1.0
+
+## Portion of space on disk partition that can be used or reserved before
+## put and update portion of feed is blocked.
+writefilter.disklimit double default = 1.0
+
+## Interval between sampling of disk and memory usage. Default is 60 seconds.
+writefilter.sampleinterval double default = 60.0
diff --git a/searchcore/src/vespa/searchcore/fdispatch/.gitignore b/searchcore/src/vespa/searchcore/fdispatch/.gitignore
new file mode 100644
index 00000000000..fe8e24952a5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/.gitignore
@@ -0,0 +1 @@
+ID
diff --git a/searchcore/src/vespa/searchcore/fdispatch/OWNERS b/searchcore/src/vespa/searchcore/fdispatch/OWNERS
new file mode 100644
index 00000000000..3e9fc8ab356
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/OWNERS
@@ -0,0 +1,2 @@
+balder
+tegge
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/fdispatch/common/CMakeLists.txt
new file mode 100644
index 00000000000..3e3d2f9acc8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_fdcommon STATIC
+    SOURCES
+    appcontext.cpp
+    perftask.cpp
+    queryperf.cpp
+    rpc.cpp
+    search.cpp
+    timestat.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp
new file mode 100644
index 00000000000..720bf2e6377
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".appcontext");
+#include "appcontext.h"
+
+
+FastS_TimeKeeper::FastS_TimeKeeper()
+    : _clock(0.010),
+      _thread_pool(128 * 1024)
+{
+    bool ok = _thread_pool.NewThread(&_clock);
+    assert(ok);
+}
+
+
+FastS_TimeKeeper::~FastS_TimeKeeper()
+{
+    _clock.stop();
+    _thread_pool.Close();
+}
+
+//---------------------------------------------------------------------
+
+FastS_AppContext::FastS_AppContext()
+    : _timeKeeper(),
+      _createTime()
+{
+    _createTime = _timeKeeper.GetTime();
+}
+
+
+FastS_AppContext::~FastS_AppContext()
+{
+}
+
+
+FNET_Transport *
+FastS_AppContext::GetFNETTransport()
+{
+    return NULL;
+}
+
+
+FNET_Scheduler *
+FastS_AppContext::GetFNETScheduler()
+{
+    return NULL;
+}
+
+
+FastS_NodeManager *
+FastS_AppContext::GetNodeManager()
+{
+    return NULL;
+}
+
+
+FastS_DataSetCollection *
+FastS_AppContext::GetDataSetCollection()
+{
+    return NULL;
+}
+
+
+FastOS_ThreadPool *
+FastS_AppContext::GetThreadPool()
+{
+    return NULL;
+}
+
+
+void
+FastS_AppContext::logPerformance()
+{
+}
+
+
+uint32_t
+FastS_AppContext::getDispatchLevel()
+{
+    return 0u;
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h
new file mode 100644
index 00000000000..1b27d647616
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/util/clock.h>
+
+class FastS_NodeManager;
+class FNET_Transport;
+class FNET_Scheduler;
+class FastS_DataSetCollection;
+
+
+class FastS_TimeKeeper
+{
+private:
+    vespalib::Clock   _clock;
+    FastOS_ThreadPool _thread_pool;
+
+public:
+    FastS_TimeKeeper();
+    ~FastS_TimeKeeper();
+
+    double GetTime() const { return _clock.getTimeNSAssumeRunning().sec(); }
+};
+
+
+class FastS_AppContext
+{
+private:
+    FastS_TimeKeeper _timeKeeper;
+    double           _createTime;
+
+public:
+    FastS_AppContext();
+    virtual ~FastS_AppContext();
+
+    FastS_TimeKeeper *GetTimeKeeper() { return &_timeKeeper; }
+
+    virtual FastS_NodeManager *GetNodeManager();
+    virtual FNET_Transport *GetFNETTransport();
+    virtual FNET_Scheduler *GetFNETScheduler();
+    virtual FastS_DataSetCollection *GetDataSetCollection();
+    virtual FastOS_ThreadPool *GetThreadPool();
+    virtual void logPerformance();
+    virtual uint32_t getDispatchLevel();
+private:
+};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/perftask.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/perftask.cpp
new file mode 100644
index 00000000000..b3778d6d449
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/perftask.cpp
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2005 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".perftask");
+#include <vespa/fnet/fnet.h>
+#include <vespa/searchcore/fdispatch/common/perftask.h>
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+
+
+FastS_PerfTask::FastS_PerfTask(FastS_AppContext &ctx, double delay)
+    : FNET_Task(ctx.GetFNETScheduler()),
+      _ctx(ctx),
+      _delay(delay),
+      _valid(ctx.GetFNETScheduler() != NULL)
+{
+    if (_valid) {
+        ScheduleNow();
+    } else {
+        LOG(warning, "Performance monitoring disabled; "
+            "no scheduler found in application context");
+    }
+}
+
+
+FastS_PerfTask::~FastS_PerfTask()
+{
+    if (_valid) {
+        Kill();
+    }
+}
+
+
+void
+FastS_PerfTask::PerformTask()
+{
+    Schedule(_delay);
+    _ctx.logPerformance();
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/perftask.h b/searchcore/src/vespa/searchcore/fdispatch/common/perftask.h
new file mode 100644
index 00000000000..da7648e5443
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/perftask.h
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2005 Overture Services Norway AS
+
+#pragma once
+
+class FastS_AppContext;
+
+class FastS_PerfTask : public FNET_Task
+{
+private:
+    FastS_AppContext &_ctx;
+    double            _delay;
+    bool              _valid;
+
+    FastS_PerfTask(const FastS_PerfTask &);
+    FastS_PerfTask &operator=(const FastS_PerfTask &);
+
+public:
+    FastS_PerfTask(FastS_AppContext &ctx, double delay);
+    ~FastS_PerfTask();
+    virtual void PerformTask();
+    bool isValid() const { return _valid; }
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/properties.h b/searchcore/src/vespa/searchcore/fdispatch/common/properties.h
new file mode 100644
index 00000000000..948572bf616
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/properties.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1999-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+class FastS_IProperties
+{
+public:
+    /**
+     * Destructor.  No cleanup needed for base class.
+     */
+    virtual ~FastS_IProperties(void) { }
+
+    virtual bool        IsSet    (const char *key)                         = 0;
+    virtual bool        BoolVal  (const char *key)                         = 0;
+    virtual const char *StrVal   (const char *key, const char *def = NULL) = 0;
+    virtual int         IntVal   (const char *key, int def = -1)           = 0;
+    virtual double      DoubleVal(const char *key, double def = 0.0)       = 0;
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.cpp
new file mode 100644
index 00000000000..34f7b1a3a55
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.cpp
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2005 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".queryperf");
+#include <vespa/fnet/fnet.h>
+#include <vespa/searchcore/fdispatch/common/queryperf.h>
+
+
+FastS_QueryPerf::FastS_QueryPerf()
+    : queueLen(0),
+      activeCnt(0),
+      queryCnt(0),
+      queryTime(0),
+      dropCnt(0),
+      timeoutCnt(0),
+      _lastQueryCnt(0),
+      _lastQueryTime(0.0)
+{
+}
+
+void
+FastS_QueryPerf::reset()
+{
+    queueLen   = 0;
+    activeCnt  = 0;
+    queryCnt   = 0;
+    queryTime  = 0;
+    dropCnt    = 0;
+    timeoutCnt = 0;
+}
+
+void
+FastS_QueryPerf::log()
+{
+    EV_VALUE("queued_queries",   queueLen);
+    EV_VALUE("active_queries",   activeCnt);
+    EV_COUNT("queries",          queryCnt);
+    EV_COUNT("dropped_queries",  dropCnt);
+    EV_COUNT("timedout_queries", timeoutCnt);
+    if (queryCnt > _lastQueryCnt) {
+        double avgQueryTime = (queryTime - _lastQueryTime)
+                              / ((double)(queryCnt - _lastQueryCnt));
+        EV_VALUE("query_eval_time_avg_s", avgQueryTime);
+    }
+    _lastQueryCnt  = queryCnt;
+    _lastQueryTime = queryTime;
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.h b/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.h
new file mode 100644
index 00000000000..36d52cec820
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2005 Overture Services Norway AS
+
+#pragma once
+
+struct FastS_QueryPerf
+{
+    uint32_t queueLen;
+    uint32_t activeCnt;
+    uint32_t queryCnt;
+    double   queryTime;
+    uint32_t dropCnt;
+    uint32_t timeoutCnt;
+
+    FastS_QueryPerf();
+
+    /**
+     * reset all values except the cached 'old' values. This will
+     * prepare the object for reuse logging wise.
+     **/
+    void reset();
+    void log();
+
+private:
+    uint32_t _lastQueryCnt;
+    double   _lastQueryTime;
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp
new file mode 100644
index 00000000000..94d881ad075
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".rpc");
+
+#include <vespa/fnet/frt/frt.h>
+
+#include <vespa/searchcore/util/log.h>
+#include "rpc.h"
+#include <vespa/searchsummary/docsummary/resultconfig.h>
+
+extern char FastS_VersionTag[];
+
+FastS_RPC::FastS_RPC(FastS_AppContext *appCtx) :
+   _appCtx(appCtx),
+   _transport(),
+   _supervisor(&_transport, _appCtx->GetThreadPool()),
+   _sbregister(_supervisor, slobrok::ConfiguratorFactory("admin/slobrok.0"))
+{
+}
+
+bool
+FastS_RPC::Init(int port, const vespalib::string &myHeartbeatId)
+{
+    bool rc = true;
+
+    char spec[4096];
+    snprintf(spec, 4096, "tcp/%d", port);
+    rc = rc && _supervisor.Listen(spec);
+    if (rc) {
+        FRT_ReflectionBuilder rb(&_supervisor);
+        RegisterMethods(&rb);
+        _sbregister.registerName(myHeartbeatId);
+    }
+    return rc;
+}
+
+
+void
+FastS_RPC::RegisterMethods(FRT_ReflectionBuilder *rb)
+{
+    rb->DefineMethod("fs.admin.getNodeType", "", "s", true,
+                     FRT_METHOD(FastS_RPC::RPC_GetNodeType), this);
+    rb->MethodDesc("Get string indicating the node type");
+    rb->ReturnDesc("type",  "node type");
+    //---------------------------------------------------------------//
+    rb->DefineMethod("fs.admin.getCompileInfo", "", "*", true,
+                     FRT_METHOD(FastS_RPC::RPC_GetCompileInfo), this);
+    rb->MethodDesc("Obtain compile info for this node");
+    rb->ReturnDesc("info",  "any number of descriptive strings");
+    //---------------------------------------------------------------//
+}
+
+
+void
+FastS_RPC::RPC_GetCompileInfo(FRT_RPCRequest *req)
+{
+    FRT_Values &ret = *req->GetReturn();
+    ret.AddString("using juniper (api version 2)");
+
+#ifdef NO_MONITOR_LATENCY_CHECK
+    ret.AddString("monitor latency check disabled");
+#endif
+#ifdef CUSTOM_TEST_SHUTDOWN
+    ret.AddString("Win32: debug shutdown for memory leak detection enabled");
+#endif
+    ret.AddString("default transport is 'fnet'");
+
+    const char *prefix = "version tag: ";
+    uint32_t prefix_len = strlen(prefix);
+    uint32_t len = prefix_len + strlen(FastS_VersionTag);
+    if (len == prefix_len) {
+        ret.AddString("version tag not available");
+    } else {
+        char *str = ret.AddString(len + 1);
+        sprintf(str, "%s%s", prefix, FastS_VersionTag);
+    }
+
+    ret.AddString("fastos X current");
+    ret.AddString(FNET_Info::GetFNETVersion());
+}
+
+
+void
+FastS_RPC::RPC_GetNodeType_Proxy(FRT_RPCRequest *req)
+{
+    RPC_GetNodeType(req);
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h
new file mode 100644
index 00000000000..9ac19c3fea7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/fnet/frt/frt.h>
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/slobrok/sbregister.h>
+
+class FastS_RPC : public FRT_Invokable
+{
+private:
+    FastS_RPC(const FastS_RPC &);
+    FastS_RPC& operator=(const FastS_RPC &);
+
+    FastS_AppContext         *_appCtx;
+    FNET_Transport            _transport;
+    FRT_Supervisor            _supervisor;
+    slobrok::api::RegisterAPI _sbregister;
+
+public:
+    FastS_RPC(FastS_AppContext *appCtx);
+    virtual ~FastS_RPC() {}
+
+    FastS_AppContext *GetAppCtx() { return _appCtx; }
+    FRT_Supervisor *GetSupervisor() { return &_supervisor; }
+    bool Init(int port, const vespalib::string& myHeartbeatId);
+    bool Start() { return _transport.Start(_appCtx->GetThreadPool()); }
+    void ShutDown() { _transport.ShutDown(true); }
+
+    // Register RPC Methods
+
+    virtual void RegisterMethods(FRT_ReflectionBuilder *rb);
+
+    // RPC methods implemented here
+
+    void RPC_GetCompileInfo(FRT_RPCRequest *req);
+    void RPC_GetResultConfig(FRT_RPCRequest *req);
+
+    // RPC Proxy Methods
+
+    void RPC_GetNodeType_Proxy(FRT_RPCRequest *req);
+
+    // RPC methods to be implemented by subclasses
+
+    virtual void RPC_GetNodeType(FRT_RPCRequest *req) = 0;
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp
new file mode 100644
index 00000000000..e4c713bf901
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp
@@ -0,0 +1,252 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".search");
+#include <vespa/searchcore/util/log.h>
+#include "search.h"
+
+//---------------------------------------------------------------------
+
+FastS_SearchAdapter::FastS_SearchAdapter(FastS_ISearch *search)
+    : _search(search)
+{
+}
+
+
+FastS_SearchAdapter::~FastS_SearchAdapter()
+{
+}
+
+
+bool
+FastS_SearchAdapter::IsAsync()
+{
+    return _search->IsAsync();
+}
+
+
+uint32_t
+FastS_SearchAdapter::GetDataSetID()
+{
+    return _search->GetDataSetID();
+}
+
+
+FastS_SearchInfo *
+FastS_SearchAdapter::GetSearchInfo()
+{
+    return _search->GetSearchInfo();
+}
+
+
+FastS_ISearch::RetCode
+FastS_SearchAdapter::SetAsyncArgs(FastS_ISearchOwner *owner,
+                                  FastS_SearchContext context)
+{
+    return _search->SetAsyncArgs(owner, context);
+}
+
+
+FastS_ISearch::RetCode
+FastS_SearchAdapter::setSearchRequest(const search::engine::SearchRequest * request)
+{
+    return _search->setSearchRequest(request);
+}
+
+
+FastS_ISearch::RetCode
+FastS_SearchAdapter::SetGetDocsumArgs(search::docsummary::GetDocsumArgs *docsumArgs)
+{
+    return _search->SetGetDocsumArgs(docsumArgs);
+}
+
+
+FastS_ISearch::RetCode
+FastS_SearchAdapter::Search(uint32_t searchOffset,
+                            uint32_t maxhits, uint32_t minhits)
+{
+    return _search->Search(searchOffset, maxhits, minhits);
+}
+
+
+FastS_ISearch::RetCode
+FastS_SearchAdapter::ProcessQueryDone()
+{
+    return _search->ProcessQueryDone();
+}
+
+
+FastS_QueryResult *
+FastS_SearchAdapter::GetQueryResult()
+{
+    return _search->GetQueryResult();
+}
+
+
+FastS_ISearch::RetCode
+FastS_SearchAdapter::GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt)
+{
+    return _search->GetDocsums(hits, hitcnt);
+}
+
+
+FastS_ISearch::RetCode
+FastS_SearchAdapter::ProcessDocsumsDone()
+{
+    return _search->ProcessDocsumsDone();
+}
+
+
+FastS_DocsumsResult *
+FastS_SearchAdapter::GetDocsumsResult()
+{
+    return _search->GetDocsumsResult();
+}
+
+
+search::engine::ErrorCode
+FastS_SearchAdapter::GetErrorCode()
+{
+    return _search->GetErrorCode();
+}
+
+
+const char *
+FastS_SearchAdapter::GetErrorMessage()
+{
+    return _search->GetErrorMessage();
+}
+
+
+void
+FastS_SearchAdapter::Interrupt()
+{
+    _search->Interrupt();
+}
+
+
+void
+FastS_SearchAdapter::Free()
+{
+    _search->Free();
+    delete this;
+}
+
+//---------------------------------------------------------------------
+
+FastS_SyncSearchAdapter::FastS_SyncSearchAdapter(FastS_ISearch *search)
+    : FastS_SearchAdapter(search),
+      _cond(),
+      _waitQuery(false),
+      _queryDone(false),
+      _waitDocsums(false),
+      _docsumsDone(false)
+{
+}
+
+
+FastS_SyncSearchAdapter::~FastS_SyncSearchAdapter()
+{
+}
+
+
+
+FastS_ISearch *
+FastS_SyncSearchAdapter::Adapt(FastS_ISearch *search)
+{
+    if (!search->IsAsync())
+        return search;
+
+    FastS_SyncSearchAdapter *ret = new FastS_SyncSearchAdapter(search);
+    FastS_assert(ret != NULL);
+    search->SetAsyncArgs(ret, FastS_SearchContext());
+    return ret;
+}
+
+
+
+void
+FastS_SyncSearchAdapter::DoneQuery(FastS_ISearch *,
+                                   FastS_SearchContext)
+{
+    Lock();
+    _queryDone = true;
+    if (_waitQuery)
+        Signal();
+    Unlock();
+}
+
+
+void
+FastS_SyncSearchAdapter::DoneDocsums(FastS_ISearch *,
+                                     FastS_SearchContext)
+{
+    Lock();
+    _docsumsDone = true;
+    if (_waitDocsums)
+        Signal();
+    Unlock();
+}
+
+
+void
+FastS_SyncSearchAdapter::WaitQueryDone()
+{
+    Lock();
+    _waitQuery = true;
+    while (!_queryDone)
+        Wait();
+    Unlock();
+}
+
+
+void
+FastS_SyncSearchAdapter::WaitDocsumsDone()
+{
+    Lock();
+    _waitDocsums = true;
+    while (!_docsumsDone)
+        Wait();
+    Unlock();
+}
+
+
+bool
+FastS_SyncSearchAdapter::IsAsync()
+{
+    return false;
+}
+
+
+FastS_ISearch::RetCode
+FastS_SyncSearchAdapter::SetAsyncArgs(FastS_ISearchOwner *,
+                                      FastS_SearchContext)
+{
+    return RET_ERROR;
+}
+
+
+FastS_ISearch::RetCode
+FastS_SyncSearchAdapter::Search(uint32_t searchOffset,
+                                uint32_t maxhits, uint32_t minhits)
+{
+    RetCode res = _search->Search(searchOffset, maxhits, minhits);
+    if (res == RET_INPROGRESS) {
+        WaitQueryDone();
+    }
+    return (res == RET_ERROR) ? RET_ERROR : RET_OK;
+}
+
+
+FastS_ISearch::RetCode
+FastS_SyncSearchAdapter::GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt)
+{
+    RetCode res = _search->GetDocsums(hits, hitcnt);
+    if (res == RET_INPROGRESS) {
+        WaitDocsumsDone();
+    }
+    return (res == RET_ERROR) ? RET_ERROR : RET_OK;
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/search.h b/searchcore/src/vespa/searchcore/fdispatch/common/search.h
new file mode 100644
index 00000000000..93fa12f9237
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/search.h
@@ -0,0 +1,503 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1999-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/document/base/globalid.h>
+#include <vespa/searchlib/common/fslimits.h>
+#include <vespa/searchlib/engine/errorcodes.h>
+#include <vespa/searchsummary/docsummary/getdocsumargs.h>
+#include <vespa/searchlib/engine/searchrequest.h>
+
+#include <limits>
+
+class FastS_ISearch;
+
+//----------------------------------------------------------------
+
+class FastS_SearchContext
+{
+public:
+    union {
+        uint32_t  INT;
+        void     *VOIDP;
+    } _value;
+
+    FastS_SearchContext()
+        : _value()
+    {
+        _value.VOIDP = NULL;
+    }
+    explicit FastS_SearchContext(void *value)
+        : _value()
+    {
+        _value.VOIDP = value;
+    }
+    explicit FastS_SearchContext(uint32_t value)
+        : _value()
+    {
+        _value.INT   = value;
+    }
+};
+
+//----------------------------------------------------------------
+
+class FastS_ISearchOwner
+{
+public:
+    /**
+     * Destructor.  No cleanup needed for base class.
+     */
+    virtual ~FastS_ISearchOwner(void) { }
+
+    virtual void DoneQuery(FastS_ISearch *search,
+                           FastS_SearchContext context) = 0;
+
+    virtual void DoneDocsums(FastS_ISearch *search,
+                             FastS_SearchContext context) = 0;
+};
+
+//----------------------------------------------------------------
+
+class FastS_hitresult
+{
+public:
+    const document::GlobalId & HT_GetGlobalID() const { return _gid; }
+    search::HitRank HT_GetMetric()   const { return      _metric; }
+    uint32_t HT_GetPartID()   const { return   _partition; }
+    uint32_t getDistributionKey() const { return    _distributionKey; }
+
+    void HT_SetGlobalID(const document::GlobalId & val) { _gid = val; }
+    void HT_SetMetric(search::HitRank val)   {    _metric = val; }
+    void HT_SetPartID(uint32_t val)   { _partition = val; }
+    void setDistributionKey(uint32_t val) {  _distributionKey = val; }
+    document::GlobalId _gid;
+    search::HitRank    _metric;
+    uint32_t           _partition;
+private:
+    uint32_t           _distributionKey;
+};
+
+//----------------------------------------------------------------
+
+struct FastS_fullresult {
+    uint32_t              _partition;
+    uint32_t              _docid;
+    document::GlobalId    _gid;
+    search::HitRank       _metric;
+    search::fs4transport::FS4Packet_DOCSUM::Buf _buf;
+};
+
+//----------------------------------------------------------------
+
+class FastS_SearchInfo
+{
+public:
+    uint32_t  _searchOffset;
+    uint32_t  _maxHits;
+    uint64_t  _coverageDocs;
+    uint64_t  _activeDocs;
+
+    FastS_SearchInfo()
+        : _searchOffset(0),
+          _maxHits(0),
+          _coverageDocs(0),
+          _activeDocs(0)
+    {
+    }
+};
+
+//----------------------------------------------------------------
+
+class FastS_QueryResult
+{
+private:
+    FastS_QueryResult(const FastS_QueryResult &);
+    FastS_QueryResult& operator=(const FastS_QueryResult &);
+
+public:
+    FastS_hitresult  *_hitbuf;
+    uint32_t          _hitCount;
+    uint64_t          _totalHitCount;
+    search::HitRank   _maxRank;
+    double            _queryResultTime;
+
+    uint32_t    _groupResultLen;
+    const char *_groupResult;
+
+    uint32_t   *_sortIndex;
+    const char *_sortData;
+
+    FastS_QueryResult()
+        : _hitbuf(NULL),
+          _hitCount(0),
+          _totalHitCount(0),
+          _maxRank(std::numeric_limits<search::HitRank>::is_integer ?
+                   std::numeric_limits<search::HitRank>::min() :
+                   - std::numeric_limits<search::HitRank>::max()),
+          _queryResultTime(0.0),
+          _groupResultLen(0),
+          _groupResult(NULL),
+          _sortIndex(NULL),
+          _sortData(NULL)
+    {}
+};
+
+//----------------------------------------------------------------
+
+class FastS_DocsumsResult
+{
+private:
+    FastS_DocsumsResult(const FastS_DocsumsResult &);
+    FastS_DocsumsResult& operator=(const FastS_DocsumsResult &);
+
+public:
+    FastS_fullresult *_fullresult;
+    uint32_t          _fullResultCount;
+    double            _queryDocSumTime;
+
+    FastS_DocsumsResult()
+        : _fullresult(NULL),
+          _fullResultCount(0),
+          _queryDocSumTime(0.0)
+    {}
+};
+
+//----------------------------------------------------------------
+
+class FastS_ISearch
+{
+public:
+    /**
+     * Destructor.  No cleanup needed for base class.
+     */
+    virtual ~FastS_ISearch(void) { }
+
+
+    enum RetCode {
+        RET_OK         = 0, // sync operation performed
+        RET_INPROGRESS = 1, // async operation started
+        RET_ERROR      = 2  // illegal method invocation
+    };
+
+    // OBTAIN META-DATA
+
+    virtual bool IsAsync() = 0;
+    virtual uint32_t GetDataSetID() = 0;
+    virtual FastS_SearchInfo *GetSearchInfo() = 0;
+
+    // SET PARAMETERS
+
+    virtual RetCode SetAsyncArgs(FastS_ISearchOwner *owner, FastS_SearchContext context) = 0;
+    virtual RetCode setSearchRequest(const search::engine::SearchRequest * request) = 0;
+    virtual RetCode SetGetDocsumArgs(search::docsummary::GetDocsumArgs *docsumArgs) = 0;
+
+    // SEARCH API
+
+    virtual RetCode Search(uint32_t searchOffset,
+                           uint32_t maxhits, uint32_t minhits = 0) = 0;
+    virtual RetCode ProcessQueryDone() = 0;
+    virtual FastS_QueryResult *GetQueryResult() = 0;
+
+    // DOCSUM API
+
+    virtual RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt) = 0;
+    virtual RetCode ProcessDocsumsDone() = 0;
+    virtual FastS_DocsumsResult *GetDocsumsResult() = 0;
+
+    // ERROR HANDLING
+
+    virtual search::engine::ErrorCode GetErrorCode() = 0;
+    virtual const char *GetErrorMessage() = 0;
+
+    // INTERRUPT OPERATION
+
+    virtual void Interrupt() = 0;
+
+    // GET RID OF OBJECT
+
+    virtual void Free() = 0;
+};
+
+//----------------------------------------------------------------
+
+class FastS_SearchBase : public FastS_ISearch
+{
+private:
+    FastS_SearchBase(const FastS_SearchBase &);
+    FastS_SearchBase& operator=(const FastS_SearchBase &);
+
+protected:
+    uint32_t             _dataSetID;
+    search::engine::ErrorCode _errorCode;
+    char                *_errorMessage;
+    const search::engine::SearchRequest *_queryArgs;
+    search::docsummary::GetDocsumArgs *_docsumArgs;
+    FastS_SearchInfo     _searchInfo;
+    FastS_QueryResult    _queryResult;
+    FastS_DocsumsResult  _docsumsResult;
+
+public:
+    FastS_SearchBase(uint32_t dataSetID)
+        : _dataSetID(dataSetID),
+          _errorCode(search::engine::ECODE_NO_ERROR),
+          _errorMessage(NULL),
+          _queryArgs(NULL),
+          _docsumArgs(NULL),
+          _searchInfo(),
+          _queryResult(),
+          _docsumsResult()
+    {
+    }
+
+    virtual ~FastS_SearchBase()
+    {
+        free(_errorMessage);
+    }
+
+    const search::engine::SearchRequest * GetQueryArgs()   { return _queryArgs;  }
+    search::docsummary::GetDocsumArgs * GetGetDocsumArgs() { return _docsumArgs; }
+
+    void SetError(search::engine::ErrorCode errorCode, const char *errorMessage)
+    {
+        _errorCode = errorCode;
+        if (errorMessage != NULL)
+            _errorMessage = strdup(errorMessage);
+        else
+            _errorMessage = NULL;
+    }
+
+
+    virtual uint32_t GetDataSetID()
+    {
+        return _dataSetID;
+    }
+
+    virtual FastS_SearchInfo *GetSearchInfo()
+    {
+        return &_searchInfo;
+    }
+
+    virtual RetCode setSearchRequest(const search::engine::SearchRequest * request)
+    {
+        _queryArgs = request;
+        return RET_OK;
+    }
+
+    virtual RetCode SetGetDocsumArgs(search::docsummary::GetDocsumArgs *docsumArgs)
+    {
+        _docsumArgs = docsumArgs;
+        return RET_OK;
+    }
+
+    virtual RetCode Search(uint32_t searchOffset,
+                           uint32_t maxhits, uint32_t minhits = 0)
+    {
+        (void) minhits;
+        _searchInfo._searchOffset = searchOffset;
+        _searchInfo._maxHits      = maxhits;
+        return RET_OK;
+    }
+
+    virtual RetCode ProcessQueryDone()
+    {
+        return RET_OK;
+    }
+
+    virtual FastS_QueryResult *GetQueryResult()
+    {
+        return &_queryResult;
+    }
+
+    virtual RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt)
+    {
+        (void) hits;
+        (void) hitcnt;
+        return RET_OK;
+    }
+
+    virtual RetCode ProcessDocsumsDone()
+    {
+        return RET_OK;
+    }
+
+    virtual FastS_DocsumsResult *GetDocsumsResult()
+    {
+        return &_docsumsResult;
+    }
+
+    virtual search::engine::ErrorCode GetErrorCode()
+    {
+        return _errorCode;
+    }
+
+    virtual const char *GetErrorMessage()
+    {
+        if (_errorMessage != NULL)
+            return _errorMessage;
+        return search::engine::getStringFromErrorCode(_errorCode);
+    }
+
+    virtual void Interrupt()
+    {
+    }
+
+    virtual void Free()
+    {
+        delete this;
+    }
+};
+
+//----------------------------------------------------------------
+
+class FastS_FailedSearch : public FastS_SearchBase
+{
+private:
+    bool _async;
+
+public:
+    FastS_FailedSearch(uint32_t dataSetID,
+                       bool async,
+                       search::engine::ErrorCode errorCode,
+                       const char *errorMessage)
+        : FastS_SearchBase(dataSetID),
+          _async(async)
+    {
+        SetError(errorCode, errorMessage);
+    }
+    virtual ~FastS_FailedSearch() {}
+
+    virtual bool IsAsync() { return _async; }
+
+    virtual RetCode SetAsyncArgs(FastS_ISearchOwner *owner,
+                                 FastS_SearchContext context)
+    {
+        (void) owner;
+        (void) context;
+        return (_async) ? RET_OK : RET_ERROR;
+    }
+};
+
+//----------------------------------------------------------------
+
+class FastS_SyncSearch : public FastS_SearchBase
+{
+public:
+    FastS_SyncSearch(uint32_t dataSetID)
+        : FastS_SearchBase(dataSetID) {}
+
+    bool IsAsync() { return false; }
+
+    virtual RetCode SetAsyncArgs(FastS_ISearchOwner *,
+                                 FastS_SearchContext)
+    {
+        return RET_ERROR;
+    }
+};
+
+//----------------------------------------------------------------
+
+class FastS_AsyncSearch : public FastS_SearchBase
+{
+private:
+    FastS_AsyncSearch(const FastS_AsyncSearch &);
+    FastS_AsyncSearch& operator=(const FastS_AsyncSearch &);
+
+protected:
+    FastS_ISearchOwner  *_searchOwner;
+    FastS_SearchContext  _searchContext;
+
+public:
+    FastS_AsyncSearch(uint32_t dataSetID)
+        : FastS_SearchBase(dataSetID),
+          _searchOwner(NULL),
+          _searchContext(FastS_SearchContext()) {}
+
+    bool IsAsync() { return true; }
+
+    virtual RetCode SetAsyncArgs(FastS_ISearchOwner *owner,
+                                 FastS_SearchContext context)
+    {
+        _searchOwner   = owner;
+        _searchContext = context;
+        return RET_OK;
+    }
+};
+
+//----------------------------------------------------------------
+
+class FastS_SearchAdapter : public FastS_ISearch
+{
+private:
+    FastS_SearchAdapter(const FastS_SearchAdapter &);
+    FastS_SearchAdapter& operator=(const FastS_SearchAdapter &);
+
+protected:
+    FastS_ISearch *_search;
+
+public:
+    explicit FastS_SearchAdapter(FastS_ISearch *search);
+    virtual ~FastS_SearchAdapter();
+
+    virtual bool IsAsync();
+    virtual uint32_t GetDataSetID();
+    virtual FastS_SearchInfo *GetSearchInfo();
+    virtual RetCode SetAsyncArgs(FastS_ISearchOwner *owner,
+                                 FastS_SearchContext context);
+    virtual RetCode setSearchRequest(const search::engine::SearchRequest * request);
+    virtual RetCode SetGetDocsumArgs(search::docsummary::GetDocsumArgs *docsumArgs);
+    virtual RetCode Search(uint32_t searchOffset,
+                           uint32_t maxhits, uint32_t minhits = 0);
+    virtual RetCode ProcessQueryDone();
+    virtual FastS_QueryResult *GetQueryResult();
+    virtual RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt);
+    virtual RetCode ProcessDocsumsDone();
+    virtual FastS_DocsumsResult *GetDocsumsResult();
+    virtual search::engine::ErrorCode GetErrorCode();
+    virtual const char *GetErrorMessage();
+    virtual void Interrupt();
+    virtual void Free();
+};
+
+//----------------------------------------------------------------
+
+class FastS_SyncSearchAdapter : public FastS_SearchAdapter,
+                                public FastS_ISearchOwner
+{
+private:
+    FastOS_Cond _cond;
+    bool _waitQuery;
+    bool _queryDone;
+    bool _waitDocsums;
+    bool _docsumsDone;
+
+protected:
+    explicit FastS_SyncSearchAdapter(FastS_ISearch *search);
+
+public:
+    virtual ~FastS_SyncSearchAdapter();
+
+    static FastS_ISearch *Adapt(FastS_ISearch *search);
+
+    void Lock()    { _cond.Lock();   }
+    void Unlock()  { _cond.Unlock(); }
+    void Wait()    { _cond.Wait();   }
+    void Signal()  { _cond.Signal(); }
+
+    virtual void DoneQuery(FastS_ISearch *, FastS_SearchContext);
+    virtual void DoneDocsums(FastS_ISearch *, FastS_SearchContext);
+
+    void WaitQueryDone();
+    void WaitDocsumsDone();
+
+    virtual bool IsAsync();
+    virtual RetCode SetAsyncArgs(FastS_ISearchOwner *owner,
+                                 FastS_SearchContext context);
+    virtual RetCode Search(uint32_t searchOffset,
+                           uint32_t maxhits, uint32_t minhits = 0);
+    virtual RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt);
+};
+
+//----------------------------------------------------------------
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/stdincl.h b/searchcore/src/vespa/searchcore/fdispatch/common/stdincl.h
new file mode 100644
index 00000000000..f0378b951a4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/stdincl.h
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1999-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+/**
+ * This method defines the illegal/undefined value for unsigned 32-bit
+ * integer ids.
+ **/
+inline uint32_t FastS_NoID32() { return static_cast<uint32_t>(-1); }
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp
new file mode 100644
index 00000000000..57f951dc482
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp
@@ -0,0 +1,120 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/searchcore/fdispatch/common/timestat.h>
+
+
+void
+FastS_TimeStatHistory::Reset()
+{
+    _sampleAccTime = 0.0;
+    _totalAccTime = 0.0;
+    _sampleIdx = 0;
+    _sampleCount = 0;
+    _totalCount = 0;
+    for (uint32_t i = 0; i < _timestatssize; i++)
+        _sampleTimes[i] = Sample();
+}
+
+
+double
+FastS_TimeStatHistory::GetMaxTime(void) const
+{
+    double max = 0.0;
+    uint32_t idx = _sampleIdx;
+    for (uint32_t residue = _sampleCount;
+         residue > 0; residue--)
+    {
+        if (idx > 0)
+            idx--;
+        else
+            idx = _timestatssize - 1;
+        if (_sampleTimes[idx]._time > max)
+            max = _sampleTimes[idx]._time;
+    }
+    return max;
+}
+
+
+void
+FastS_TimeStatHistory::Update(double tnow, double t, bool timedout)
+{
+    uint32_t timeIdx = getTimeIdx(tnow);
+    if (_slotCount == 0u) {
+        _timeSlots[_slotIdx].init(timeIdx);
+        ++_slotCount;
+    } else {
+        TimeSlot &ts = _timeSlots[_slotIdx];
+        if (ts._timeIdx > timeIdx)
+            timeIdx = ts._timeIdx;
+        if (ts._timeIdx < timeIdx) {
+            if (_slotCount < NUM_TIMESLOTS)
+                ++_slotCount;
+            _slotIdx = nextTimeSlot(_slotIdx);
+            _timeSlots[_slotIdx].init(timeIdx);
+        }
+    }
+    _timeSlots[_slotIdx].update(t, timedout);
+
+    _totalAccTime += t;
+    ++_totalCount;
+    if (timedout)
+        ++_totalTimeouts;
+    if (_sampleCount >= _timestatssize) {
+        const Sample &s = _sampleTimes[_sampleIdx];
+        _sampleAccTime -= s._time;
+        if (s._timedout)
+            --_sampleTimeouts;
+        --_sampleCount;
+    }
+    _sampleTimes[_sampleIdx] = Sample(t, timedout);
+    _sampleAccTime += t;
+    if (timedout)
+        ++_sampleTimeouts;
+    _sampleIdx++;
+    if (_sampleIdx >= _timestatssize)
+        _sampleIdx = 0;
+    ++_sampleCount;
+}
+
+
+void
+FastS_TimeStatHistory::getRecentStats(double tsince,
+                                      FastS_TimeStatTotals &totals)
+{
+    uint32_t timeIdx = getTimeIdx(tsince);
+    uint32_t slotCount = _slotCount;
+    uint32_t slotIdx = _slotIdx;
+    for (; slotCount > 0u && _timeSlots[slotIdx]._timeIdx >= timeIdx;
+         --slotCount, slotIdx = prevTimeSlot(slotIdx)) {
+        TimeSlot &ts = _timeSlots[slotIdx];
+        totals._totalCount += ts._count;
+        totals._totalTimeouts += ts._timeouts;
+        totals._totalAccTime += ts._accTime;
+    }
+}
+
+
+double
+FastS_TimeStatHistory::getLoadTime(double tsince, double tnow)
+{
+    const uint32_t holeSize = 2;	// 2 missing slots => hole
+    const uint32_t minSlotLoad = 4;	// Mininum load for not being "missing"
+    uint32_t sinceTimeIdx = getTimeIdx(tsince);
+    uint32_t timeIdx = getTimeIdx(tnow);
+    uint32_t slotCount = _slotCount;
+    uint32_t slotIdx = _slotIdx;
+    uint32_t doneTimeIdx = timeIdx;
+    for (; slotCount > 0u; --slotCount, slotIdx = prevTimeSlot(slotIdx)) {
+        TimeSlot &ts = _timeSlots[slotIdx];
+        if (ts._timeIdx + holeSize < doneTimeIdx)
+            break;	// Found hole, i.e. holeSize missing slots
+        if (ts._timeIdx + holeSize < sinceTimeIdx)
+            break;	// No point in looking further back
+        if (ts._count >= minSlotLoad)
+            doneTimeIdx = ts._timeIdx;
+    }
+    return tnow - getTimeFromTimeIdx(doneTimeIdx);
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h
new file mode 100644
index 00000000000..66cab230589
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h
@@ -0,0 +1,215 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+
+class FastS_TimeStatTotals
+{
+public:
+    int _totalCount;
+    uint32_t _totalTimeouts;
+    double _totalAccTime;
+
+    FastS_TimeStatTotals(void)
+        : _totalCount(0),
+          _totalTimeouts(0u),
+          _totalAccTime(0)
+    {
+    }
+
+    FastS_TimeStatTotals &
+    operator+=(const FastS_TimeStatTotals &rhs)
+    {
+        _totalCount += rhs._totalCount;
+        _totalTimeouts += rhs._totalTimeouts;
+        _totalAccTime += rhs._totalAccTime;
+        return *this;
+    }
+
+    FastS_TimeStatTotals &
+    operator-=(const FastS_TimeStatTotals &rhs)
+    {
+        _totalCount -= rhs._totalCount;
+        _totalTimeouts -= rhs._totalTimeouts;
+        _totalAccTime -= rhs._totalAccTime;
+        return *this;
+    }
+
+    double
+    getAvgTime(void) const
+    {
+        if (_totalCount == 0)
+            return 0.0;
+        return (_totalAccTime / _totalCount);
+    }
+
+    double
+    getTimeoutRate(void) const
+    {
+        if (_totalCount == 0)
+            return 0.0;
+        return (static_cast<double>(_totalTimeouts) / _totalCount);
+    }
+};
+
+
+class FastS_TimeStatHistory
+{
+    enum {
+        _timestatssize = 100
+    };
+    enum {
+        SLOT_SIZE = 5,
+        NUM_TIMESLOTS = 128
+    };
+    double   _sampleAccTime;
+    double   _totalAccTime;
+    uint32_t _sampleIdx;
+    uint32_t _sampleCount;
+    uint32_t _sampleTimeouts;
+    uint32_t _totalCount;
+    uint32_t _totalTimeouts;
+    struct Sample {
+        double  _time;
+        bool _timedout;
+
+        Sample()
+            : _time(0.0),
+              _timedout(false)
+        {
+        }
+
+        Sample(double time, bool timedout)
+            : _time(time),
+              _timedout(timedout)
+        {
+        }
+    };
+    Sample _sampleTimes[_timestatssize];
+
+    struct TimeSlot
+    {
+        double _accTime;
+        uint32_t _count;
+        uint32_t _timeouts;
+        uint32_t _timeIdx;
+
+        TimeSlot()
+            : _accTime(0.0),
+              _count(0u),
+              _timeouts(0u),
+              _timeIdx(0u)
+        {
+        }
+
+        void
+        init(uint32_t timeIdx)
+        {
+            _accTime = 0.0;
+            _count = 0u;
+            _timeouts = 0u;
+            _timeIdx = timeIdx;
+        }
+
+        void
+        update(double t, bool timedout)
+        {
+            _accTime += t;
+            ++_count;
+            if (timedout)
+                ++_timeouts;
+        }
+    };
+
+    static uint32_t
+    nextTimeSlot(uint32_t timeSlot)
+    {
+        return (timeSlot >= (NUM_TIMESLOTS - 1)) ? 0u : timeSlot + 1;
+    }
+
+    static uint32_t
+    prevTimeSlot(uint32_t timeSlot)
+    {
+        return (timeSlot == 0u) ? (NUM_TIMESLOTS - 1) : timeSlot - 1;
+    }
+
+
+    TimeSlot _timeSlots[NUM_TIMESLOTS];
+    uint32_t _slotCount;
+    uint32_t _slotIdx;
+
+    static uint32_t
+    getTimeIdx(double t)
+    {
+        // Each SLOT_SIZE second period has it's own slot of statistics
+        return static_cast<uint32_t>(t / SLOT_SIZE);
+    }
+
+    static double
+    getTimeFromTimeIdx(uint32_t timeIdx)
+    {
+        return static_cast<double>(timeIdx) * SLOT_SIZE;
+    }
+public:
+    FastS_TimeStatHistory(void)
+        : _sampleAccTime(0.0),
+          _totalAccTime(0.0),
+          _sampleIdx(0),
+          _sampleCount(0),
+          _sampleTimeouts(0),
+          _totalCount(0),
+          _totalTimeouts(0u),
+          _sampleTimes(),
+          _timeSlots(),
+          _slotCount(0u),
+          _slotIdx(0u)
+    {
+        Reset();
+    }
+
+    void Reset(void);
+
+    double  GetSampleAccTime(void) const { return _sampleAccTime; }
+    uint32_t GetSampleCount(void) const { return _sampleCount; }
+
+    uint32_t
+    getSampleTimeouts(void) const
+    {
+        return _sampleTimeouts;
+    }
+
+    double GetAvgTime() const
+    {
+        if (_sampleCount == 0)
+            return 0.0;
+        return (_sampleAccTime / _sampleCount);
+    }
+    double GetMaxTime(void) const;
+
+    void Update(double tnow, double t, bool timedout);
+
+    uint32_t GetTotalCount(void) const { return _totalCount; }
+    double  GetTotalAccTime(void) const { return _totalAccTime; }
+
+    uint32_t
+    getTotalTimeouts(void) const
+    {
+        return _totalTimeouts;
+    }
+
+    void AddTotal(FastS_TimeStatTotals *totals) {
+        totals->_totalCount += GetTotalCount();
+        totals->_totalTimeouts += getTotalTimeouts();
+        totals->_totalAccTime += GetTotalAccTime();
+    }
+
+    void
+    getRecentStats(double tsince, FastS_TimeStatTotals &totals);
+
+    double
+    getLoadTime(double tsince, double tnow);
+};
+
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/.gitignore b/searchcore/src/vespa/searchcore/fdispatch/program/.gitignore
new file mode 100644
index 00000000000..27eb860d05b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/.gitignore
@@ -0,0 +1,13 @@
+*.exp
+*.ilk
+*.lib
+*.pdb
+.depend
+ID
+Makefile
+access_log
+debugmalloc
+fdispatch
+fdispatch.exe
+sfdispatch
+sfdispatch.exe
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/CMakeLists.txt b/searchcore/src/vespa/searchcore/fdispatch/program/CMakeLists.txt
new file mode 100644
index 00000000000..374f1b94413
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_fdispatch_program STATIC
+    SOURCES
+    fdispatch.cpp
+    rpc.cpp
+    engineadapter.cpp
+    searchadapter.cpp
+    docsumadapter.cpp
+    DEPENDS
+    searchcore_fconfig
+)
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/description.html b/searchcore/src/vespa/searchcore/fdispatch/program/description.html
new file mode 100644
index 00000000000..98e9a5b18c9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/description.html
@@ -0,0 +1,3 @@
+<!-- Short description for make kdoc. -->
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+FDispatch program.
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp
new file mode 100644
index 00000000000..32362dba422
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".fdispatch.docsumadapter");
+#include "docsumadapter.h"
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchlib/common/packets.h>
+#include <vespa/searchlib/fef/queryproperties.h>
+#include <vespa/searchlib/parsequery/simplequerystack.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+
+namespace fdispatch {
+
+void
+DocsumAdapter::setupRequest()
+{
+    const DocsumRequest &req = *_request.get();
+    _args.initFromDocsumRequest(req);
+    _hitcnt = req.hits.size();
+    LOG(debug, "DocsumAdapter::setupRequest : hitcnt=%d", _hitcnt);
+    if (_hitcnt > 0) {
+        _hitbuf = (FastS_hitresult *) malloc(req.hits.size() * sizeof(FastS_hitresult));
+    }
+    for (uint32_t i = 0; i < _hitcnt; i++) {
+        _hitbuf[i]._gid       = req.hits[i].gid;
+        _hitbuf[i]._partition = req.hits[i].path;
+        LOG(debug, "DocsumAdapter::setupRequest : hit[%d] (gid=%s,part=%d)",
+            i, _hitbuf[i]._gid.toString().c_str(), _hitbuf[i]._partition);
+    }
+}
+
+void
+DocsumAdapter::handleRequest()
+{
+    _dsc = _appCtx->GetDataSetCollection();
+    FastS_assert(_dsc != NULL);
+    _search = _dsc->CreateSearch(FastS_NoID32(), _appCtx->GetTimeKeeper());
+    FastS_assert(_search != NULL);
+    _docsumsResult = _search->GetDocsumsResult();
+    _search->SetGetDocsumArgs(&_args);
+    _search->GetDocsums(_hitbuf, _hitcnt);
+    _search->ProcessDocsumsDone();
+}
+
+void
+DocsumAdapter::createReply()
+{
+    DocsumReply::UP reply(new DocsumReply());
+    DocsumReply &r = *reply;
+
+    FastS_fullresult *hitbuf = _docsumsResult->_fullresult;
+    uint32_t          hitcnt = _docsumsResult->_fullResultCount;
+
+    LOG(debug, "DocsumAdapter::createReply : hitcnt=%d", hitcnt);
+    r.docsums.reserve(hitcnt);
+    for (uint32_t i = 0; i < hitcnt; i++) {
+        if ( ! hitbuf[i]._buf.empty() ) {
+            r.docsums.push_back(DocsumReply::Docsum());
+            DocsumReply::Docsum & d = r.docsums.back();
+            d.docid = hitbuf[i]._docid;
+            d.gid = hitbuf[i]._gid;
+            d.data.swap(hitbuf[i]._buf);
+        } else {
+            LOG(debug, "DocsumAdapter::createReply : No buf for hit=%d", i);
+        }
+    }
+    r.request = _request.release();
+    _client.getDocsumsDone(std::move(reply));
+}
+
+void
+DocsumAdapter::writeLog()
+{
+    // no access log for docsums
+}
+
+void
+DocsumAdapter::cleanup()
+{
+    if (_search != NULL) {
+        _search->Free();
+    }
+    if (_dsc != NULL) {
+        _dsc->subRef();
+    }
+    free(_hitbuf);
+    _hitcnt = 0;
+    _hitbuf = 0;
+}
+
+void
+DocsumAdapter::Run(FastOS_ThreadInterface *, void *)
+{
+    setupRequest();
+    handleRequest();
+    createReply();
+    writeLog();
+    cleanup();
+    delete this;
+}
+
+DocsumAdapter::DocsumAdapter(FastS_AppContext *appCtx,
+                             DocsumRequest::Source request,
+                             DocsumClient &client)
+    : _appCtx(appCtx),
+      _request(std::move(request)),
+      _client(client),
+      _args(),
+      _hitcnt(0),
+      _hitbuf(0),
+      _dsc(0),
+      _search(0),
+      _docsumsResult(0)
+{
+}
+
+} // namespace fdispatch
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.h b/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.h
new file mode 100644
index 00000000000..a168ca1e5a1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/engine/docsumapi.h>
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+#include <vespa/searchsummary/docsummary/getdocsumargs.h>
+
+namespace fdispatch {
+
+/**
+ * Implementation of the common search api for the fdispatch server
+ * application.
+ **/
+class DocsumAdapter : public FastOS_Runnable
+{
+public:
+    typedef search::engine::DocsumRequest DocsumRequest;
+    typedef search::engine::DocsumReply   DocsumReply;
+    typedef search::engine::DocsumClient  DocsumClient;
+
+private:
+    FastS_AppContext          *_appCtx;
+    DocsumRequest::Source      _request;
+    DocsumClient              &_client;
+
+    // internal docsum related state
+    search::docsummary::GetDocsumArgs  _args;
+    uint32_t                           _hitcnt;
+    FastS_hitresult                   *_hitbuf;
+    FastS_DataSetCollection           *_dsc;
+    FastS_ISearch                     *_search;
+    FastS_DocsumsResult               *_docsumsResult;
+
+    void setupRequest();
+    void handleRequest();
+    void createReply();
+    void writeLog();
+    void cleanup();
+
+    virtual void Run(FastOS_ThreadInterface *, void *);
+
+public:
+    DocsumAdapter(FastS_AppContext *appCtx,
+                  DocsumRequest::Source request,
+                  DocsumClient &client);
+};
+
+} // namespace fdispatch
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp
new file mode 100644
index 00000000000..670e3425f59
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".fdispatch.engineadapter");
+#include "engineadapter.h"
+#include <vespa/searchlib/common/packets.h>
+#include <vespa/searchcore/fdispatch/search/child_info.h>
+#include <vespa/searchcore/fdispatch/search/nodemanager.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+#include <vespa/searchcore/fdispatch/search/dataset_base.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+#include "searchadapter.h"
+#include "docsumadapter.h"
+
+namespace fdispatch {
+
+EngineAdapter::
+EngineAdapter(FastS_AppContext *appCtx, FastOS_ThreadPool *threadPool)
+    : _appCtx(appCtx),
+      _mypool(threadPool)
+{
+}
+
+EngineAdapter::SearchReply::UP
+EngineAdapter::search(SearchRequest::Source request, SearchClient &client)
+{
+    SearchAdapter *sa = new SearchAdapter(_appCtx, std::move(request), client);
+    if (_mypool == 0 || _mypool->NewThread(sa) == 0) {
+        LOG(error, "could not allocate thread for incoming search request");
+        SearchReply::UP reply(new SearchReply());
+        reply->useWideHits = true; // mld
+        if (sa->allowError()) {
+            reply->errorCode = search::engine::ECODE_OVERLOADED;
+            reply->errorMessage = "could not allocate thread for query";
+        }
+        delete sa;
+        return reply;
+    }
+    return SearchReply::UP();
+}
+
+EngineAdapter::DocsumReply::UP
+EngineAdapter::getDocsums(DocsumRequest::Source request, DocsumClient &client)
+{
+    DocsumAdapter *da = new DocsumAdapter(_appCtx, std::move(request), client);
+    if (_mypool == 0 || _mypool->NewThread(da) == 0) {
+        LOG(error, "could not allocate thread for incoming docsum request");
+        delete da;
+        return DocsumReply::UP(new DocsumReply());
+    }
+    return DocsumReply::UP();
+}
+
+EngineAdapter::MonitorReply::UP
+EngineAdapter::ping(MonitorRequest::UP request, MonitorClient &)
+{
+    MonitorReply::UP reply(new MonitorReply());
+    MonitorReply &mr = *reply;
+
+    uint32_t timeStamp = 0;
+    FastS_NodeManager *nm = _appCtx->GetNodeManager();
+
+    ChildInfo ci = nm->getChildInfo();
+    timeStamp = nm->GetMldDocstamp();
+    // TODO: Report softoffline upwards when fdispatch has been requested
+    // to go down in a controlled manner (along with zero docstamp).
+    mr.partid      = nm->GetMldPartition();
+    mr.timestamp   = timeStamp;
+    mr.mld         = true;
+    mr.totalNodes  = ci.maxNodes;
+    mr.activeNodes = ci.activeNodes;
+    mr.totalParts  = ci.maxParts;
+    mr.activeParts = ci.activeParts;
+    if (ci.activeDocs.valid) {
+        mr.activeDocs = ci.activeDocs.count;
+        mr.activeDocsRequested = request->reportActiveDocs;
+    }
+    return reply;
+}
+
+} // namespace fdispatch
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.h b/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.h
new file mode 100644
index 00000000000..2667664a7db
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/engine/searchapi.h>
+#include <vespa/searchlib/engine/docsumapi.h>
+#include <vespa/searchlib/engine/monitorapi.h>
+
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+
+namespace fdispatch {
+
+/**
+ * Implementation of the common search api for the fdispatch server
+ * application.
+ **/
+class EngineAdapter : public search::engine::SearchServer,
+                      public search::engine::DocsumServer,
+                      public search::engine::MonitorServer
+{
+private:
+    FastS_AppContext         *_appCtx;
+    FastOS_ThreadPool        *_mypool;
+
+public:
+    typedef search::engine::SearchRequest SearchRequest;
+    typedef search::engine::DocsumRequest DocsumRequest;
+    typedef search::engine::MonitorRequest MonitorRequest;
+
+    typedef search::engine::SearchReply SearchReply;
+    typedef search::engine::DocsumReply DocsumReply;
+    typedef search::engine::MonitorReply MonitorReply;
+
+    typedef search::engine::SearchClient SearchClient;
+    typedef search::engine::DocsumClient DocsumClient;
+    typedef search::engine::MonitorClient MonitorClient;
+
+    EngineAdapter(FastS_AppContext *appCtx,
+                  FastOS_ThreadPool *threadPool);
+
+    virtual SearchReply::UP search(SearchRequest::Source request, SearchClient &client);
+    virtual DocsumReply::UP getDocsums(DocsumRequest::Source request, DocsumClient &client);
+    virtual MonitorReply::UP ping(MonitorRequest::UP request, MonitorClient &client);
+};
+
+} // namespace fdispatch
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp
new file mode 100644
index 00000000000..193c4365189
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp
@@ -0,0 +1,428 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".fdispatch");
+
+#include "fdispatch.h" 
+
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/util/eventloop.h>
+
+#include <vespa/searchcore/fdispatch/search/querycacheutil.h>
+
+#include <vespa/searchcore/fdispatch/search/nodemanager.h>
+#include "engineadapter.h"
+#include "rpc.h"
+
+#include <vespa/fnet/frt/frt.h>
+#include <vespa/searchlib/common/packets.h>
+
+#ifndef V_TAG
+#define V_TAG "NOTAG"
+#endif
+
+using search::fs4transport::FS4PersistentPacketStreamer;
+using vespa::config::search::core::FdispatchrcConfig;
+using vespa::config::search::core::internal::InternalFdispatchrcType;
+using document::CompressionConfig;
+
+char FastS_VersionTag[] = V_TAG;
+
+int		FastS_verbose 		= 0;
+/** @todo Use a config file for these variables */
+int		FastS_nsearches;
+double		FastS_searchtime;
+
+
+namespace fdispatch
+{
+
+FastS_FNETAdapter::FastS_FNETAdapter(FastS_AppContext *appCtx)
+    : _appCtx(appCtx),
+      _nodeManager(NULL),
+      _timeKeeper(NULL),
+      _transport(NULL),
+      _last_now(0.0),
+      _live_counter(0),
+      _task()
+{
+}
+
+FastS_FNETAdapter::~FastS_FNETAdapter()
+{
+    fini();
+}
+
+void
+FastS_FNETAdapter::init()
+{
+    _nodeManager = _appCtx->GetNodeManager();
+    _timeKeeper  = _appCtx->GetTimeKeeper();
+    _transport   = _appCtx->GetFNETTransport();
+    _last_now    = _timeKeeper->GetTime();
+    _task.reset(new MyTask(_transport->GetScheduler(), *this));
+    _task->ScheduleNow();
+}
+
+void
+FastS_FNETAdapter::perform()
+{
+    double now = _timeKeeper->GetTime();
+    double delta = now - _last_now;
+    if (delta >= 3.0) {
+        LOG(warning, "FNET loop high latency: %.3f", delta);
+    }
+    _last_now = now;
+    ++_live_counter;
+    _nodeManager->CheckEvents(_timeKeeper);
+}
+
+void
+FastS_FNETAdapter::fini()
+{
+    if (_task) {
+        _task->Kill();
+        _task.reset();
+    }
+}
+
+
+Fdispatch::~Fdispatch(void)
+{
+    if (_transportServer != NULL) {
+        _transportServer->shutDown(); // sync shutdown
+    }
+    _FNET_adapter.fini();
+    if (_nodeManager != NULL)
+        _nodeManager->ShutdownConfig();
+    if (_transport != NULL && _transportStarted)
+        _transport->ShutDown(true); // sync shutdown
+    if (_rpc != NULL)
+        _rpc->ShutDown(); // sync shutdown
+
+    LOG(debug, "Will close threadpool");
+    _mypool->Close();
+    LOG(debug, "Has closed threadpool");
+    delete _transportServer;
+    delete _engineAdapter;
+    delete _nodeManager;
+    if (_transport != NULL) {
+        delete _transport;
+    }
+    delete _rpc;
+    delete _mypool;
+}
+
+
+FNET_Transport *
+Fdispatch::GetFNETTransport()
+{
+    return _transport;
+}
+
+
+FNET_Scheduler *
+Fdispatch::GetFNETScheduler()
+{
+    return (_transport == NULL) ?
+                        NULL :  _transport->GetScheduler();
+}
+
+
+FastS_NodeManager *
+Fdispatch::GetNodeManager()
+{
+    return _nodeManager;
+}
+
+
+FastS_DataSetCollection *
+Fdispatch::GetDataSetCollection()
+{
+    if (_nodeManager == NULL)
+        return NULL;
+    return _nodeManager->GetDataSetCollection();
+}
+
+
+FastOS_ThreadPool *
+Fdispatch::GetThreadPool()
+{
+    return _mypool;
+}
+
+
+bool
+Fdispatch::Failed(void)
+{
+    return ( (_transportServer != NULL && _transportServer->isFailed()));
+}
+
+
+bool
+Fdispatch::CheckTempFail(void)
+{
+    bool ret;
+    bool failflag = _nodeManager->GetTempFail();
+    unsigned int FNETLiveCounter;
+
+    ret = true;
+
+    FNETLiveCounter = _FNET_adapter.GetLiveCounter();
+    if (FNETLiveCounter == _lastFNETLiveCounter) {
+        if (_FNETLiveCounterFailed) {
+            failflag = true;				// Still failure
+        } else if (!_FNETLiveCounterDanger) {
+            _FNETLiveCounterDanger = true;
+            _FNETLiveCounterDangerStart.SetNow();
+        } else if (_FNETLiveCounterDangerStart.MilliSecsToNow() >= 6000) {
+            LOG(error, "fdispatch::Fdispatch::CheckTempFail: "
+                "FNET inactive for 6 seconds, deadlock ?");
+            _FNETLiveCounterFailed = true;		// Note that we failed
+            failflag = true;				// Force temporary failure
+        } else if (_FNETLiveCounterDangerStart.MilliSecsToNow() >= 3000 &&
+                   !_FNETLiveCounterWarned) {
+            _FNETLiveCounterWarned = true;
+            LOG(warning,
+                "fdispatch::Fdispatch::CheckTempFail: "
+                "FNET inactive for 3 seconds");
+        }
+    } else {
+        if (_FNETLiveCounterFailed || _FNETLiveCounterWarned) {
+            LOG(warning,
+                "fdispatch::Fdispatch::CheckTempFail: FNET active again");
+        }
+        _FNETLiveCounterFailed = false;
+        _FNETLiveCounterWarned = false;
+        _FNETLiveCounterDanger = false;
+        _lastFNETLiveCounter = FNETLiveCounter;
+    }
+
+    if (failflag == _tempFail)
+        return ret;
+
+    if (_transportServer != NULL) {
+        if (failflag) {
+            _transportServer->setListen(false);
+            LOG(error, "Disabling fnet server interface");
+        } else {
+            _transportServer->setListen(true);
+            LOG(info, "Reenabling fnet server interface");
+        }
+    }
+    _tempFail = failflag;
+    return ret;
+}
+
+
+/**
+ * Make the httpd and Monitor, and let a Thread execute each.
+ * Set up stuff as specified in the fdispatch-rc-file.
+ */
+Fdispatch::Fdispatch(const config::ConfigUri &configUri)
+    : _mypool(NULL),
+      _engineAdapter(NULL),
+      _transportServer(NULL),
+      _componentConfig(),
+      _nodeManager(NULL),
+      _transport(NULL),
+      _FNET_adapter(this),
+      _rpc(NULL),
+      _config(),
+      _configUri(configUri),
+      _partition(0),
+      _tempFail(false),
+      _FNETLiveCounterDanger(false),
+      _FNETLiveCounterWarned(false),
+      _FNETLiveCounterFailed(false),
+      _transportStarted(false),
+      _lastFNETLiveCounter(false),
+      _FNETLiveCounterDangerStart(),
+      _timeouts(0u),
+      _checkLimit(0u),
+      _healthPort(0)
+{
+    int64_t cfgGen = -1;
+    _config = config::ConfigGetter<FdispatchrcConfig>::
+              getConfig(cfgGen,
+                        _configUri.getConfigId(),
+                        _configUri.getContext());
+    LOG(config, "fdispatch version %s (RPC-port: %d, transport at %d)",
+                FastS_VersionTag, _config->frtport, _config->ptport);
+
+    _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("fdispatch", cfgGen,
+                                       "config only obtained at startup"));
+}
+
+
+void
+Fdispatch::CheckCacheMaxEntries(unsigned int queryCacheMaxEntries,
+                                      unsigned int queryAttrCacheMaxEntries)
+{
+    if (queryAttrCacheMaxEntries == 0)
+        return;
+
+    if ((queryAttrCacheMaxEntries <= queryCacheMaxEntries) ||
+        (queryCacheMaxEntries == 0)) {
+        FastS_abort("Please edit fdispatchrc such that "
+                    "queryattrcachequeries > querycachequeries.");
+    }
+}
+
+namespace {
+
+CompressionConfig::Type
+convert(InternalFdispatchrcType::Packetcompresstype type)
+{
+    switch (type) {
+      case InternalFdispatchrcType::LZ4: return CompressionConfig::LZ4;
+      default: return CompressionConfig::LZ4;
+    }
+}
+
+}
+
+bool
+Fdispatch::Init(void)
+{
+    int  maxthreads;
+
+    _tempFail = false;
+    _FNETLiveCounterDanger = false;
+    _FNETLiveCounterWarned = false;
+    _FNETLiveCounterFailed = false;
+    _lastFNETLiveCounter = 0;
+    _timeouts = 0;
+    _checkLimit = 60;
+
+    FS4PersistentPacketStreamer::Instance.SetCompressionLimit(
+            _config->packetcompresslimit);
+    FS4PersistentPacketStreamer::Instance.SetCompressionLevel(
+            _config->packetcompresslevel);
+    FS4PersistentPacketStreamer::Instance.SetCompressionType(
+            convert(_config->packetcompresstype));
+    
+
+    LOG(debug, "Creating FNET transport");
+    _transport = new FNET_Transport(_config->transportthreads);
+
+    // grab node slowness limit defaults
+
+    FastS_DataSetDesc::SetDefaultSlowQueryLimitFactor(_config->defaultslowquerylimitfactor);
+    FastS_DataSetDesc::SetDefaultSlowQueryLimitBias(_config->defaultslowquerylimitbias);
+    FastS_DataSetDesc::SetDefaultSlowDocsumLimitFactor(_config->defaultslowdocsumlimitfactor);
+    FastS_DataSetDesc::SetDefaultSlowDocsumLimitBias(_config->defaultslowdocsumlimitbias);
+
+    maxthreads = _config->maxthreads;
+    _mypool = new FastOS_ThreadPool(256 * 1024, maxthreads);
+
+    // Max interval betw read from socket.
+    FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent] =
+        _config->maxsocksilent;
+
+    if (_transport != NULL)
+        _transport->SetIOCTimeOut((uint32_t)
+                                  (FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent] * 1000.0));
+
+    char timestr[40];
+    FastS_TimeOut::WriteTime(timestr, sizeof(timestr),
+                             FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent]);
+    LOG(debug,
+        "VERBOSE: Max time between successful read from a socket: %s",
+        timestr);
+
+    FastS_QueryCacheUtil::_systemMaxHits = std::numeric_limits<int>::max();
+    LOG(debug, "VERBOSE: maxhits: %d", FastS_QueryCacheUtil::_systemMaxHits);
+
+    FastS_QueryCacheUtil::_maxOffset = std::numeric_limits<int>::max();
+    const uint32_t linesize = 1;
+    if (FastS_QueryCacheUtil::_systemMaxHits < linesize
+        && FastS_QueryCacheUtil::_maxOffset < linesize - FastS_QueryCacheUtil::_systemMaxHits) {
+        LOG(warning,
+            "maxoffset must be >= %d! (overriding config value)",
+            linesize - FastS_QueryCacheUtil::_systemMaxHits);
+        FastS_QueryCacheUtil::_maxOffset = linesize - FastS_QueryCacheUtil::_systemMaxHits;
+    }
+    LOG(debug, "VERBOSE: maxoffset: %d", FastS_QueryCacheUtil::_maxOffset);
+
+    _partition = _config->partition;
+
+    int ptportnum = _config->ptport;
+
+    LOG(debug, "Using port number %d", ptportnum);
+
+    _nodeManager = new FastS_NodeManager(_componentConfig, this, _partition);
+
+
+    GetFNETTransport()->SetTCPNoDelay(_config->transportnodelay);
+    GetFNETTransport()->SetDirectWrite(_config->transportdirectwrite);
+
+    assert (ptportnum != 0);
+
+    _engineAdapter = new fdispatch::EngineAdapter(this, _mypool);
+    _transportServer = new search::engine::TransportServer
+        (*_engineAdapter, *_engineAdapter, *_engineAdapter, ptportnum, search::engine::TransportServer::DEBUG_ALL);
+    _transportServer->setTCPNoDelay(_config->transportnodelay);
+    _transportServer->setDirectWrite(_config->transportdirectwrite);
+
+    if (!_transportServer->start()) {
+        delete _transportServer;
+        _transportServer = NULL;
+        delete _engineAdapter;
+        _engineAdapter = NULL;
+        LOG(error, "CRITICAL: Failed to init upwards FNET transport on port %d", ptportnum);
+        return false;
+    }
+
+    _nodeManager->SubscribePartMap(_configUri);
+
+    if (_config->frtport != 0) {
+        _rpc = new FastS_fdispatch_RPC(this);
+        FastS_assert(_rpc != NULL);
+        if (!_rpc->Init(_config->frtport, _configUri.getConfigId())) {
+            LOG(error, "RPC init failed");
+            delete _rpc;
+            _rpc = NULL;
+        }
+    } else {
+        _rpc = NULL;
+    }
+
+    // Kick off fdispatch administrative threads.
+    if (_transport != NULL) {
+        _FNET_adapter.init();
+        bool rc = _transport->Start(_mypool);
+        if (rc) {
+            LOG(debug, "Started FNET transport");
+            _transportStarted = true;
+        } else {
+            LOG(error, "Failed to start FNET transport");
+        }
+    }
+    FastOS_Thread::Sleep(1000);
+    if (_rpc != NULL) {
+        _rpc->Start();
+    }
+    _healthPort = _config->healthport;
+    return true;
+}
+
+
+void
+Fdispatch::logPerformance()
+{
+    _nodeManager->logPerformance();
+}
+
+uint32_t
+Fdispatch::getDispatchLevel()
+{
+    return _config->dispatchlevel;
+}
+
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h
new file mode 100644
index 00000000000..b4ff8d9f37e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fnet/fnet.h>
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+#include <vespa/searchlib/engine/transportserver.h>
+#include <vespa/searchcore/config/config-fdispatchrc.h>
+#include <vespa/config/subscription/configuri.h>
+#include <vespa/vespalib/net/simple_component_config_producer.h>
+
+class FastS_NodeManager;
+class FastS_fdispatch_RPC;
+
+namespace fdispatch
+{
+
+class EngineAdapter;
+
+class FastS_FNETAdapter
+{
+private:
+    FastS_AppContext  *_appCtx;
+    FastS_NodeManager *_nodeManager;
+    FastS_TimeKeeper  *_timeKeeper;
+    FNET_Transport    *_transport;
+    double             _last_now;     // latency check
+    uint32_t           _live_counter; // latency check
+
+    struct MyTask : FNET_Task {
+        FastS_FNETAdapter &self;
+        MyTask(FNET_Scheduler *scheduler, FastS_FNETAdapter &self_in)
+            : FNET_Task(scheduler), self(self_in) {}
+        virtual void PerformTask() {
+            self.perform();
+            ScheduleNow();
+        }
+    };
+    std::unique_ptr<MyTask> _task;
+
+public:
+    FastS_FNETAdapter(FastS_AppContext *appCtx);
+    ~FastS_FNETAdapter();
+    void init();
+    void perform();
+    uint32_t GetLiveCounter() const { return _live_counter; }
+    void fini();
+};
+
+
+/**
+ * Note: There is only one instance of this.
+ */
+class Fdispatch : public FastS_AppContext
+{
+private:
+    Fdispatch(const Fdispatch &);
+    Fdispatch& operator=(const Fdispatch &);
+
+    FastOS_ThreadPool *_mypool;
+    EngineAdapter        *_engineAdapter;
+    search::engine::TransportServer *_transportServer;
+    vespalib::SimpleComponentConfigProducer _componentConfig;
+    FastS_NodeManager *_nodeManager;
+    FNET_Transport *_transport;
+    FastS_FNETAdapter _FNET_adapter;
+    FastS_fdispatch_RPC *_rpc;
+    std::unique_ptr<vespa::config::search::core::FdispatchrcConfig> _config;
+    config::ConfigUri _configUri;
+    unsigned int _partition;
+    bool _tempFail;
+    bool _FNETLiveCounterDanger;
+    bool _FNETLiveCounterWarned;
+    bool _FNETLiveCounterFailed;
+    bool _transportStarted;
+    unsigned int _lastFNETLiveCounter;
+    FastOS_Time _FNETLiveCounterDangerStart;
+    unsigned int _timeouts;
+    unsigned int _checkLimit;
+    int _healthPort;
+public:
+    // Implements FastS_AppContext
+    virtual FNET_Transport *GetFNETTransport();
+    virtual FNET_Scheduler *GetFNETScheduler();
+    virtual FastS_NodeManager *GetNodeManager();
+    virtual FastS_DataSetCollection *GetDataSetCollection();
+    virtual FastOS_ThreadPool *GetThreadPool();
+    virtual void logPerformance();
+    virtual uint32_t getDispatchLevel();
+    bool CheckTempFail(void);
+    bool Failed(void);
+    bool Init(void);
+    int getHealthPort() const { return _healthPort; }
+    vespalib::SimpleComponentConfigProducer &getComponentConfig() { return _componentConfig; }
+
+    void
+    CheckCacheMaxEntries(unsigned int queryCacheMaxEntries,
+                         unsigned int queryAttrCacheMaxEntries);
+
+    Fdispatch(const config::ConfigUri &configUri);
+    ~Fdispatch(void);
+};
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp
new file mode 100644
index 00000000000..5f571f0ecd1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <functional>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".rpc");
+
+#include <vespa/fnet/frt/frt.h>
+
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchlib/common/transport.h>
+#include <vespa/searchlib/parsequery/simplequerystack.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+#include <vespa/searchcore/fdispatch/search/engine_base.h>
+#include <vespa/searchcore/fdispatch/search/plain_dataset.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+
+#include <vespa/searchcore/fdispatch/program/rpc.h>
+
+
+void
+FastS_fdispatch_RPC::RegisterMethods(FRT_ReflectionBuilder *rb)
+{
+    FastS_RPC::RegisterMethods(rb);
+    //------------------------------------------------------------------
+    rb->DefineMethod("fs.admin.enableEngine", "s", "i", true,
+                     FRT_METHOD(FastS_fdispatch_RPC::RPC_EnableEngine), this);
+    rb->MethodDesc("Enable the given engine (clear badness).");
+    rb->ParamDesc("name",  "engine name");
+    rb->ReturnDesc("count", "number of engines affected");
+    //------------------------------------------------------------------
+    rb->DefineMethod("fs.admin.disableEngine", "s", "i", true,
+                     FRT_METHOD(FastS_fdispatch_RPC::RPC_DisableEngine), this);
+    rb->MethodDesc("Disable the given engine (mark as admin bad).");
+    rb->ParamDesc("name",  "engine name");
+    rb->ReturnDesc("count", "number of engines affected");
+}
+
+
+void
+FastS_fdispatch_RPC::RPC_GetNodeType(FRT_RPCRequest *req)
+{
+    req->GetReturn()->AddString("dispatch");
+}
+
+namespace {
+
+template<class FUN>
+struct ExecuteWhenEqualName_t {
+    FUN _successFun;
+    const char* _targetName;
+    uint32_t _cnt;
+
+    ExecuteWhenEqualName_t(const char* targetName, FUN successFun)
+        : _successFun(successFun),
+          _targetName(targetName),
+          _cnt(0)
+    {}
+
+    void operator()(FastS_EngineBase* engine) {
+        if (strcmp(engine->GetName(), _targetName) ==  0 ) {
+            _cnt++;
+            _successFun(engine);
+        }
+    }
+};
+
+template <class FUN>
+ExecuteWhenEqualName_t<FUN>
+ExecuteWhenEqualName(const char* targetName, FUN successFun) {
+    return ExecuteWhenEqualName_t<FUN>(targetName, successFun);
+}
+
+
+} //anonymous namespace
+
+void
+FastS_fdispatch_RPC::RPC_EnableEngine(FRT_RPCRequest *req)
+{
+    const char *name = req->GetParams()->GetValue(0)._string._str;
+    FastS_DataSetCollection *dsc = GetAppCtx()->GetDataSetCollection();
+    uint32_t cnt = 0;
+
+    for (uint32_t i = 0; i < dsc->GetMaxNumDataSets(); i++) {
+        FastS_DataSetBase *ds;
+        FastS_PlainDataSet *ds_plain;
+        if ((ds = dsc->PeekDataSet(i)) == NULL ||
+            (ds_plain = ds->GetPlainDataSet()) == NULL)
+            continue;
+
+        cnt += ds_plain->ForEachEngine(
+                ExecuteWhenEqualName(name,
+                        std::mem_fun( &FastS_EngineBase::ClearBad )))
+                       ._cnt;
+    }
+
+    dsc->subRef();
+    req->GetReturn()->AddInt32(cnt);
+}
+
+
+void
+FastS_fdispatch_RPC::RPC_DisableEngine(FRT_RPCRequest *req)
+{
+    const char *name = req->GetParams()->GetValue(0)._string._str;
+    FastS_DataSetCollection *dsc = GetAppCtx()->GetDataSetCollection();
+    uint32_t cnt = 0;
+
+    for (uint32_t i = 0; i < dsc->GetMaxNumDataSets(); i++) {
+        FastS_DataSetBase *ds;
+        FastS_PlainDataSet *ds_plain;
+        if ((ds = dsc->PeekDataSet(i)) == NULL ||
+            (ds_plain = ds->GetPlainDataSet()) == NULL)
+            continue;
+
+        uint32_t badness = FastS_EngineBase::BAD_ADMIN;
+        cnt += ds_plain->ForEachEngine(
+                ExecuteWhenEqualName(name,
+                        std::bind2nd(
+                                std::mem_fun( &FastS_EngineBase::MarkBad ),
+                                badness)))
+               ._cnt;
+    }
+    dsc->subRef();
+    req->GetReturn()->AddInt32(cnt);
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h
new file mode 100644
index 00000000000..eb4f87610b9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/fnet/frt/frt.h>
+#include <vespa/searchcore/fdispatch/common/rpc.h>
+
+
+class FastS_fdispatch_RPC : public FastS_RPC
+{
+public:
+    FastS_fdispatch_RPC(FastS_AppContext *appCtx)
+        : FastS_RPC(appCtx) {}
+    virtual ~FastS_fdispatch_RPC() {}
+
+    // Register RPC Methods
+
+    virtual void RegisterMethods(FRT_ReflectionBuilder *rb);
+
+    // methods registered by superclass
+
+    virtual void RPC_GetNodeType(FRT_RPCRequest *req);
+
+    // methods registered by us
+
+    void RPC_EnableEngine(FRT_RPCRequest *req);
+    void RPC_DisableEngine(FRT_RPCRequest *req);
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp
new file mode 100644
index 00000000000..0b99510726a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp
@@ -0,0 +1,125 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".fdispatch.searchadapter");
+#include "searchadapter.h"
+#include <vespa/searchlib/common/packets.h>
+#include <vespa/searchlib/fef/queryproperties.h>
+#include <vespa/searchlib/parsequery/simplequerystack.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+#include <vespa/searchcore/fdispatch/search/dataset_base.h>
+#include <vespa/searchcore/fdispatch/search/nodemanager.h>
+
+namespace fdispatch {
+
+void
+SearchAdapter::handleRequest()
+{
+    _dsc = _appCtx->GetDataSetCollection();
+    FastS_assert(_dsc != NULL);
+
+    uint32_t dataset = _dsc->SuggestDataSet();
+
+    _search = _dsc->CreateSearch(dataset, _appCtx->GetTimeKeeper());
+    FastS_assert(_search != NULL);
+
+    _searchInfo = _search->GetSearchInfo();
+    _queryResult = _search->GetQueryResult();
+    _search->setSearchRequest(_request.get());
+    _search->Search(_request->offset, _request->maxhits, /* minhits */ 0);
+    _search->ProcessQueryDone();
+}
+
+SearchAdapter::SearchReply::UP
+SearchAdapter::createReply()
+{
+    SearchReply::UP reply(new SearchReply());
+    SearchReply &r = *reply;
+    r.useWideHits = true; // mld
+    if (_search->GetErrorCode() != search::engine::ECODE_NO_ERROR) {
+        if (allowError()) {
+            r.errorCode = _search->GetErrorCode();
+            r.errorMessage = _search->GetErrorMessage();
+        }
+        return reply;
+    }
+
+    if ((_request->queryFlags &
+         search::fs4transport::QFLAG_REPORT_QUEUELEN) != 0) {
+        // FIXME
+        r.useQueueLen = true;
+        r.queueLen    = 1;
+    }
+
+    uint32_t hitcnt = _queryResult->_hitCount;
+    r.offset = _searchInfo->_searchOffset;
+    r.totalHitCount = _queryResult->_totalHitCount;
+    r.maxRank = _queryResult->_maxRank;
+    r.setDistributionKey(_appCtx->GetNodeManager()->GetMldDocstamp());
+
+    if (_queryResult->_sortIndex != NULL && hitcnt > 0) {
+        r.sortIndex.assign(_queryResult->_sortIndex, _queryResult->_sortIndex + hitcnt + 1);
+        r.sortData.assign(_queryResult->_sortData, _queryResult->_sortData + _queryResult->_sortIndex[hitcnt]);
+    }
+
+    if (_queryResult->_groupResultLen > 0) {
+        r.groupResult.assign(_queryResult->_groupResult,
+                             _queryResult->_groupResult + _queryResult->_groupResultLen);
+    }
+
+    if ((_request->queryFlags &
+         search::fs4transport::QFLAG_REPORT_COVERAGE) != 0)
+    {
+        r.useCoverage = true;
+        r.coverage = SearchReply::Coverage(_searchInfo->_activeDocs, _searchInfo->_coverageDocs);
+    }
+
+    FastS_hitresult *hitbuf = _queryResult->_hitbuf;
+    r.hits.resize(hitcnt);
+
+    for (uint32_t cur = 0; cur < hitcnt; cur++) {
+        r.hits[cur].gid      = hitbuf[cur]._gid;
+        r.hits[cur].metric   = hitbuf[cur]._metric;
+        r.hits[cur].path     = hitbuf[cur]._partition;
+        r.hits[cur].setDistributionKey(hitbuf[cur].getDistributionKey());
+    }
+    r.request = _request.release();
+    return reply;
+}
+
+void
+SearchAdapter::cleanup()
+{
+    if (_search != NULL) {
+        _search->Free();
+    }
+    if (_dsc != NULL) {
+        _dsc->subRef();
+    }
+}
+
+void
+SearchAdapter::Run(FastOS_ThreadInterface *, void *)
+{
+    handleRequest();
+    SearchReply::UP reply = createReply();
+    cleanup();
+    _client.searchDone(std::move(reply));
+    delete this;
+}
+
+SearchAdapter::SearchAdapter(FastS_AppContext *appCtx,
+                             SearchRequest::Source request,
+                             SearchClient &client)
+    : _appCtx(appCtx),
+      _request(std::move(request)),
+      _client(client),
+      _dsc(0),
+      _search(0),
+      _searchInfo(0),
+      _queryResult(0)
+{
+}
+
+} // namespace fdispatch
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h
new file mode 100644
index 00000000000..e0528508c24
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/engine/searchapi.h>
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+
+namespace fdispatch {
+
+/**
+ * Implementation of the common search api for the fdispatch server
+ * application.
+ **/
+class SearchAdapter : public FastOS_Runnable
+{
+public:
+    typedef search::engine::SearchRequest SearchRequest;
+    typedef search::engine::SearchReply   SearchReply;
+    typedef search::engine::SearchClient  SearchClient;
+
+private:
+    FastS_AppContext          *_appCtx;
+    SearchRequest::Source      _request;
+    SearchClient              &_client;
+
+    // internal search related state
+    FastS_DataSetCollection   *_dsc;
+    FastS_ISearch             *_search;
+    FastS_SearchInfo          *_searchInfo;
+    FastS_QueryResult	      *_queryResult;
+
+    void handleRequest();
+    SearchReply::UP createReply();
+    void writeLog();
+    void cleanup();
+
+    virtual void Run(FastOS_ThreadInterface *, void *);
+
+public:
+    SearchAdapter(FastS_AppContext *appCtx,
+                  SearchRequest::Source request,
+                  SearchClient &client);
+
+    bool allowError() const {
+        return ((_request->queryFlags &
+                 search::fs4transport::QFLAG_ALLOW_ERRORPACKET) != 0);
+    }
+};
+
+} // namespace fdispatch
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/.gitignore b/searchcore/src/vespa/searchcore/fdispatch/search/.gitignore
new file mode 100644
index 00000000000..ca1a057edea
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/.gitignore
@@ -0,0 +1,3 @@
+*.lib
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/CMakeLists.txt b/searchcore/src/vespa/searchcore/fdispatch/search/CMakeLists.txt
new file mode 100644
index 00000000000..ded3bf47fe8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/CMakeLists.txt
@@ -0,0 +1,21 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_fdispatch_search STATIC
+    SOURCES
+    configdesc.cpp
+    dataset_base.cpp
+    datasetcollection.cpp
+    engine_base.cpp
+    fnet_dataset.cpp
+    fnet_engine.cpp
+    fnet_search.cpp
+    mergehits.cpp
+    nodemanager.cpp
+    partitioned_array.cpp
+    plain_dataset.cpp
+    query.cpp
+    querycacheutil.cpp
+    rowstate.cpp
+    search_path.cpp
+    DEPENDS
+    searchcore_fconfig
+)
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/child_info.h b/searchcore/src/vespa/searchcore/fdispatch/search/child_info.h
new file mode 100644
index 00000000000..ce54bc6b38d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/child_info.h
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+
+#include "poss_count.h"
+
+struct ChildInfo {
+    uint32_t maxNodes;
+    uint32_t activeNodes;
+    uint32_t maxParts;
+    uint32_t activeParts;
+    PossCount activeDocs;
+
+    ChildInfo()
+        : maxNodes(0),
+          activeNodes(0),
+          maxParts(0),
+          activeParts(0),
+          activeDocs()
+    {}
+};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.cpp
new file mode 100644
index 00000000000..4a71bc1f3fa
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.cpp
@@ -0,0 +1,345 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+
+LOG_SETUP(".search.configdesc");
+
+//----------------------------------------------------------------------
+
+double FastS_DataSetDesc::_defaultSlowQueryLimitFactor  = 0.0;
+double FastS_DataSetDesc::_defaultSlowQueryLimitBias    = 100.0;
+double FastS_DataSetDesc::_defaultSlowDocsumLimitFactor = 0.0;
+double FastS_DataSetDesc::_defaultSlowDocsumLimitBias   = 100.0;
+
+
+FastS_DataSetDesc::FastS_DataSetDesc(uint32_t datasetid)
+    : _id(datasetid),
+      _queryDistributionMode(QueryDistributionMode::AUTOMATIC, 100.0, 10000),
+      _unitRefCost(0),
+      _partBits(6),
+      _rowBits(0),
+      _numParts(0),
+      _firstPart(0),
+      _minChildParts(0),
+      _maxNodesDownPerFixedRow(0),
+      _useRoundRobinForFixedRow(true),
+      _maxHitsPerNode(static_cast<uint32_t>(-1)),
+      _estimateParts(1),
+      _estPartCutoff(1),
+      _estimatePartsSet(false),
+      _estPartCutoffSet(false),
+      _minOurActive(500),
+      _maxOurActive(500),
+      _cutoffOurActive(1000),
+      _minEstActive(500),
+      _maxEstActive(1000),
+      _cutoffEstActive(1000),
+      _queueDrainRate(400.0),
+      _queueMaxDrain(40.0),
+      _slowQueryLimitFactor(_defaultSlowQueryLimitFactor),
+      _slowQueryLimitBias(_defaultSlowQueryLimitBias),
+      _slowDocsumLimitFactor(_defaultSlowDocsumLimitFactor),
+      _slowDocsumLimitBias(_defaultSlowDocsumLimitBias),
+      _monitorInterval(1.0),
+      _higherCoverageMaxSearchWait(1.0),
+      _higherCoverageMinSearchWait(0.0),
+      _higherCoverageBaseSearchWait(0.1),
+      _minimalSearchCoverage(100.0),
+      _higherCoverageMaxDocSumWait(0.3),
+      _higherCoverageMinDocSumWait(0.1),
+      _higherCoverageBaseDocSumWait(0.1),
+      _minimalDocSumCoverage(100.0),
+      _engineCnt(0),
+      _enginesHead(NULL),
+      _enginesTail(NULL),
+      _mpp(1)
+{
+}
+
+
+FastS_DataSetDesc::~FastS_DataSetDesc()
+{
+    while (_enginesHead != NULL) {
+        FastS_EngineDesc *engine = _enginesHead;
+        _enginesHead = engine->GetNext();
+        delete engine;
+    }
+}
+
+
+FastS_EngineDesc *
+FastS_DataSetDesc::AddEngine(const char *name)
+{
+    FastS_EngineDesc *engine = new FastS_EngineDesc(name);
+    FastS_assert(engine != NULL);
+
+    engine->SetNext(NULL);
+    if (_enginesHead == NULL)
+        _enginesHead = engine;
+    else
+        _enginesTail->SetNext(engine);
+    _enginesTail = engine;
+    _engineCnt++;
+    return engine;
+}
+
+
+void
+FastS_DataSetDesc::FinalizeConfig()
+{
+    /* assume 1 partition if number of partitions was not specified */
+    if (GetNumParts() == 0) {
+        LOG(warning,
+            "Setting partitions to 1 in dataset %u",
+            (unsigned int) GetID());
+        SetNumParts(1);
+    }
+
+    if (!_estPartCutoffSet ||
+        _estPartCutoff > _numParts ||
+        _estPartCutoff == 0)
+        _estPartCutoff = _numParts;
+}
+
+//----------------------------------------------------------------------
+
+bool
+FastS_DataSetCollDesc::CheckIntegrity()
+{
+    bool rc = true;
+
+    for (uint32_t i = 0; i < _datasets_size; i++) {
+        FastS_DataSetDesc *d = _datasets[i];
+        if (d != NULL) {
+            if (d->GetEngineCnt() == 0) {
+                LOG(warning, "plain dataset %d has no engines", d->GetID());
+            }
+
+            if (d->GetNumParts() == 0) {
+                LOG(warning, "plain dataset %d has no partitions", d->GetID());
+            }
+
+            // check engine configuration
+            {
+                uint32_t partBits = d->GetPartBits();
+                uint32_t rowBits  = d->GetRowBits();
+                uint32_t minPart  = d->GetFirstPart();
+                uint32_t maxPart  = minPart + (1 << partBits) - 2;
+                uint32_t maxRow   = (rowBits > 0)? (1 << rowBits) - 1 : 0;
+                uint32_t enginePartCnt = 0;
+                FastS_assert(partBits > 0);
+                bool *partidUsed = new bool[maxPart];
+                for (uint32_t j = 0; j < maxPart; j++)
+                    partidUsed[j] = false;
+
+                for (FastS_EngineDesc *engine = d->GetEngineList();
+                     engine != NULL; engine = engine->GetNext()) {
+
+                    bool     bad    = false;
+                    uint32_t partid = engine->GetConfPartID();
+                    uint32_t rowid  = engine->GetConfRowID();
+
+                    if (partid != FastS_NoID32() &&
+                        (partid < minPart || partid > maxPart))
+                    {
+                        LOG(error, "engine '%s' in dataset %d has partid %d, legal range is [%d,%d] (partbits = %d)",
+                            engine->GetName(), d->GetID(), partid,
+                            minPart, maxPart, partBits);
+                        bad = true;
+                    }
+
+                    if (rowid && rowid != FastS_NoID32()) {
+                        if (rowBits == 0) {
+                            LOG(warning, "rowid (%d) on engine '%s' in dataset %d "
+                                "will be ignored because rowbits is 0",
+                                rowid, engine->GetName(), d->GetID());
+                        } else if (rowid > maxRow) {
+                            LOG(error, "engine '%s' in dataset %d has rowid %d, legal range is [%d,%d] (rowbits = %d)",
+                                engine->GetName(), d->GetID(), rowid,
+                                0, maxRow, rowBits);
+                            bad = true;
+                        }
+                    }
+                    if (bad) {
+                        LOG(error, "marking engine '%s' in dataset %d as BAD due to illegal configuration",
+                            engine->GetName(), d->GetID());
+                        engine->MarkBad();
+                    }
+
+                    if (partid != FastS_NoID32() &&
+                        (partid >= minPart || partid <= maxPart)) {
+                        if (!partidUsed[partid]) {
+                            enginePartCnt++;
+                            partidUsed[partid] = true;
+                        }
+                    } else {
+                        enginePartCnt++;
+                    }
+                }
+                delete [] partidUsed;
+                if (d->GetNumParts() < enginePartCnt) {
+                    LOG(warning,
+                        "plain dataset %d has "
+                        "%d engines with different partids, "
+                        "but only %d partitions",
+                        d->GetID(),
+                        enginePartCnt,
+                        d->GetNumParts());
+                }
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+
+FastS_DataSetCollDesc::FastS_DataSetCollDesc()
+    : _datasets(NULL),
+      _datasets_size(0),
+      _frozen(false),
+      _error(false)
+{
+}
+
+
+FastS_DataSetCollDesc::~FastS_DataSetCollDesc()
+{
+    if (_datasets != NULL) {
+        for (uint32_t i = 0; i < _datasets_size; i++) {
+            if (_datasets[i] != NULL) {
+                delete _datasets[i];
+            }
+        }
+        delete [] _datasets;
+    }
+}
+
+
+FastS_DataSetDesc *
+FastS_DataSetCollDesc::LookupCreateDataSet(uint32_t datasetid)
+{
+    FastS_assert(!_frozen);
+
+    if (datasetid >= _datasets_size) {
+        uint32_t newSize = datasetid + 1;
+
+        FastS_DataSetDesc **newArray = new FastS_DataSetDesc*[newSize];
+        FastS_assert(newArray != NULL);
+
+        uint32_t i;
+        for (i = 0; i < _datasets_size; i++)
+            newArray[i] = _datasets[i];
+
+        for (; i < newSize; i++)
+            newArray[i] = NULL;
+
+        delete [] _datasets;
+        _datasets = newArray;
+        _datasets_size = newSize;
+    }
+
+    if (_datasets[datasetid] == NULL) {
+        _datasets[datasetid] = new FastS_DataSetDesc(datasetid);
+    }
+
+    return _datasets[datasetid];
+}
+
+
+bool
+FastS_DataSetCollDesc::Freeze()
+{
+    if (!_frozen) {
+        _frozen = true;
+
+        for (uint32_t i = 0; i < _datasets_size; i++)
+            if (_datasets[i] != NULL)
+                _datasets[i]->FinalizeConfig();
+
+        _error = !CheckIntegrity();
+    }
+    return !_error;
+}
+
+//----------------------------------------------------------------------
+bool
+FastS_DataSetCollDesc::ReadConfig(const PartitionsConfig& partmap)
+{
+    FastS_assert(!_frozen);
+
+    int datasetcnt = partmap.dataset.size();
+
+    if (datasetcnt < 1) {
+        LOG(error, "no datasets in partitions config");
+        return false;
+    }
+    for (int i=0; i < datasetcnt; i++) {
+        typedef PartitionsConfig::Dataset Dsconfig;
+        const Dsconfig dsconfig = partmap.dataset[i];
+
+        FastS_DataSetDesc *dataset = LookupCreateDataSet(dsconfig.id);
+
+        dataset->SetUnitRefCost(dsconfig.refcost);
+        dataset->SetPartBits(dsconfig.partbits);
+        dataset->SetRowBits(dsconfig.rowbits);
+        dataset->SetNumParts(dsconfig.numparts);
+        dataset->SetMinChildParts(dsconfig.minpartitions);
+        dataset->setMaxNodesDownPerFixedRow(dsconfig.maxnodesdownperfixedrow);
+        dataset->useRoundRobinForFixedRow(dsconfig.useroundrobinforfixedrow);
+        dataset->SetMaxHitsPerNode(dsconfig.maxhitspernode);
+        dataset->SetFirstPart(dsconfig.firstpart);
+        dataset->SetMinOurActive(dsconfig.minactive);
+        dataset->SetMaxOurActive(dsconfig.maxactive);
+        dataset->SetCutoffOurActive(dsconfig.cutoffactive);
+        dataset->SetMinEstActive(dsconfig.minestactive);
+        dataset->SetMaxEstActive(dsconfig.maxestactive);
+        dataset->SetCutoffEstActive(dsconfig.cutoffestactive);
+        dataset->SetQueueDrainRate(dsconfig.queuedrainrate);
+        dataset->SetQueueMaxDrain(dsconfig.queuedrainmax);
+        dataset->SetSlowQueryLimitFactor(dsconfig.slowquerylimitfactor);
+        dataset->SetSlowQueryLimitBias(dsconfig.slowquerylimitbias);
+        dataset->SetSlowDocsumLimitFactor(dsconfig.slowdocsumlimitfactor);
+        dataset->SetSlowDocsumLimitBias(dsconfig.slowdocsumlimitbias);
+        dataset->setMonitorInterval(dsconfig.monitorinterval);
+        dataset->setHigherCoverageMaxSearchWait(dsconfig.higherCoverageMaxsearchwait);
+        dataset->setHigherCoverageMinSearchWait(dsconfig.higherCoverageMinsearchwait);
+        dataset->setHigherCoverageBaseSearchWait(dsconfig.higherCoverageBasesearchwait);
+        dataset->setMinimalSearchCoverage(dsconfig.minimalSearchcoverage);
+        dataset->setHigherCoverageMaxDocSumWait(dsconfig.higherCoverageMaxdocsumwait);
+        dataset->setHigherCoverageMinDocSumWait(dsconfig.higherCoverageMindocsumwait);
+        dataset->setHigherCoverageBaseDocSumWait(dsconfig.higherCoverageBasedocsumwait);
+        dataset->setMinimalDocSumCoverage(dsconfig.minimalDocsumcoverage);
+        FastS_DataSetDesc::QueryDistributionMode distMode(dsconfig.querydistribution,
+                                                          dsconfig.minGroupCoverage,
+                                                          dsconfig.latencyDecayRate);
+        distMode.setMinActivedocsCoverage(dsconfig.minActivedocsCoverage);
+        dataset->SetQueryDistributionMode(distMode);
+        dataset->setMPP(dsconfig.mpp);
+        if (dsconfig.estparts > 0)
+            dataset->SetEstimateParts(dsconfig.estparts);
+        if (dsconfig.estpartcutoff > 0)
+            dataset->SetEstPartCutoff(dsconfig.estpartcutoff);
+
+        int enginecnt = dsconfig.engine.size();
+
+        for (int j=0; j < enginecnt; j++) {
+            const Dsconfig::Engine& engconfig = dsconfig.engine[j];
+
+            FastS_EngineDesc *engine = dataset->AddEngine(engconfig.nameAndPort.c_str());
+
+            engine->SetUnitRefCost(engconfig.refcost);
+            engine->SetConfRowID(engconfig.rowid);
+            engine->SetConfPartID(engconfig.partid);
+            if (engconfig.overridepartids)
+                engine->SetConfPartIDOverrides();
+        }
+    }
+    return true;
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h b/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h
new file mode 100644
index 00000000000..863047e4695
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h
@@ -0,0 +1,380 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/searchlib/common/fslimits.h>
+#include <vespa/searchcore/fdispatch/common/stdincl.h>
+#include <vespa/searchcore/config/config-partitions.h>
+
+using vespa::config::search::core::PartitionsConfig;
+
+//-----------------------------------------------------------------------
+
+class FastS_EngineDesc
+{
+private:
+    FastS_EngineDesc(const FastS_EngineDesc &);
+    FastS_EngineDesc& operator=(const FastS_EngineDesc &);
+
+    FastS_EngineDesc  *_next;
+    std::string        _name;
+    uint32_t           _confPartID;
+    uint32_t           _confRowID;
+    uint32_t           _unitrefcost;
+    bool               _isBad;
+    bool               _confPartIDOverrides;
+
+public:
+    explicit FastS_EngineDesc(const char *name)
+        : _next(NULL),
+          _name(name),
+          _confPartID(FastS_NoID32()),
+          _confRowID(FastS_NoID32()),
+          _unitrefcost(1),
+          _isBad(false),
+          _confPartIDOverrides(false)
+    {
+    }
+
+    void SetNext(FastS_EngineDesc *next) { _next = next; }
+    void SetConfPartID(int32_t value) { assert(value >= 0); _confPartID = value; }
+    void SetConfPartIDOverrides(void) { _confPartIDOverrides = true; }
+    void SetConfRowID(int32_t value) { assert(value >= 0); _confRowID = value; }
+    void SetUnitRefCost(uint32_t value) { _unitrefcost = value; }
+    void MarkBad(void) { _isBad = true; }
+    FastS_EngineDesc * GetNext(void) const { return _next; }
+    const char * GetName(void) const { return _name.c_str(); }
+    uint32_t GetConfPartID(void) const { return _confPartID; }
+    bool GetConfPartIDOverrides(void) const { return _confPartIDOverrides; }
+    uint32_t GetConfRowID(void) const { return _confRowID; }
+    uint32_t GetUnitRefCost(void) const { return _unitrefcost; }
+    bool IsBad(void) const { return _isBad; }
+};
+
+//-----------------------------------------------------------------------
+
+class FastS_DataSetDesc
+{
+private:
+    FastS_DataSetDesc(const FastS_DataSetDesc &);
+    FastS_DataSetDesc& operator=(const FastS_DataSetDesc &);
+
+    static double _defaultSlowQueryLimitFactor;
+    static double _defaultSlowQueryLimitBias;
+    static double _defaultSlowDocsumLimitFactor;
+    static double _defaultSlowDocsumLimitBias;
+
+public:
+
+    class QueryDistributionMode {
+    public:
+        enum Mode { 
+            RANDOM        = PartitionsConfig::Dataset::RANDOM,
+            AUTOMATIC     = PartitionsConfig::Dataset::AUTOMATIC,
+            FIXEDROW      = PartitionsConfig::Dataset::FIXEDROW
+        };
+
+        struct InvalidModeException {
+            Mode _mode;
+            InvalidModeException(Mode mode) : _mode(mode)
+            {}
+        };
+
+        QueryDistributionMode(Mode mode, double minGroupCoverage, double latencyDecayRate) :
+            _mode(mode),
+            _minGroupCoverage(minGroupCoverage),
+            _latencyDecayRate(latencyDecayRate),
+            _minActivedocsCoverage(0.0)
+        { }
+
+        QueryDistributionMode(PartitionsConfig::Dataset::Querydistribution mode, double minGroupCoverage, double latencyDecayRate) :
+            QueryDistributionMode(static_cast<Mode>(mode), minGroupCoverage, latencyDecayRate)
+        {
+        }
+
+        bool operator==(const QueryDistributionMode & rhs) const {
+            return _mode == rhs._mode;
+        }
+        bool operator == (Mode rhs) const {
+            return _mode == rhs;
+        }
+        double getMinGroupCoverage() const { return _minGroupCoverage; }
+        double getLatencyDecayRate() const { return _latencyDecayRate; }
+        double getMinActivedocsCoverage() const { return _minActivedocsCoverage; }
+
+        void setMinActivedocsCoverage(double val) { _minActivedocsCoverage = val; }
+    private:
+        Mode    _mode;
+        double  _minGroupCoverage;
+        double  _latencyDecayRate;
+        double  _minActivedocsCoverage;
+    };
+
+    static void SetDefaultSlowQueryLimitFactor(double value)
+    { _defaultSlowQueryLimitFactor = value; }
+
+    static void SetDefaultSlowQueryLimitBias(double value)
+    { _defaultSlowQueryLimitBias = value; }
+
+    static void SetDefaultSlowDocsumLimitFactor(double value)
+    { _defaultSlowDocsumLimitFactor = value; }
+
+    static void SetDefaultSlowDocsumLimitBias(double value)
+    { _defaultSlowDocsumLimitBias = value; }
+
+private:
+    uint32_t _id;
+    QueryDistributionMode _queryDistributionMode;
+
+    uint32_t _unitRefCost;           // Cost to reference us
+    uint32_t _partBits;              // # bits used to encode part id
+    uint32_t _rowBits;               // # bits used to encode row id
+    uint32_t _numParts;	           // Number of partitions
+    uint32_t _firstPart;             // First partition
+    uint32_t _minChildParts;	   // Minimum partitions live to avoid tempfail
+    uint32_t _maxNodesDownPerFixedRow; // max number of nodes down in a row before considering another row.
+    bool     _useRoundRobinForFixedRow; // Either plain roundrobin or random.
+    uint32_t _maxHitsPerNode;        // max hits requested from single node
+    uint32_t _estimateParts;         // number of partitions used for estimate
+    uint32_t _estPartCutoff;         // First partition not used for estimate
+    bool     _estimatePartsSet;      // has _estimateParts been set ?
+    bool     _estPartCutoffSet;      // has _estimatePartsCutoff been set ?
+    uint32_t _minOurActive;          // below ==> activate, skip estimates
+    uint32_t _maxOurActive;          // above ==> queue
+    uint32_t _cutoffOurActive;       // Above ==> cutoff
+    uint32_t _minEstActive;          // est below ==> activate
+    uint32_t _maxEstActive;          // est below ==> queue, est above cutoff > 0%
+    uint32_t _cutoffEstActive;       // est above ==> cutoff 100%
+    double   _queueDrainRate;        // max queue drain per second
+    double   _queueMaxDrain;         // max queue drain at once
+    double   _slowQueryLimitFactor;
+    double   _slowQueryLimitBias;
+    double   _slowDocsumLimitFactor;
+    double   _slowDocsumLimitBias;
+    double   _monitorInterval;
+    double   _higherCoverageMaxSearchWait;
+    double   _higherCoverageMinSearchWait;
+    double   _higherCoverageBaseSearchWait;
+    double   _minimalSearchCoverage;
+    double   _higherCoverageMaxDocSumWait;
+    double   _higherCoverageMinDocSumWait;
+    double   _higherCoverageBaseDocSumWait;
+    double   _minimalDocSumCoverage;
+
+    uint32_t          _engineCnt;    // number of search engines in dataset
+    FastS_EngineDesc *_enginesHead;  // first engine in dataset
+    FastS_EngineDesc *_enginesTail;  // last engine in dataset
+
+    uint32_t _mpp;	// Minimum number of engines per partition
+public:
+    explicit FastS_DataSetDesc(uint32_t datasetid);
+    ~FastS_DataSetDesc(void);
+
+    uint32_t GetID(void) const { return _id; }
+    void SetUnitRefCost(uint32_t value) { _unitRefCost = value; }
+
+    void SetPartBits(uint32_t value) {
+       if (value >= MIN_PARTBITS && value <= MAX_PARTBITS)
+           _partBits = value;
+    }
+
+    void SetRowBits(uint32_t value) {
+        if (value <= MAX_ROWBITS)
+            _rowBits = value;
+    }
+
+    void SetNumParts(uint32_t value) { _numParts = value; }
+    void SetFirstPart(uint32_t value) { _firstPart = value; }
+    void SetMinChildParts(uint32_t value) { _minChildParts         = value; }
+    void setMaxNodesDownPerFixedRow(uint32_t value) { _maxNodesDownPerFixedRow = value; }
+    void useRoundRobinForFixedRow(bool value) { _useRoundRobinForFixedRow = value; }
+    void SetMaxHitsPerNode(uint32_t value) { _maxHitsPerNode        = value; }
+    void SetEstimateParts(uint32_t value) {
+        _estimateParts         = value;
+        _estimatePartsSet      = true;
+    }
+
+    void SetEstPartCutoff(uint32_t value) {
+        _estPartCutoff         = value;
+        _estPartCutoffSet      = true;
+    }
+
+    void SetMinOurActive(uint32_t value) { _minOurActive = value; }
+    void SetMaxOurActive(uint32_t value) { _maxOurActive = value; }
+    void SetCutoffOurActive(uint32_t value) { _cutoffOurActive = value; }
+    void SetMinEstActive(uint32_t value) { _minEstActive = value; }
+    void SetMaxEstActive(uint32_t value) { _maxEstActive = value; }
+    void SetCutoffEstActive(uint32_t value) { _cutoffEstActive = value; }
+    void SetQueueDrainRate(double value) { _queueDrainRate = value; }
+    void SetQueueMaxDrain(double value) { _queueMaxDrain = value; }
+    void SetSlowQueryLimitFactor(double value) { _slowQueryLimitFactor = value; }
+    void SetSlowQueryLimitBias(double value) { _slowQueryLimitBias = value; }
+    void SetSlowDocsumLimitFactor(double value) { _slowDocsumLimitFactor = value; }
+    void SetSlowDocsumLimitBias(double value) { _slowDocsumLimitBias = value; }
+
+    void SetQueryDistributionMode(QueryDistributionMode queryDistributionMode) {
+        _queryDistributionMode = queryDistributionMode;
+    }
+
+    QueryDistributionMode GetQueryDistributionMode() { return _queryDistributionMode; }
+
+    FastS_EngineDesc * AddEngine(const char *name);
+    uint32_t GetUnitRefCost(void) const { return _unitRefCost; }
+    uint32_t GetPartBits(void) const { return _partBits; }
+
+    uint32_t GetRowBits(void) const { return _rowBits; }
+    uint32_t GetNumParts(void) const { return _numParts; }
+    uint32_t GetFirstPart(void) const { return _firstPart; }
+    uint32_t GetMinChildParts(void) const { return _minChildParts; }
+    uint32_t getMaxNodesDownPerFixedRow(void) const { return _maxNodesDownPerFixedRow; }
+    bool useRoundRobinForFixedRow(void) const { return _useRoundRobinForFixedRow; }
+    uint32_t GetMaxHitsPerNode(void) const { return _maxHitsPerNode; }
+    uint32_t GetEstimateParts(void) const { return _estimateParts; }
+    uint32_t GetEstPartCutoff(void) const { return _estPartCutoff; }
+    bool IsEstimatePartsSet(void) const { return _estimatePartsSet; }
+    bool IsEstPartCutoffSet(void) const { return _estPartCutoffSet; }
+    uint32_t GetMinOurActive(void) const { return _minOurActive; }
+    uint32_t GetMaxOurActive(void) const { return _maxOurActive; }
+    uint32_t GetCutoffOurActive(void) const { return _cutoffOurActive; }
+    uint32_t GetMinEstActive(void) const { return _minEstActive; }
+    uint32_t GetMaxEstActive(void) const { return _maxEstActive; }
+    uint32_t GetCutoffEstActive(void) const { return _cutoffEstActive; }
+    double GetQueueDrainRate(void) const { return _queueDrainRate; }
+    double GetQueueMaxDrain(void) const { return _queueMaxDrain; }
+    double GetSlowQueryLimitFactor(void) const { return _slowQueryLimitFactor; }
+    double GetSlowQueryLimitBias(void) const { return _slowQueryLimitBias; }
+    double GetSlowDocsumLimitFactor(void) const { return _slowDocsumLimitFactor; }
+    double GetSlowDocsumLimitBias(void) const { return _slowDocsumLimitBias; }
+    uint32_t GetEngineCnt(void) const { return _engineCnt; }
+    FastS_EngineDesc * GetEngineList(void) const { return _enginesHead; }
+    void setMPP(uint32_t mpp) { _mpp = mpp; }
+    uint32_t getMPP(void) const { return _mpp; }
+
+    void
+    setMonitorInterval(double monitorInterval) { _monitorInterval = monitorInterval; }
+    double getMonitorInterval(void) const { return _monitorInterval; }
+
+    void
+    setHigherCoverageMaxSearchWait(double higherCoverageMaxSearchWait) {
+        _higherCoverageMaxSearchWait = higherCoverageMaxSearchWait;
+    }
+
+    double
+    getHigherCoverageMaxSearchWait(void) const {
+        return _higherCoverageMaxSearchWait;
+    }
+
+    void
+    setHigherCoverageMinSearchWait(double higherCoverageMinSearchWait) {
+        _higherCoverageMinSearchWait = higherCoverageMinSearchWait;
+    }
+
+    double
+    getHigherCoverageMinSearchWait(void) const {
+        return _higherCoverageMinSearchWait;
+    }
+
+    void
+    setHigherCoverageBaseSearchWait(double higherCoverageBaseSearchWait) {
+        _higherCoverageBaseSearchWait = higherCoverageBaseSearchWait;
+    }
+
+    double
+    getHigherCoverageBaseSearchWait(void) const {
+        return _higherCoverageBaseSearchWait;
+    }
+
+    void
+    setMinimalSearchCoverage(double minimalSearchCoverage) {
+        _minimalSearchCoverage = minimalSearchCoverage;
+    }
+
+    double
+    getMinimalSearchCoverage(void) const {
+        return _minimalSearchCoverage;
+    }
+
+    void
+    setHigherCoverageMaxDocSumWait(double higherCoverageMaxDocSumWait) {
+        _higherCoverageMaxDocSumWait = higherCoverageMaxDocSumWait;
+    }
+
+    double
+    getHigherCoverageMaxDocSumWait(void) const {
+        return _higherCoverageMaxDocSumWait;
+    }
+
+    void
+    setHigherCoverageMinDocSumWait(double higherCoverageMinDocSumWait) {
+        _higherCoverageMinDocSumWait = higherCoverageMinDocSumWait;
+    }
+
+    double
+    getHigherCoverageMinDocSumWait(void) const {
+        return _higherCoverageMinDocSumWait;
+    }
+
+    void
+    setHigherCoverageBaseDocSumWait(double higherCoverageBaseDocSumWait) {
+        _higherCoverageBaseDocSumWait = higherCoverageBaseDocSumWait;
+    }
+
+    double
+    getHigherCoverageBaseDocSumWait(void) const {
+        return _higherCoverageBaseDocSumWait;
+    }
+
+    void
+    setMinimalDocSumCoverage(double minimalDocSumCoverage) {
+        _minimalDocSumCoverage = minimalDocSumCoverage;
+    }
+
+    double
+    getMinimalDocSumCoverage(void) const {
+        return _minimalDocSumCoverage;
+    }
+
+    void FinalizeConfig(void);
+};
+
+//-----------------------------------------------------------------------
+
+class FastS_DataSetCollDesc
+{
+private:
+    FastS_DataSetCollDesc(const FastS_DataSetCollDesc &);
+    FastS_DataSetCollDesc& operator=(const FastS_DataSetCollDesc &);
+
+    FastS_DataSetDesc **_datasets;
+    uint32_t            _datasets_size;
+
+    bool                _frozen;
+    bool                _error;
+
+    void HandleDeprecatedFPEstPartsOption(void);
+    bool CheckIntegrity(void);
+
+public:
+    FastS_DataSetCollDesc(void);
+    ~FastS_DataSetCollDesc(void);
+
+    FastS_DataSetDesc *LookupCreateDataSet(uint32_t datasetid);
+
+    bool Freeze(void);
+
+    uint32_t GetMaxNumDataSets(void) const { return _datasets_size; }
+
+    FastS_DataSetDesc *GetDataSet(uint32_t datasetid) const {
+        return (datasetid < _datasets_size)
+            ? _datasets[datasetid]
+            : NULL;
+    }
+
+    bool ReadConfig(const PartitionsConfig& partmap);
+};
+
+//-----------------------------------------------------------------------
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp
new file mode 100644
index 00000000000..4bc535a3f29
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp
@@ -0,0 +1,332 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".search.dataset_base");
+#include <vespa/searchlib/common/fslimits.h>
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+#include <vespa/vespalib/util/atomic.h>
+#include <vespa/searchcore/fdispatch/common/queryperf.h>
+
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+#include <vespa/searchcore/fdispatch/search/engine_base.h>
+#include <vespa/searchcore/fdispatch/search/dataset_base.h>
+#include <vespa/searchcore/fdispatch/search/nodemanager.h>
+
+//--------------------------------------------------------------------------
+
+FastS_DataSetBase::total_t::total_t()
+    : _estimates(0),
+      _nTimedOut(0),
+      _nOverload(0),
+      _normalTimeStat()
+{
+    for (uint32_t i = 0; i < _timestatslots; i++)
+        _timestats[i] = 0;
+}
+
+//--------------------------------------------------------------------------
+
+FastS_DataSetBase::overload_t::overload_t(FastS_DataSetDesc *desc)
+    : _drainRate(desc->GetQueueDrainRate()),
+      _drainMax(desc->GetQueueMaxDrain()),
+      _minouractive(desc->GetMinOurActive()),
+      _maxouractive(desc->GetMaxOurActive()),
+      _cutoffouractive(desc->GetCutoffOurActive()),
+      _minestactive(desc->GetMinEstActive()),
+      _maxestactive(desc->GetMaxEstActive()),
+      _cutoffestactive(desc->GetCutoffEstActive())
+{
+}
+
+//--------------------------------------------------------------------------
+
+FastS_DataSetBase::queryQueue_t::queryQueue_t(FastS_DataSetDesc *desc)
+    : _head(NULL),
+      _tail(NULL),
+      _queueLen(0),
+      _active(0),
+      _drainAllowed(0.0),
+      _drainStamp(0.0),
+      _overload(desc)
+{
+}
+
+
+FastS_DataSetBase::queryQueue_t::~queryQueue_t()
+{
+    FastS_assert(_active == 0);
+}
+
+
+void
+FastS_DataSetBase::queryQueue_t::QueueTail(queryQueued_t *newqueued)
+{
+    FastS_assert(newqueued->_next == NULL &&
+                 _head != newqueued &&
+                 _tail != newqueued);
+    if (_tail != NULL)
+        _tail->_next = newqueued;
+    else
+        _head = newqueued;
+    _tail = newqueued;
+    _queueLen++;
+}
+
+
+void
+FastS_DataSetBase::queryQueue_t::DeQueueHead(void)
+{
+    queryQueued_t *queued = _head;
+    FastS_assert(_queueLen > 0);
+    FastS_assert(queued->_next != NULL || _tail == queued);
+    _head = queued->_next;
+    if (queued->_next == NULL)
+        _tail = NULL;
+    queued->_next = NULL;
+    _queueLen--;
+}
+
+//--------------------------------------------------------------------------
+
+FastS_DataSetBase::FastS_DataSetBase(FastS_AppContext *appCtx,
+                                     FastS_DataSetDesc *desc)
+    : _appCtx(appCtx),
+      _lock(),
+      _createtime(),
+      _queryQueue(desc),
+      _total(),
+      _id(desc->GetID()),
+      _unitrefcost(desc->GetUnitRefCost()),
+      _totalrefcost(0),
+      _mldDocStamp(0u)
+
+{
+    _createtime.SetNow();
+}
+
+
+FastS_DataSetBase::~FastS_DataSetBase()
+{
+    FastS_assert(_totalrefcost == 0);
+}
+
+void
+FastS_DataSetBase::ScheduleCheckTempFail()
+{
+    _appCtx->GetNodeManager()->ScheduleCheckTempFail(_id);
+}
+
+
+void
+FastS_DataSetBase::DeQueueHeadWakeup_HasLock(void)
+{
+    queryQueued_t *queued;
+    queued = _queryQueue.GetFirst();
+    FastS_assert(queued->IsQueued());
+    queued->LockCond();
+    //SetNowFromMonitor();
+    _queryQueue.DeQueueHead();
+    queued->UnmarkQueued();
+    FNET_Task *dequeuedTask = queued->getDequeuedTask();
+    if (dequeuedTask != NULL) {
+        dequeuedTask->ScheduleNow();
+    } else {
+        queued->SignalCond();
+    }
+    queued->UnlockCond();
+}
+
+
+void
+FastS_DataSetBase::SetActiveQuery_HasLock()
+{
+    _queryQueue.SetActiveQuery();
+}
+
+
+void
+FastS_DataSetBase::SetActiveQuery()
+{
+    LockDataset();
+    SetActiveQuery_HasLock();
+    UnlockDataset();
+}
+
+
+void
+FastS_DataSetBase::ClearActiveQuery_HasLock(FastS_TimeKeeper *timeKeeper)
+{
+    FastS_assert(_queryQueue._active > 0);
+    _queryQueue.ClearActiveQuery();
+
+    CheckQueryQueue_HasLock(timeKeeper);
+}
+
+
+void
+FastS_DataSetBase::ClearActiveQuery(FastS_TimeKeeper *timeKeeper)
+{
+    LockDataset();
+    ClearActiveQuery_HasLock(timeKeeper);
+    UnlockDataset();
+}
+
+
+void
+FastS_DataSetBase::CheckQueryQueue_HasLock(FastS_TimeKeeper *timeKeeper)
+{
+    queryQueued_t *queued;
+    unsigned int active;
+    unsigned int estactive;
+    uint32_t dispatchnodes;
+    double delay;
+    double fnow;
+
+    active = _queryQueue.GetActiveQueries();		// active from us
+    estactive = CalculateQueueLens_HasLock(dispatchnodes);// active from us and others
+
+    if (dispatchnodes == 0)
+        dispatchnodes = 1;
+
+    fnow = timeKeeper->GetTime();
+    delay = fnow - _queryQueue._drainStamp;
+    if (delay >= 0.0) {
+        if (delay > 2.0) {
+            delay = 2.0;
+            if (_queryQueue._drainStamp == 0.0)
+                _queryQueue._drainStamp = fnow;
+            else
+                _queryQueue._drainStamp += 2.0;
+        } else
+            _queryQueue._drainStamp = fnow;
+    } else
+        delay = 0.0;
+
+    _queryQueue._drainAllowed += delay * _queryQueue._overload._drainRate;
+    if (_queryQueue._drainAllowed >=
+        _queryQueue._overload._drainMax + dispatchnodes - 1)
+        _queryQueue._drainAllowed =
+            _queryQueue._overload._drainMax + dispatchnodes - 1;
+
+    while (_queryQueue._drainAllowed >= (double) dispatchnodes ||
+           active < _queryQueue._overload._minouractive) {
+        queued = _queryQueue.GetFirst();
+        if (queued == NULL) {
+            return;
+        }
+
+        if (active >= _queryQueue._overload._maxouractive)
+            return;				// hard limit for how much we queue
+
+        if (active >= _queryQueue._overload._minouractive &&
+            estactive >= _queryQueue._overload._minestactive)
+            return;
+
+        // Dequeue query, count it active and wakeup thread handling query
+        SetActiveQuery_HasLock();
+        DeQueueHeadWakeup_HasLock();
+
+        active++;				// one more active from us
+        estactive += dispatchnodes;		// Assume other nodes do likewise
+        if (_queryQueue._drainAllowed >= (double) dispatchnodes)
+            _queryQueue._drainAllowed -= dispatchnodes;	// Rate limitation
+        else
+            _queryQueue._drainAllowed = 0.0;
+    }
+}
+
+
+void
+FastS_DataSetBase::AbortQueryQueue_HasLock()
+{
+    queryQueued_t *queued;
+
+    /*
+     * Don't allow new queries to be queued.
+     * Abort currently queued queries.
+     */
+    _queryQueue._overload._minouractive = 0;
+    _queryQueue._overload._cutoffouractive = 0;
+    for (;;) {
+        queued = _queryQueue.GetFirst();
+        if (queued == NULL)
+            break;
+        // Doesn't lock query, but other thread is waiting on queue
+        queued->MarkAbort();
+        DeQueueHeadWakeup_HasLock();
+    }
+}
+
+
+void
+FastS_DataSetBase::AddCost()
+{
+    _totalrefcost += _unitrefcost;
+}
+
+
+void
+FastS_DataSetBase::SubCost()
+{
+    FastS_assert(_totalrefcost >= _unitrefcost);
+    _totalrefcost -= _unitrefcost;
+}
+
+
+void
+FastS_DataSetBase::UpdateSearchTime(double tnow,
+                                    double elapsed, bool timedout)
+{
+    int slot;
+    LockDataset();
+    slot = (int) (elapsed * 10);
+    if (slot >= _total._timestatslots)
+        slot = _total._timestatslots - 1;
+    else if (slot < 0)
+        slot = 0;
+    _total._timestats[slot]++;
+    _total._normalTimeStat.Update(tnow, elapsed, timedout);
+    UnlockDataset();
+}
+
+
+void
+FastS_DataSetBase::UpdateEstimateCount()
+{
+    ++_total._estimates;
+}
+
+
+void
+FastS_DataSetBase::CountTimeout()
+{
+    ++_total._nTimedOut;
+}
+
+
+void
+FastS_DataSetBase::addPerformance(FastS_QueryPerf &qp)
+{
+    FastS_TimeStatTotals totals;
+    LockDataset();
+    _total._normalTimeStat.AddTotal(&totals);
+    qp.queueLen   += _queryQueue.GetQueueLen();
+    qp.activeCnt  += _queryQueue.GetActiveQueries();
+    qp.queryCnt   += totals._totalCount;
+    qp.queryTime  += totals._totalAccTime;
+    qp.dropCnt    += _total._nOverload;
+    qp.timeoutCnt += _total._nTimedOut;
+    UnlockDataset();
+}
+
+
+ChildInfo
+FastS_DataSetBase::getChildInfo() const
+{
+    return ChildInfo();
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h
new file mode 100644
index 00000000000..6590c76e03b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h
@@ -0,0 +1,236 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include "child_info.h"
+#include <vespa/searchcore/fdispatch/common/timestat.h>
+#include <vespa/searchcore/util/log.h>
+#include <atomic>
+
+class FastS_TimeKeeper;
+
+class FastS_DataSetDesc;
+class FastS_EngineDesc;
+class FastS_DataSetCollection;
+class FastS_ISearch;
+class FastS_QueryResult;
+class FastS_QueryCache;
+class FastS_PlainDataSet;
+class FastS_DataSetInfo;
+class FastS_FNET_DataSet;
+class FastS_AppContext;
+class FastS_QueryPerf;
+class FNET_Task;
+
+//---------------------------------------------------------------------------
+
+class FastS_DataSetBase
+{
+    friend class FastS_DataSetCollection;
+private:
+    FastS_DataSetBase(const FastS_DataSetBase &);
+    FastS_DataSetBase& operator=(const FastS_DataSetBase &);
+
+public:
+
+    //----------------------------------------------------------------
+    // total query stats
+    //----------------------------------------------------------------
+
+    class total_t
+    {
+    public:
+        enum {
+            _timestatslots = 100
+        };
+        std::atomic<uint32_t> _estimates;
+        std::atomic<uint32_t> _nTimedOut;
+        uint32_t _nOverload;
+        uint32_t _timestats[_timestatslots];
+        FastS_TimeStatHistory _normalTimeStat;
+        total_t();
+    };
+
+    //----------------------------------------------------------------
+    // parameters used by query queue
+    //----------------------------------------------------------------
+
+    class overload_t
+    {
+    public:
+        double   _drainRate;       // Queue drain rate
+        double   _drainMax;	       // Max queue drain at once
+        uint32_t _minouractive;    // minimum active requests from us
+        uint32_t _maxouractive;    // maximum active requests from us (queue)
+        uint32_t _cutoffouractive; // cutoff active requests
+        uint32_t _minestactive;    // minimum estimated requests before queueing
+        uint32_t _maxestactive;    // maximum estimated requests (start earlydrop)
+        uint32_t _cutoffestactive; // cutoff estimated requests  (end earlydrop)
+
+        overload_t(FastS_DataSetDesc *desc);
+    };
+
+    //----------------------------------------------------------------
+    // class used to wait for a query queue
+    //----------------------------------------------------------------
+
+    class queryQueue_t;
+    class queryQueued_t
+    {
+        friend class queryQueue_t;
+    private:
+        queryQueued_t(const queryQueued_t &);
+        queryQueued_t& operator=(const queryQueued_t &);
+
+        FastOS_Cond _queueCond;
+        queryQueued_t *_next;
+        bool _isAborted;
+        bool _isQueued;
+        FNET_Task *const _deQueuedTask;
+    public:
+        queryQueued_t(FNET_Task *const deQueuedTask)
+            : _queueCond(),
+              _next(NULL),
+              _isAborted(false),
+              _isQueued(false),
+              _deQueuedTask(deQueuedTask)
+        {
+        }
+
+        ~queryQueued_t(void)
+        {
+            FastS_assert(!_isQueued);
+        }
+        void Wait(void) {
+            _queueCond.Lock();
+            while (_isQueued) {
+                _queueCond.Wait();
+            }
+            _queueCond.Unlock();
+        }
+        bool IsAborted(void) const { return _isAborted; }
+        void MarkAbort(void) { _isAborted = true; }
+        void MarkQueued(void) { _isQueued = true; }
+        void UnmarkQueued(void) { _isQueued = false; }
+        bool IsQueued(void) const { return _isQueued; }
+        void LockCond(void) { _queueCond.Lock(); }
+        void UnlockCond(void) { _queueCond.Unlock(); }
+        void SignalCond(void) { _queueCond.Signal(); }
+
+        FNET_Task *
+        getDequeuedTask(void) const
+        {
+            return _deQueuedTask;
+        }
+    };
+
+    //----------------------------------------------------------------
+    // per dataset query queue
+    //----------------------------------------------------------------
+
+    class queryQueue_t
+    {
+        friend class FastS_DataSetBase;
+
+    private:
+        queryQueue_t(const queryQueue_t &);
+        queryQueue_t& operator=(const queryQueue_t &);
+
+        queryQueued_t	  *_head;
+        queryQueued_t	  *_tail;
+        unsigned int           _queueLen;
+        unsigned int           _active;
+
+    public:
+        double                 _drainAllowed; // number of drainable request
+        double                 _drainStamp;   // stamp of last drain check
+        overload_t             _overload;     // queue parameters
+
+    public:
+        queryQueue_t(FastS_DataSetDesc *desc);
+        ~queryQueue_t();
+        void QueueTail(queryQueued_t *newquery);
+        void DeQueueHead(void);
+        unsigned int GetQueueLen() const        { return _queueLen; }
+        unsigned int GetActiveQueries() const   { return _active; }
+        void SetActiveQuery()                   { _active++; }
+        void ClearActiveQuery()                 { _active--; }
+        queryQueued_t *GetFirst(void) const { return _head; }
+    };
+
+    //----------------------------------------------------------------
+
+protected:
+    FastS_AppContext *_appCtx;
+    FastOS_Mutex  _lock;
+    FastOS_Time  _createtime;
+    queryQueue_t _queryQueue;
+    total_t      _total;
+    uint32_t     _id;
+    uint32_t     _unitrefcost;
+
+    // Total cost as seen by referencing objects
+    std::atomic<uint32_t>  _totalrefcost;
+    uint32_t	 _mldDocStamp;
+
+public:
+    FastS_DataSetBase(FastS_AppContext *appCtx,
+                      FastS_DataSetDesc *desc);
+    virtual ~FastS_DataSetBase();
+
+    // locking stuff
+    //--------------
+    void LockDataset()   { _lock.Lock();   }
+    void UnlockDataset() { _lock.Unlock(); }
+    FastOS_Mutex & getMutex() { return _lock; }
+
+    // query queue related methods
+    //----------------------------
+    void SetActiveQuery_HasLock();
+    void SetActiveQuery();
+    void ClearActiveQuery_HasLock(FastS_TimeKeeper *timeKeeper);
+    void ClearActiveQuery(FastS_TimeKeeper *timeKeeper);
+    void CheckQueryQueue_HasLock(FastS_TimeKeeper *timeKeeper);
+    void AbortQueryQueue_HasLock();
+
+    // common dataset methods
+    //-----------------------
+    uint32_t GetID() { return _id; }
+    double Uptime() { return _createtime.MilliSecsToNow() / 1000.0; }
+    FastS_AppContext *GetAppContext() const { return _appCtx; }
+    void AddCost();
+    void SubCost();
+    void UpdateSearchTime(double tnow, double elapsed, bool timedout);
+    void UpdateEstimateCount();
+    void CountTimeout();
+
+    void ScheduleCheckTempFail();
+    virtual void DeQueueHeadWakeup_HasLock(void);
+    virtual ChildInfo getChildInfo() const;
+    uint32_t GetMldDocStamp(void) const { return _mldDocStamp; }
+    void SetMldDocStamp(uint32_t mldDocStamp) { _mldDocStamp = mldDocStamp; }
+
+    // common dataset API
+    //-------------------
+    virtual uint32_t CalculateQueueLens_HasLock(uint32_t &dispatchnodes) = 0;
+    virtual bool AddEngine(FastS_EngineDesc *desc) = 0;
+    virtual void ConfigDone(FastS_DataSetCollection *) {}
+    virtual void ScheduleCheckBad() {}
+    virtual bool AreEnginesReady() = 0;
+    virtual FastS_ISearch *CreateSearch(FastS_DataSetCollection *dsc,
+                                        FastS_TimeKeeper *timeKeeper,
+                                        bool async) = 0;
+    virtual void Free() = 0;
+    virtual void addPerformance(FastS_QueryPerf &qp);
+
+    // typesafe down-cast
+    //-------------------
+    virtual FastS_PlainDataSet     *GetPlainDataSet()     { return NULL; }
+    virtual FastS_FNET_DataSet     *GetFNETDataSet()      { return NULL; }
+};
+
+//---------------------------------------------------------------------------
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp
new file mode 100644
index 00000000000..baa324bae0b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp
@@ -0,0 +1,279 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/fnet/fnet.h>
+#include <vespa/fastlib/io/bufferedfile.h>
+
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchlib/common/fslimits.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+#include <vespa/searchlib/parsequery/simplequerystack.h>
+#include <vespa/searchcore/fdispatch/search/fnet_dataset.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+
+
+LOG_SETUP(".search.datasetcollection");
+
+FastS_DataSetBase *
+FastS_DataSetCollection::CreateDataSet(FastS_DataSetDesc *desc)
+{
+    FastS_DataSetBase *ret = NULL;
+
+    FNET_Transport *transport = _appCtx->GetFNETTransport();
+    FNET_Scheduler *scheduler = _appCtx->GetFNETScheduler();
+    if (transport != NULL && scheduler != NULL) {
+        ret = new FastS_FNET_DataSet(transport, scheduler, _appCtx, desc);
+    } else {
+        LOG(error, "Non-available dataset transport: FNET");
+    }
+    return ret;
+}
+
+
+bool
+FastS_DataSetCollection::AddDataSet(FastS_DataSetDesc *desc)
+{
+    uint32_t datasetid = desc->GetID();
+
+    if (datasetid >= _datasets_size) {
+        uint32_t newSize = datasetid + 1;
+
+        FastS_DataSetBase **newArray = new FastS_DataSetBase*[newSize];
+        FastS_assert(newArray != NULL);
+
+        uint32_t i;
+        for (i = 0; i < _datasets_size; i++)
+            newArray[i] = _datasets[i];
+
+        for (; i < newSize; i++)
+            newArray[i] = NULL;
+
+        delete [] _datasets;
+        _datasets = newArray;
+        _datasets_size = newSize;
+    }
+    FastS_assert(_datasets[datasetid] == NULL);
+    FastS_DataSetBase *dataset = CreateDataSet(desc);
+    if (dataset == NULL)
+        return false;
+    _datasets[datasetid] = dataset;
+
+    for (FastS_EngineDesc *engineDesc = desc->GetEngineList();
+         engineDesc != NULL; engineDesc = engineDesc->GetNext()) {
+
+        dataset->AddEngine(engineDesc);
+    }
+    dataset->ConfigDone(this);
+    return true;
+}
+
+
+
+FastS_DataSetCollection::FastS_DataSetCollection(FastS_AppContext *appCtx)
+    : _nextOld(NULL),
+      _configDesc(NULL),
+      _appCtx(appCtx),
+      _datasets(NULL),
+      _datasets_size(0),
+      _gencnt(0),
+      _frozen(false),
+      _error(false)
+{
+}
+
+
+FastS_DataSetCollection::~FastS_DataSetCollection()
+{
+    if (_datasets != NULL) {
+        for (uint32_t i = 0; i < _datasets_size; i++) {
+            if (_datasets[i] != NULL) {
+                _datasets[i]->Free();
+                _datasets[i] = NULL;
+            }
+        }
+    }
+
+    delete [] _datasets;
+    delete _configDesc;
+}
+
+
+bool
+FastS_DataSetCollection::Configure(FastS_DataSetCollDesc *cfgDesc,
+                                   uint32_t gencnt)
+{
+    bool rc = false;
+
+    if (_frozen) {
+        delete cfgDesc;
+    } else {
+        FastS_assert(_configDesc == NULL);
+        if (cfgDesc == NULL) {
+            _configDesc = new FastS_DataSetCollDesc();
+        } else {
+            _configDesc = cfgDesc;
+        }
+        _gencnt     = gencnt;
+        _frozen     = true;
+        _error      = !_configDesc->Freeze();
+        rc          = !_error;
+
+        for (uint32_t i = 0; rc && i < _configDesc->GetMaxNumDataSets(); i++) {
+            FastS_DataSetDesc *datasetDesc = _configDesc->GetDataSet(i);
+            if (datasetDesc != NULL) {
+                FastS_assert(datasetDesc->GetID() == i);
+                rc = AddDataSet(datasetDesc);
+            }
+        }
+
+        _error      = !rc;
+    }
+    return rc;
+}
+
+
+uint32_t
+FastS_DataSetCollection::SuggestDataSet()
+{
+    FastS_assert(_frozen);
+
+    FastS_DataSetBase *dataset = NULL;
+
+    for (uint32_t i = 0; i < _datasets_size; i++) {
+        FastS_DataSetBase *tmp = _datasets[i];
+        if (tmp == NULL || tmp->_unitrefcost == 0)
+            continue;
+
+        // NB: cost race condition
+
+        if (dataset == NULL ||
+            dataset->_totalrefcost + dataset->_unitrefcost >
+            tmp->_totalrefcost + tmp->_unitrefcost)
+            dataset = tmp;
+    }
+
+    return (dataset == NULL)
+                     ? FastS_NoID32()
+                     : dataset->GetID();
+}
+
+
+FastS_DataSetBase *
+FastS_DataSetCollection::GetDataSet(uint32_t datasetid)
+{
+    FastS_assert(_frozen);
+
+    FastS_DataSetBase *dataset =
+        (datasetid < _datasets_size) ?
+        _datasets[datasetid] : NULL;
+
+    if (dataset != NULL)
+        dataset->AddCost();
+
+    return dataset;
+}
+
+
+FastS_DataSetBase *
+FastS_DataSetCollection::GetDataSet()
+{
+    FastS_assert(_frozen);
+
+    FastS_DataSetBase *dataset = NULL;
+
+    for (uint32_t i = 0; i < _datasets_size; i++) {
+        FastS_DataSetBase *tmp = _datasets[i];
+        if (tmp == NULL || tmp->_unitrefcost == 0)
+            continue;
+
+        // NB: cost race condition
+
+        if (dataset == NULL ||
+            dataset->_totalrefcost + dataset->_unitrefcost >
+            tmp->_totalrefcost + tmp->_unitrefcost)
+            dataset = tmp;
+    }
+
+    if (dataset != NULL)
+        dataset->AddCost();
+
+    return dataset;
+}
+
+
+bool
+FastS_DataSetCollection::AreEnginesReady()
+{
+    bool ready = true;
+
+    for (uint32_t datasetidx = 0;
+         ready && (datasetidx < GetMaxNumDataSets());
+         datasetidx++)
+    {
+        FastS_DataSetBase *dataset = PeekDataSet(datasetidx);
+        ready = (dataset != NULL && !dataset->AreEnginesReady());
+    }
+    return ready;
+}
+
+
+FastS_ISearch *
+FastS_DataSetCollection::CreateSearch(uint32_t dataSetID,
+                                      FastS_TimeKeeper *timeKeeper)
+{
+    FastS_ISearch *ret = NULL;
+    FastS_DataSetBase *dataset;
+
+    if (dataSetID == FastS_NoID32()) {
+        dataset = GetDataSet();
+        if (dataset != NULL)
+            dataSetID = dataset->GetID();
+    } else {
+        dataset = GetDataSet(dataSetID);
+    }
+    if (dataset == NULL) {
+        ret = new FastS_FailedSearch(dataSetID, false,
+                                     search::engine::ECODE_ILLEGAL_DATASET, NULL);
+    } else {
+        dataset->LockDataset();
+        dataset->SetActiveQuery_HasLock();
+        dataset->UnlockDataset();
+        /* XXX: Semantic change: precounted as active in dataset */
+        ret = dataset->CreateSearch(this, timeKeeper, /* async = */ false);
+    }
+    FastS_assert(ret != NULL);
+    return ret;
+}
+
+
+void
+FastS_DataSetCollection::CheckQueryQueues(FastS_TimeKeeper *timeKeeper)
+{
+    for (uint32_t datasetidx(0); datasetidx < GetMaxNumDataSets(); datasetidx++) {
+        FastS_DataSetBase *dataset = PeekDataSet(datasetidx);
+
+        if (dataset != NULL) {
+            dataset->LockDataset();
+            dataset->CheckQueryQueue_HasLock(timeKeeper);
+            dataset->UnlockDataset();
+        }
+    }
+}
+
+
+void
+FastS_DataSetCollection::AbortQueryQueues(void)
+{
+    for (uint32_t datasetidx(0); datasetidx < GetMaxNumDataSets(); datasetidx++) {
+        FastS_DataSetBase *dataset = PeekDataSet(datasetidx);
+
+        if (dataset != NULL) {
+            dataset->LockDataset();
+            dataset->AbortQueryQueue_HasLock();
+            dataset->UnlockDataset();
+        }
+    }
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h
new file mode 100644
index 00000000000..af47e6d953c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/vespalib/util/referencecounter.h>
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+
+class FastS_DataSetBase;
+class FastS_ISearch;
+
+class FastS_DataSetCollection : public vespalib::ReferenceCounter
+{
+private:
+    FastS_DataSetCollection(const FastS_DataSetCollection &);
+    FastS_DataSetCollection& operator=(const FastS_DataSetCollection &);
+
+public:
+    // used by Monitor to service old query queues.
+    FastS_DataSetCollection *_nextOld;
+
+private:
+    FastS_DataSetCollDesc *_configDesc;
+    FastS_AppContext      *_appCtx;
+
+    FastS_DataSetBase    **_datasets;
+    uint32_t               _datasets_size;
+
+    uint32_t               _gencnt;
+    bool                   _frozen;
+    bool                   _error;
+
+    FastS_DataSetBase *CreateDataSet(FastS_DataSetDesc *desc);
+    bool AddDataSet(FastS_DataSetDesc *desc);
+
+public:
+    explicit FastS_DataSetCollection(FastS_AppContext *appCtx);
+    virtual ~FastS_DataSetCollection();
+
+    /**
+     * Configure this dataset collection. Note that the given config
+     * description is handed over to this object when this method is
+     * called. Also note that this method replaces the old methods used
+     * to add datasets and engines as well as the Freeze method. In
+     * other words; this method uses the given config description to
+     * create a new node setup and then freezing it. Using a NULL
+     * pointer for the config description is legal; it denotes the empty
+     * configuration.
+     *
+     * @return true(ok)/false(fail)
+     * @param cfgDesc configuration description
+     * @param gencnt the generation of this node setup
+     **/
+    bool Configure(FastS_DataSetCollDesc *cfgDesc, uint32_t gencnt);
+
+    /**
+     * This method may be used to verify that this dataset collection
+     * has been successfully configured. See @ref Configure.
+     *
+     * @return true if successfully configured
+     **/
+    bool IsValid() { return (_frozen && !_error); }
+
+    FastS_DataSetCollDesc *GetConfigDesc() { return _configDesc; }
+
+    FastS_AppContext *GetAppContext() { return _appCtx; }
+
+    uint32_t GetMaxNumDataSets() { return _datasets_size; }
+
+    FastS_DataSetBase *PeekDataSet(uint32_t datasetid)
+    { return (datasetid < _datasets_size) ? _datasets[datasetid] : NULL; }
+
+    uint32_t SuggestDataSet();
+    FastS_DataSetBase *GetDataSet(uint32_t datasetid);
+    FastS_DataSetBase *GetDataSet();
+
+    bool AreEnginesReady();
+
+    // create search
+    FastS_ISearch *CreateSearch(uint32_t dataSetID, FastS_TimeKeeper *timeKeeper);
+
+    // handle old query queues
+    bool IsLastRef(void) { return (refCount() == 1); }
+    void CheckQueryQueues(FastS_TimeKeeper *timeKeeper);
+    void AbortQueryQueues(void);
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp
new file mode 100644
index 00000000000..718d270ccd4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp
@@ -0,0 +1,422 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".search.engine_base");
+#include <vespa/searchcore/util/log.h>
+#include <vespa/vespalib/util/atomic.h>
+
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+#include <vespa/searchcore/fdispatch/search/plain_dataset.h>
+#include <vespa/searchcore/fdispatch/search/engine_base.h>
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+
+//---------------------------------------------------------------------------
+
+FastS_EngineBase::stats_t::stats_t()
+    : _fliptime(),
+      _floptime(),
+      _slowQueryCnt(0),
+      _slowDocsumCnt(0),
+      _slowQuerySecs(0.0),
+      _slowDocsumSecs(0.0),
+      _queueLenSampleAcc(0),
+      _queueLenSampleCnt(0),
+      _activecntSampleAcc(0),
+      _activecntSampleCnt(0),
+      _queueLenAcc(0.0),
+      _activecntAcc(0.0),
+      _queueLenIdx(0),
+      _queueLenValid(0)
+{
+    uint32_t i;
+
+    _fliptime.SetNow();
+    _floptime.SetNow();
+    for (i = 0; i < _queuestatsize; i++) {
+        _queueLens[i]._queueLen  = 0.0;
+        _queueLens[i]._activecnt = 0.0;
+    }
+}
+
+//---------------------------------------------------------------------------
+
+FastS_EngineBase::reported_t::reported_t()
+    : _queueLen(0),
+      _dispatchers(0),
+      _mld(false),
+      _reportedPartID(FastS_NoID32()),
+      _actNodes(0),
+      _maxNodes(0),
+      _actParts(0),
+      _maxParts(0),
+      _activeDocs(),
+      _docstamp(FastS_EngineBase::NoDocStamp())
+{
+    _activeDocs.valid = true;
+}
+
+
+FastS_EngineBase::reported_t::~reported_t(void)
+{
+}
+
+//---------------------------------------------------------------------------
+
+FastS_EngineBase::config_t::config_t(FastS_EngineDesc *desc)
+    : _name(NULL),
+      _unitrefcost(desc->GetUnitRefCost()),
+      _confPartID(desc->GetConfPartID()),
+      _confRowID(desc->GetConfRowID()),
+      _confPartIDOverrides(desc->GetConfPartIDOverrides())
+{
+    _name = strdup(desc->GetName());
+    FastS_assert(_name != NULL);
+}
+
+
+FastS_EngineBase::config_t::~config_t()
+{
+    free(_name);
+}
+
+//---------------------------------------------------------------------------
+
+FastS_EngineBase::FastS_EngineBase(FastS_EngineDesc *desc,
+                                   FastS_PlainDataSet *dataset)
+    : _stats(),
+      _reported(),
+      _config(desc),
+      _isUp(false),
+      _badness(BAD_NOT),
+      _partid(FastS_NoID32()),
+      _totalrefcost(0),
+      _activecnt(0),
+      _dataset(dataset),
+      _nextds(NULL),
+      _prevpart(NULL),
+      _nextpart(NULL)
+{
+    FastS_assert(_dataset != NULL);
+}
+
+
+FastS_EngineBase::~FastS_EngineBase()
+{
+    FastS_assert(_nextds   == NULL);
+    FastS_assert(_prevpart == NULL);
+    FastS_assert(_nextpart == NULL);
+    FastS_assert(_totalrefcost == 0);
+    FastS_assert(_activecnt == 0);
+}
+
+
+void
+FastS_EngineBase::SlowQuery(double limit, double secs, bool silent)
+{
+    LockEngine();
+    _stats._slowQueryCnt++;
+    _stats._slowQuerySecs += secs;
+    UnlockEngine();
+    if (!silent)
+        LOG(warning,
+            "engine %s query slow by %.3fs + %.3fs",
+            _config._name, limit, secs);
+}
+
+
+void
+FastS_EngineBase::SlowDocsum(double limit, double secs)
+{
+    LockEngine();
+    _stats._slowDocsumCnt++;
+    _stats._slowDocsumSecs += secs;
+    UnlockEngine();
+    LOG(warning,
+        "engine %s docsum slow by %.3fs + %.3fs",
+        _config._name, limit, secs);
+}
+
+
+void
+FastS_EngineBase::AddCost()
+{
+    _totalrefcost += _config._unitrefcost;
+    ++_activecnt;
+}
+
+
+void
+FastS_EngineBase::SubCost()
+{
+    FastS_assert(_totalrefcost >= _config._unitrefcost);
+    _totalrefcost -= _config._unitrefcost;
+    FastS_assert(_activecnt >= 1);
+    --_activecnt;
+}
+
+
+void
+FastS_EngineBase::SaveQueueLen_NoLock(uint32_t queueLen, uint32_t dispatchers)
+{
+    _reported._queueLen = queueLen;
+    _reported._dispatchers = dispatchers;
+    _stats._queueLenSampleAcc += queueLen;
+    _stats._queueLenSampleCnt++;
+    _stats._activecntSampleAcc += _activecnt;
+    _stats._activecntSampleCnt++;
+}
+
+
+void
+FastS_EngineBase::SampleQueueLens()
+{
+    double queueLen;
+    double activecnt;
+
+    LockEngine();
+    if (_stats._queueLenSampleCnt > 0)
+        queueLen = (double) _stats._queueLenSampleAcc / (double) _stats._queueLenSampleCnt;
+    else
+        queueLen = 0;
+    if (_stats._activecntSampleCnt > 0)
+        activecnt = (double) _stats._activecntSampleAcc / (double) _stats._activecntSampleCnt;
+    else
+        activecnt = 0;
+
+    _stats._queueLenSampleAcc = 0;
+    _stats._queueLenSampleCnt = 0;
+    _stats._activecntSampleAcc = 0;
+    _stats._activecntSampleCnt = 0;
+
+    _stats._queueLenAcc -= _stats._queueLens[_stats._queueLenIdx]._queueLen;
+    _stats._queueLens[_stats._queueLenIdx]._queueLen = queueLen;
+    _stats._queueLenAcc += queueLen;
+
+    _stats._activecntAcc -= _stats._queueLens[_stats._queueLenIdx]._activecnt;
+    _stats._queueLens[_stats._queueLenIdx]._activecnt = activecnt;
+    _stats._activecntAcc += activecnt;
+
+    _stats._queueLenIdx++;
+    if (_stats._queueLenIdx >= _stats._queuestatsize)
+        _stats._queueLenIdx = 0;
+    if (_stats._queueLenValid < _stats._queuestatsize)
+        _stats._queueLenValid++;
+    UnlockEngine();
+}
+
+void
+FastS_EngineBase::UpdateSearchTime(double tnow, double elapsed, bool timedout)
+{
+    (void) tnow;
+    (void) elapsed;
+    (void) timedout;
+}
+
+void
+FastS_EngineBase::MarkBad(uint32_t badness)
+{
+    bool worse = false;
+
+    LockEngine();
+    if (badness > _badness) {
+        _badness = badness;
+        worse = true;
+    }
+    UnlockEngine();
+
+    if (worse) {
+        if (badness <= BAD_NOT) {
+        } else {
+            _dataset->ScheduleCheckBad();
+        }
+    }
+}
+
+
+void
+FastS_EngineBase::ClearBad()
+{
+    LockEngine();
+    if (_badness >= BAD_CONFIG) {
+        UnlockEngine();
+        LOG(warning,
+            "engine %s still bad due to illegal config",
+            _config._name);
+        return;
+    }
+    _badness = BAD_NOT;
+    UnlockEngine();
+    HandleClearedBad();
+}
+
+
+void
+FastS_EngineBase::HandlePingResponse(uint32_t partid,
+                                     time_t docstamp,
+                                     bool mld,
+                                     uint32_t maxnodes,
+                                     uint32_t nodes,
+                                     uint32_t maxparts,
+                                     uint32_t parts,
+                                     PossCount activeDocs)
+{
+    // ignore really bad nodes
+    if (IsRealBad())
+        return;
+
+    _reported._reportedPartID = partid;
+
+    // override reported partid ?
+
+    if (_config._confPartIDOverrides && _config._confPartID != FastS_NoID32()) {
+        LOG(debug, "Partid(%d) overridden by config(%d)", partid, _config._confPartID);
+        partid = _config._confPartID;
+    }
+
+    // bad partid ?
+
+    if ((partid != _config._confPartID && _config._confPartID != FastS_NoID32()) ||
+        (partid < _dataset->GetFirstPart()) ||
+        (partid >= _dataset->GetLastPart()) ||
+        (partid >= _dataset->GetFirstPart() + (1 << _dataset->GetPartBits())))
+    {
+        LOG(warning, "Partid(%d) overridden to %d since it was bad: _confPartID(%d) dataset.first(%d), last(%d), (1 << bits)(%d)", partid, FastS_NoID32(), _config._confPartID, _dataset->GetFirstPart(), _dataset->GetLastPart(), (1 << _dataset->GetPartBits()));
+        partid = FastS_NoID32();
+    }
+
+    // what happened ?
+
+    bool onlined   = !IsUp();
+    bool bigchange = (!onlined &&
+                      (partid   != _partid             ||
+                       docstamp != _reported._docstamp));
+    bool changed   = (!onlined &&
+                      (bigchange                       ||
+                       mld      != _reported._mld      ||
+                       maxnodes != _reported._maxNodes ||
+                       nodes    != _reported._actNodes ||
+                       maxparts != _reported._maxParts ||
+                       activeDocs != _reported._activeDocs ||
+                       parts    != _reported._actParts));
+
+    bool partIdChanged = partid != _partid;
+    uint32_t oldPartID = _partid;
+    // nothing happened ?
+
+#if 0
+    LOG(info,
+        "HandlePingResponse: "
+        "engine %s (partid %d) docstamp %d, "
+        "onlined %s, changed %s",
+        _config._name,
+        static_cast<int>(partid),
+        static_cast<int>(docstamp),
+        onlined ? "true" : "false",
+        changed ? "true" : "false");
+#endif
+    if (!onlined && !changed)
+        return;
+
+    // report stuff
+
+    if (onlined) {
+        LOG(debug,
+            "Search node %s up, partition %d, docstamp %d",
+            _config._name, partid, (uint32_t) docstamp);
+    } else if (bigchange) {
+        if (partid != _partid) {
+            LOG(debug,
+                "Search node %s changed partid %u -> %u",
+                _config._name, _partid, partid);
+        }
+        if (docstamp != _reported._docstamp) {
+            LOG(debug,
+                "Search node %s changed docstamp %u -> %u",
+                _config._name,
+                (uint32_t)_reported._docstamp,
+                (uint32_t)docstamp);
+	    if (docstamp == 0) {
+	      LOG(warning, "Search node %s (partid %d) went bad (docstamp 0)",
+		  _config._name, partid);
+	    }
+        }
+    }
+
+    _dataset->LockDataset();
+    if (changed)
+        _dataset->LinkOutPart_HasLock(this);
+
+    _partid             = partid;
+    if (docstamp != _reported._docstamp) {
+        _reported._docstamp = docstamp;
+    }
+    _reported._mld      = mld;
+    _reported._maxNodes = maxnodes;
+    _reported._actNodes = nodes;
+    _reported._maxParts = maxparts;
+    _reported._actParts = parts;
+    if (_reported._activeDocs != activeDocs) {
+        _dataset->updateActiveDocs_HasLock(GetConfRowID(), activeDocs, _reported._activeDocs);
+        _reported._activeDocs = activeDocs;
+    }
+    _isUp               = true;
+
+    _dataset->LinkInPart_HasLock(this);
+
+    if (partIdChanged) {
+        _dataset->EnginePartIDChanged_HasLock(this, oldPartID);
+    }
+    _dataset->UnlockDataset();
+    _dataset->ScheduleCheckTempFail();
+
+    if (onlined) {
+        HandleUp();
+    }
+
+    // detect flipflop badness
+
+    // NB: fliphistory race with clearbad...
+
+    if (onlined || bigchange) {
+        _stats._fliptime.SetNow();
+    }
+}
+
+
+void
+FastS_EngineBase::HandleLostConnection()
+{
+    if (IsUp()) {
+        _isUp = false;
+        _stats._floptime.SetNow();
+        LOG(warning, "Search node %s down", _config._name);
+
+        _dataset->LockDataset();
+        _dataset->LinkOutPart_HasLock(this);
+        PossCount noDocs;
+        noDocs.valid = true;
+        _dataset->updateActiveDocs_HasLock(GetConfRowID(), noDocs, _reported._activeDocs);
+        _reported._activeDocs = noDocs;
+        _dataset->UnlockDataset();
+        _dataset->ScheduleCheckTempFail();
+        HandleDown(); // classic: NotifyVirtualConnsDown
+    }
+}
+
+
+void
+FastS_EngineBase::HandleNotOnline(int seconds)
+{
+    LOG(warning, "Search node %s still not up after %d seconds",
+        _config._name, seconds);
+}
+
+
+void
+FastS_EngineBase::Ping()
+{
+    SampleQueueLens();
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h
new file mode 100644
index 00000000000..002064aee77
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h
@@ -0,0 +1,199 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/searchcore/fdispatch/common/timestat.h>
+#include "plain_dataset.h"
+#include "poss_count.h"
+#include <atomic>
+
+class FastS_FNET_DataSet;
+class FastS_DataSetInfo;
+
+class FastS_FNET_Engine;
+class FastS_RPC_Engine;
+
+class FastS_EngineBase
+{
+    friend class FastS_FNET_Engine;
+    friend class FastS_RPC_Engine;
+    friend class FastS_PlainDataSet;
+    friend class FastS_FNET_DataSet;
+    friend class FastS_PartitionMap;
+    friend class FastS_DataSetInfo;
+
+private:
+    FastS_EngineBase(const FastS_EngineBase &);
+    FastS_EngineBase& operator=(const FastS_EngineBase &);
+
+public:
+
+    //----------------------------------------------------------------
+    // class holding various statistics for a search node
+    //----------------------------------------------------------------
+    class stats_t
+    {
+    public:
+        enum {
+            _queuestatsize       = 100
+        };
+
+        // the node goes up and down...
+        FastOS_Time _fliptime;	  // When state changed last to UP or big chg
+        FastOS_Time _floptime;	  // When state changed last from UP
+
+        // search/docsum slowness
+        uint32_t    _slowQueryCnt;
+        uint32_t    _slowDocsumCnt;
+        double      _slowQuerySecs;
+        double      _slowDocsumSecs;
+
+        // active cnt + queue len sampling
+        uint32_t _queueLenSampleAcc;  // sum of reported queue lengths
+        uint32_t _queueLenSampleCnt;  // number of reported queue lengths
+        uint32_t _activecntSampleAcc; // sum of our "load"
+        uint32_t _activecntSampleCnt; // number of our "load" samples
+
+        // sampled active cnt + queue len
+        struct {
+            double _queueLen;
+            double _activecnt;
+        } _queueLens[_queuestatsize];
+        double   _queueLenAcc;
+        double   _activecntAcc;
+        uint32_t _queueLenIdx;
+        uint32_t _queueLenValid;
+
+        stats_t();
+
+    };
+
+    //----------------------------------------------------------------
+    // class holding values reported from the node below
+    //----------------------------------------------------------------
+    class reported_t
+    {
+    private:
+        reported_t(const reported_t &);
+        reported_t& operator=(const reported_t &);
+
+    public:
+        uint32_t    _queueLen;        // queue len on search node
+        uint32_t    _dispatchers;     // # dispatchers using search node
+
+        bool        _mld;
+        uint32_t    _reportedPartID;  // Partid reported from node below
+        uint32_t    _actNodes;        // From _MLD_MON. # active nodes, or 1
+        uint32_t    _maxNodes;        // From _MLD_MON. total # nodes, or 1
+        uint32_t    _actParts;        // From _MLD_MON. # active parts, or 1
+        uint32_t    _maxParts;        // From _MLD_MON. total # parts, or 1
+        PossCount   _activeDocs;
+        time_t      _docstamp;
+
+        reported_t();
+        ~reported_t();
+    };
+
+    //----------------------------------------------------------------
+    // class holding config values
+    //----------------------------------------------------------------
+    class config_t
+    {
+    private:
+        config_t(const config_t &);
+        config_t& operator=(const config_t &);
+
+    public:
+        char       *_name;
+        uint32_t    _unitrefcost;	      // Cost to reference us
+        uint32_t    _confPartID;	      // Partid configured in partitions file
+        uint32_t    _confRowID;	      // What row this engine belongs to
+        bool	_confPartIDOverrides; // Ignore lower partid and use our conf value
+        config_t(FastS_EngineDesc *desc);
+        ~config_t();
+    };
+
+    // engine badness enum
+    enum {
+        BAD_NOT,
+        BAD_ADMIN,
+        BAD_CONFIG
+    };
+
+protected:
+    stats_t     _stats;
+    reported_t  _reported;
+    config_t    _config;
+
+    bool        _isUp;     // is this engine up ?
+    uint32_t    _badness;  // engine badness indicator
+    uint32_t    _partid;   // Partid we actually use
+
+    // Total cost as seen by referencing objects
+    std::atomic<uint32_t>  _totalrefcost;
+    std::atomic<uint32_t>  _activecnt;	// Our "load" on search node
+
+    FastS_PlainDataSet *_dataset; // dataset for this engine
+
+    FastS_EngineBase *_nextds;    // list of engines in dataset
+    FastS_EngineBase *_prevpart;  // list of engines in partition
+    FastS_EngineBase *_nextpart;  // list of engines in partition
+
+public:
+    FastS_EngineBase(FastS_EngineDesc *desc, FastS_PlainDataSet *dataset);
+    virtual ~FastS_EngineBase();
+
+    // common engine methods
+    //----------------------
+    static time_t NoDocStamp() { return static_cast<time_t>(-1); }
+    const char *GetName() const { return _config._name; }
+    FastS_EngineBase *GetNextDS() const { return _nextds; }
+    uint32_t GetQueueLen() const { return _reported._queueLen; }
+    uint32_t GetDispatchers() const { return _reported._dispatchers; }
+    FastS_PlainDataSet *GetDataSet() const { return _dataset; }
+    uint32_t GetConfRowID() const { return _config._confRowID; }
+    uint32_t GetPartID() const { return _partid; }
+
+    time_t GetTimeStamp() const { return _reported._docstamp; }
+    bool IsMLD() const { return _reported._mld; }
+
+    bool IsUp() const { return _isUp; }
+    bool IsRealBad() const { return (_badness > BAD_NOT); }
+    bool isAdminBad(void) const { return _badness == BAD_ADMIN; }
+    
+    bool IsReady() const { return (IsUp() || IsRealBad()); }
+    void SlowQuery(double limit, double secs, bool silent);
+    void SlowDocsum(double limit, double secs);
+    void AddCost();
+    void SubCost();
+    void SaveQueueLen_NoLock(uint32_t queueLen, uint32_t dispatchers);
+    void SampleQueueLens();
+    void UpdateSearchTime(double tnow, double elapsed, bool timedout);
+    void NotifyFailure();
+    void MarkBad(uint32_t badness);
+    void ClearBad();
+    void HandlePingResponse(uint32_t partid, time_t docstamp, bool mld,
+                            uint32_t maxnodes, uint32_t nodes,
+                            uint32_t maxparts, uint32_t parts,
+                            PossCount activeDocs);
+    void HandleLostConnection();
+    void HandleNotOnline(int seconds);
+
+    // common engine API
+    //------------------
+    virtual void LockEngine() = 0;
+    virtual void UnlockEngine() = 0;
+    virtual void Ping();
+    virtual void HandleClearedBad() {}
+    virtual void HandleUp() {}
+    virtual void HandleDown() {}
+
+    // typesafe "down"-cast
+    //---------------------
+    virtual FastS_FNET_Engine *GetFNETEngine()    { return NULL; }
+    virtual FastS_RPC_Engine  *GetRPCEngine()     { return NULL; }
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp
new file mode 100644
index 00000000000..fe45f9c97c2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp
@@ -0,0 +1,161 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".search.fnet_dataset");
+#include <vespa/fnet/fnet.h>
+#include <vespa/searchlib/common/fslimits.h>
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+#include <vespa/searchcore/util/eventloop.h>
+
+#include <vespa/searchlib/common/packets.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+#include <vespa/searchcore/fdispatch/search/fnet_dataset.h>
+#include <vespa/searchcore/fdispatch/search/fnet_engine.h>
+#include <vespa/searchcore/fdispatch/search/fnet_search.h>
+
+//--------------------------------------------------------------------------
+
+void
+FastS_FNET_DataSet::PingTask::PerformTask()
+{
+    _dataset->Ping();
+    Schedule(_delay);
+}
+
+//--------------------------------------------------------------------------
+
+FastS_FNET_DataSet::FastS_FNET_DataSet(FNET_Transport *transport,
+                                       FNET_Scheduler *scheduler,
+                                       FastS_AppContext *appCtx,
+                                       FastS_DataSetDesc *desc)
+    : FastS_PlainDataSet(appCtx, desc),
+      _transport(transport),
+      _pingTask(scheduler, this, getMonitorInterval()),
+      _failedRowsBitmask(0)
+{
+}
+
+
+FastS_FNET_DataSet::~FastS_FNET_DataSet()
+{
+}
+
+
+bool
+FastS_FNET_DataSet::AddEngine(FastS_EngineDesc *desc)
+{
+    FastS_FNET_Engine *engine;
+
+    engine = new FastS_FNET_Engine(desc, this);
+    FastS_assert(engine != NULL);
+
+    InsertEngine(engine);
+
+    if (desc->IsBad()) {
+        engine->MarkBad(FastS_EngineBase::BAD_CONFIG);
+    }
+    return true;
+}
+
+
+namespace {
+struct ConnectFNETEngine {
+    void operator()(FastS_EngineBase* engine) {
+        FastS_FNET_Engine* fnet_engine = engine->GetFNETEngine();
+        FastS_assert(fnet_engine != NULL);
+        fnet_engine->ScheduleConnect(0.0);
+        fnet_engine->StartWarnTimer();
+    }
+};
+}
+
+void
+FastS_FNET_DataSet::ConfigDone(FastS_DataSetCollection *)
+{
+    _enginesArray.ForEach( ConnectFNETEngine() );
+    _pingTask.ScheduleNow();
+}
+
+
+void
+FastS_FNET_DataSet::ScheduleCheckBad()
+{
+    _pingTask.ScheduleNow();
+}
+
+
+FastS_ISearch *
+FastS_FNET_DataSet::CreateSearch(FastS_DataSetCollection *dsc,
+                                 FastS_TimeKeeper *timeKeeper,
+                                 bool async)
+{
+    return (async)
+        ? (FastS_ISearch *) new FastS_FNET_Search(dsc, this, timeKeeper)
+        : (FastS_ISearch *) new FastS_Sync_FNET_Search(dsc, this, timeKeeper);
+}
+
+
+void
+FastS_FNET_DataSet::Free()
+{
+    _pingTask.Kill();
+
+    for (FastS_EngineBase *engine = ExtractEngine();
+         engine != NULL; engine = ExtractEngine())
+    {
+        FastS_assert(engine->GetFNETEngine() != NULL);
+        delete engine;
+    }
+
+    delete this;
+}
+
+bool
+FastS_FNET_DataSet::isGoodRow(uint32_t rowId)
+{
+    LockDataset();
+    uint64_t rowBit = 1ul << rowId;
+    bool wasBad = ((_failedRowsBitmask & rowBit) != 0);
+    bool isBad = false;
+    uint64_t candDocs = _stateOfRows.getRowState(rowId).activeDocs();
+    // demand: (candidate row active docs >= p% of average active docs)
+    // where p = min activedocs coverage
+    double p = _queryDistributionMode.getMinActivedocsCoverage() / 100.0;
+    p = std::min(p, 0.999); // max demand: 99.9 %
+    uint64_t restDocs = _stateOfRows.sumActiveDocs() - candDocs;
+    uint64_t restRows = _stateOfRows.numRowStates() - 1;
+    double restAvg = (restRows > 0) ? (restDocs / (double)restRows) : 0;
+    if (_stateOfRows.activeDocsValid() && (candDocs < (p * restAvg))) {
+        isBad = true;
+        if (!wasBad) {
+            _failedRowsBitmask |= rowBit;
+            LOG(warning, "Not enough active docs in row %d (only %lu docs, average is %g)",
+                rowId, candDocs, restAvg);
+        }
+    }
+    size_t nodesUp = countNodesUpInRow_HasLock(rowId);
+    size_t configuredParts = getNumPartitions(rowId);
+    size_t nodesAllowedDown =
+        getMaxNodesDownPerFixedRow() +
+        (configuredParts*(100.0 - getMinGroupCoverage()))/100.0;
+    if (nodesUp + nodesAllowedDown < configuredParts) {
+        isBad = true;
+        if (!wasBad) {
+            _failedRowsBitmask |= rowBit;
+            LOG(warning, "Coverage of row %d is only %ld/%ld (requires %ld)",
+                rowId, nodesUp, configuredParts, configuredParts-nodesAllowedDown);
+        }
+    }
+    if (wasBad && !isBad) {
+        _failedRowsBitmask &= ~rowBit;
+        LOG(info, "Row %d is now good again (%lu/%g active docs, coverage %ld/%ld)",
+            rowId, candDocs, restAvg, nodesUp, configuredParts);
+    }
+    UnlockDataset();
+    return !isBad;
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h
new file mode 100644
index 00000000000..9f55f336f43
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/searchcore/fdispatch/search/plain_dataset.h>
+
+
+class FastS_FNET_DataSet : public FastS_PlainDataSet
+{
+private:
+    FastS_FNET_DataSet(const FastS_FNET_DataSet &);
+    FastS_FNET_DataSet& operator=(const FastS_FNET_DataSet &);
+
+public:
+
+    //----------------------------------------------------------------
+    // class used to schedule periodic dataset pinging
+    //----------------------------------------------------------------
+
+    class PingTask : public FNET_Task
+    {
+    private:
+        PingTask(const PingTask &);
+        PingTask& operator=(const PingTask &);
+
+        FastS_FNET_DataSet *_dataset;
+        double              _delay;
+
+    public:
+        PingTask(FNET_Scheduler *scheduler,
+                 FastS_FNET_DataSet *dataset,
+                 double delay)
+            : FNET_Task(scheduler),
+              _dataset(dataset),
+              _delay(delay)
+        {}
+        void PerformTask();
+    };
+
+
+private:
+    FNET_Transport *_transport;
+    PingTask        _pingTask;
+    uint64_t        _failedRowsBitmask;
+
+public:
+    FastS_FNET_DataSet(FNET_Transport *transport,
+                       FNET_Scheduler *scheduler,
+                       FastS_AppContext *appCtx,
+                       FastS_DataSetDesc *desc);
+    virtual ~FastS_FNET_DataSet();
+
+    FNET_Transport *GetTransport() { return _transport; }
+
+    // typesafe down-cast
+    virtual FastS_FNET_DataSet *GetFNETDataSet() { return this; }
+
+    // common dataset API
+    virtual bool AddEngine(FastS_EngineDesc *desc);
+    virtual void ConfigDone(FastS_DataSetCollection *);
+    virtual void ScheduleCheckBad();
+    virtual FastS_ISearch *CreateSearch(FastS_DataSetCollection *dsc,
+                                        FastS_TimeKeeper *timeKeeper,
+                                        bool async);
+    virtual void Free();
+
+    bool isGoodRow(uint32_t rowId);
+};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp
new file mode 100644
index 00000000000..d6f9ee479e9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp
@@ -0,0 +1,251 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".search.fnet_engine");
+#include <vespa/fnet/fnet.h>
+#include <vespa/searchcore/fdispatch/common/stdincl.h>
+#include <vespa/searchlib/common/fslimits.h>
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+
+#include <vespa/searchlib/common/packets.h>
+#include <vespa/searchcore/fdispatch/search/fnet_dataset.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+#include <vespa/searchcore/fdispatch/search/fnet_engine.h>
+
+using namespace search::fs4transport;
+
+//----------------------------------------------------------------------
+
+void
+FastS_StaticMonitorQuery::Free(void)
+{
+    _lock.Lock();
+    _refcnt--;
+    if (_refcnt == 0) {
+        _lock.Unlock();
+        delete this;
+    } else
+        _lock.Unlock();
+}
+
+
+FastS_StaticMonitorQuery::FastS_StaticMonitorQuery(void)
+    : FS4Packet_MONITORQUERYX(),
+      _lock(),
+      _refcnt(1)
+{
+
+}
+
+
+FastS_StaticMonitorQuery::~FastS_StaticMonitorQuery(void)
+{
+    FastS_assert(_refcnt == 0);
+}
+
+//----------------------------------------------------------------------
+
+void
+FastS_FNET_Engine::WarnTask::PerformTask()
+{
+    _engine->HandleNotOnline(DELAY);
+}
+
+//----------------------------------------------------------------------
+
+void
+FastS_FNET_Engine::ConnectTask::PerformTask()
+{
+    _engine->Connect();
+}
+
+//----------------------------------------------------------------------
+
+void
+FastS_FNET_Engine::Connect()
+{
+    if (_conn == NULL ||
+        _conn->GetState() >= FNET_Connection::FNET_CLOSING)
+    {
+        FNET_Connection *newConn =
+            _transport->Connect(_spec.c_str(),
+                                &FS4PersistentPacketStreamer::Instance,
+                                this);
+        LockDataSet();
+        FNET_Connection *oldConn = _conn;
+        _conn = newConn;
+        UnlockDataSet();
+        if (oldConn != NULL)
+            oldConn->SubRef();
+        if (newConn == NULL && !IsRealBad())
+            ScheduleConnect(2.9);
+    }
+}
+
+
+void
+FastS_FNET_Engine::Disconnect()
+{
+    if (_conn != NULL) {
+        _conn->CloseAdminChannel();
+        LockDataSet();
+        FNET_Connection *conn = _conn;
+        _conn = NULL;
+        UnlockDataSet();
+        _transport->Close(conn, /* needref = */ false);
+    }
+}
+
+
+FastS_FNET_Engine::FastS_FNET_Engine(FastS_EngineDesc *desc,
+                                     FastS_FNET_DataSet *dataset)
+    : FastS_EngineBase(desc, dataset),
+      _lock(),
+      _spec(),
+      _transport(dataset->GetTransport()),
+      _conn(NULL),
+      _warnTask(dataset->GetAppContext()->GetFNETScheduler(), this),
+      _connectTask(dataset->GetAppContext()->GetFNETScheduler(), this),
+      _monitorQuery(NULL)
+{
+    if (strncmp(_config._name, "tcp/", 4) == 0) {
+        _spec = _config._name;
+    } else {
+        _spec = "tcp/";
+        _spec += _config._name;
+    }
+}
+
+
+FastS_FNET_Engine::~FastS_FNET_Engine()
+{
+    _warnTask.Kill();
+    _connectTask.Kill();
+    Disconnect();
+    if (IsUp()) {
+        LockDataSet();
+        _dataset->LinkOutPart_HasLock(this);
+        UnlockDataSet();
+    }
+    if (_monitorQuery != NULL) {
+        _monitorQuery->Free();
+        _monitorQuery = NULL;
+    }
+}
+
+
+void
+FastS_FNET_Engine::StartWarnTimer()
+{
+    _warnTask.Schedule(_warnTask.DELAY);
+}
+
+
+void
+FastS_FNET_Engine::ScheduleConnect(double delay)
+{
+    if (delay == 0.0) {
+        _connectTask.ScheduleNow();
+    } else {
+        _connectTask.Schedule(delay);
+    }
+}
+
+
+FNET_Channel *
+FastS_FNET_Engine::OpenChannel_HasDSLock(FNET_IPacketHandler *handler)
+{
+    return (_conn != NULL) ?  _conn->OpenChannel(handler, FNET_Context()) : NULL;
+}
+
+
+FNET_IPacketHandler::HP_RetCode
+FastS_FNET_Engine::HandlePacket(FNET_Packet *packet, FNET_Context)
+{
+    HP_RetCode ret = FNET_KEEP_CHANNEL;
+    uint32_t pcode = packet->GetPCODE();
+
+    if (packet->IsChannelLostCMD()) {
+
+        HandleLostConnection();
+        ret = FNET_FREE_CHANNEL;
+        if (!IsRealBad()) {
+            ScheduleConnect(2.9);
+        }
+
+    } else if (pcode == search::fs4transport::PCODE_MONITORRESULTX) {
+
+        FS4Packet_MONITORRESULTX *mr = (FS4Packet_MONITORRESULTX *) packet;
+
+        PossCount activeDocs;
+        activeDocs.valid = ((mr->_features & search::fs4transport::MRF_ACTIVEDOCS) != 0);
+        activeDocs.count = mr->_activeDocs;
+        if ((mr->_features & search::fs4transport::MRF_MLD) != 0) {
+            HandlePingResponse(mr->_partid, mr->_timestamp, true,
+                               mr->_totalNodes, mr->_activeNodes,
+                               mr->_totalParts, mr->_activeParts,
+                               activeDocs);
+        } else {
+            HandlePingResponse(mr->_partid, mr->_timestamp, false, 1, 1, 1, 1, activeDocs);
+        }
+    }
+
+    packet->Free();
+    return ret;
+}
+
+
+void
+FastS_FNET_Engine::Ping()
+{
+    FastS_EngineBase::Ping();
+
+    // handle badness
+    if (IsRealBad()) {
+        if (_conn != NULL) {
+            Disconnect();
+            HandleLostConnection();
+        }
+        return;
+    }
+
+    // handle ping
+    if ((_conn != NULL) && (_conn->GetState() < FNET_Connection::FNET_CLOSING)) {
+        if (_monitorQuery == NULL) {
+            _monitorQuery = new FastS_StaticMonitorQuery();
+        }
+        if (_monitorQuery->getBusy()) {
+            return;
+        }
+        _monitorQuery->markBusy();
+        uint32_t features = 0;
+        uint32_t qflags = 0;
+        qflags |= search::fs4transport::MQFLAG_REPORT_ACTIVEDOCS;
+        if (qflags != 0) {
+            features |= search::fs4transport::MQF_QFLAGS;
+        }
+        _monitorQuery->_features |= features;
+        _monitorQuery->_qflags = qflags;
+        _monitorQuery->UpdateCompatPCODE();
+        _conn->PostPacket(_monitorQuery, FastS_NoID32());
+    }
+}
+
+
+void
+FastS_FNET_Engine::HandleClearedBad()
+{
+    ScheduleConnect(0.0);
+}
+
+
+void
+FastS_FNET_Engine::HandleUp()
+{
+    _warnTask.Unschedule();
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h
new file mode 100644
index 00000000000..552caf526a3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/searchcore/fdispatch/search/engine_base.h>
+#include <vespa/searchlib/common/packets.h>
+
+//----------------------------------------------------------------------
+
+using search::fs4transport::FS4Packet_MONITORQUERYX;
+
+class FastS_StaticMonitorQuery : public FS4Packet_MONITORQUERYX
+{
+    FastOS_Mutex _lock;
+    int _refcnt;
+public:
+    virtual void Free();
+
+    bool getBusy(void) const
+    {
+        return _refcnt > 1;
+    }
+
+    void markBusy(void)
+    {
+        _lock.Lock();
+        _refcnt++;
+        _lock.Unlock();
+    }
+    FastS_StaticMonitorQuery(void);
+    ~FastS_StaticMonitorQuery(void);
+};
+
+//----------------------------------------------------------------------
+
+class FastS_FNET_Engine : public FNET_IPacketHandler,
+                          public FastS_EngineBase
+{
+private:
+    FastS_FNET_Engine(const FastS_FNET_Engine &);
+    FastS_FNET_Engine& operator=(const FastS_FNET_Engine &);
+
+public:
+    class WarnTask : public FNET_Task
+    {
+    private:
+        WarnTask(const WarnTask &);
+        WarnTask& operator=(const WarnTask &);
+
+        FastS_FNET_Engine *_engine;
+
+    public:
+        enum { DELAY = 30 };
+        WarnTask(FNET_Scheduler *scheduler,
+                 FastS_FNET_Engine *engine)
+            : FNET_Task(scheduler), _engine(engine) {}
+        virtual void PerformTask();
+    };
+    friend class FastS_FNET_Engine::WarnTask;
+
+    class ConnectTask : public FNET_Task
+    {
+    private:
+        ConnectTask(const ConnectTask &);
+        ConnectTask& operator=(const ConnectTask &);
+
+        FastS_FNET_Engine *_engine;
+
+    public:
+        ConnectTask(FNET_Scheduler *scheduler,
+                    FastS_FNET_Engine *engine)
+            : FNET_Task(scheduler), _engine(engine) {}
+        virtual void PerformTask();
+    };
+    friend class FastS_FNET_Engine::ConnectTask;
+
+private:
+    FastOS_Mutex      _lock;
+    std::string   _hostName;
+    int           _portNumber;
+    std::string   _spec;
+    FNET_Transport  *_transport;
+    FNET_Connection *_conn;
+    WarnTask         _warnTask;
+    ConnectTask      _connectTask;
+    FastS_StaticMonitorQuery *_monitorQuery;
+
+    void Connect();
+    void Disconnect();
+
+public:
+    FastS_FNET_Engine(FastS_EngineDesc *desc,
+                      FastS_FNET_DataSet *dataset);
+    virtual ~FastS_FNET_Engine();
+
+    void LockDataSet()   { _dataset->LockDataset();   }
+    void UnlockDataSet() { _dataset->UnlockDataset(); }
+
+    void StartWarnTimer();
+    void ScheduleConnect(double delay);
+    FNET_Channel *OpenChannel_HasDSLock(FNET_IPacketHandler *handler);
+
+    // handle FNET admin packets
+    //--------------------------
+    virtual HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context);
+
+    // common engine API
+    //------------------
+    virtual void LockEngine()   { _lock.Lock();   }
+    virtual void UnlockEngine() { _lock.Unlock(); }
+    virtual void Ping();
+    virtual void HandleClearedBad();
+    virtual void HandleUp();
+
+    // typesafe "down"-cast
+    //---------------------
+    virtual FastS_FNET_Engine *GetFNETEngine()    { return this; }
+
+    const char *getHostName() const { return _hostName.c_str(); }
+    int getPortNumber() const { return _portNumber; }
+};
+
+//----------------------------------------------------------------------
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp
new file mode 100644
index 00000000000..10bfdadba0e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp
@@ -0,0 +1,1522 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".fnet_search");
+#include <vespa/fnet/fnet.h>
+#include <vespa/searchlib/util/rand48.h>
+
+#include <vespa/searchlib/common/mapnames.h>
+#include <vespa/searchlib/common/packets.h>
+#include <vespa/searchlib/engine/packetconverter.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+#include <vespa/searchcore/fdispatch/search/fnet_dataset.h>
+#include <vespa/searchcore/fdispatch/search/fnet_engine.h>
+#include <vespa/searchcore/fdispatch/search/fnet_search.h>
+#include <vespa/searchcore/fdispatch/search/mergehits.h>
+#include <vespa/searchcore/util/eventloop.h>
+
+#include <set>
+#include <stdexcept>
+#include <atomic>
+
+#define IS_MLD_PART(part)		((part) > mldpartidmask)
+#define MLD_PART_TO_PARTID(part)	((part) & mldpartidmask)
+#define ENCODE_MLD_PART(part)		(((part) + 1) << partbits)
+#define DECODE_MLD_PART(part)		(((part) >> partbits) - 1)
+
+using fdispatch::SearchPath;
+using vespalib::nbostream;
+using vespalib::stringref;
+using namespace search::fs4transport;
+using search::engine::PacketConverter;
+
+//---------------------------------------------------------------------
+//
+
+FastS_FNET_SearchNode::FastS_FNET_SearchNode(FastS_FNET_Search *search, uint32_t partid)
+    : _search(search),
+      _engine(NULL),
+      _channel(NULL),
+      _subds(0),
+      _partid(partid),
+      _rowid(0),
+      _stamp(0),
+      _qresult(NULL),
+      _queryTime(0.0),
+      _flags(),
+      _docidCnt(0),
+      _pendingDocsums(0),
+      _docsumRow(0),
+      _docsum_offsets_idx(0),
+      _docsumTime(0.0),
+      _gdx(NULL),
+      _docsum_offsets(),
+      _extraDocsumNodes(),
+      _nextExtraDocsumNode(this),
+      _prevExtraDocsumNode(this),
+      _hit_beg(NULL),
+      _hit_cur(NULL),
+      _hit_end(NULL),
+      _sortDataIterator()
+{
+}
+
+
+FastS_FNET_SearchNode::~FastS_FNET_SearchNode()
+{
+    Disconnect();
+    if (_qresult != NULL) {
+        _qresult->Free();
+    }
+    if (_gdx != NULL) {
+        _gdx->Free();
+    }
+}
+
+FastS_FNET_SearchNode::FastS_FNET_SearchNode(FastS_FNET_SearchNode &&)
+{
+    // These objects are referenced everywhere and must never be either copied nor moved,
+    // but as std::vector requires this to exist so we do this little trick.
+    assert(false);
+}
+
+bool
+FastS_FNET_SearchNode::NT_InitMerge(uint32_t *numDocs,
+                                    uint64_t *totalHits,
+                                    search::HitRank *maxRank,
+                                    uint32_t *sortDataDocs)
+{
+    uint32_t myNumDocs = 0;
+    if (_qresult != NULL) {
+        myNumDocs = _qresult->_numDocs;
+        *numDocs += myNumDocs;
+        *totalHits += _qresult->_totNumDocs;
+        search::HitRank mr = _qresult->_maxRank;
+        if (mr > *maxRank)
+            *maxRank = mr;
+    }
+    if (myNumDocs > 0) {
+        _hit_beg = _qresult->_hits;
+        _hit_cur = _hit_beg;
+        _hit_end = _hit_beg + myNumDocs;
+        if ((_qresult->_features & search::fs4transport::QRF_SORTDATA) != 0) {
+            _sortDataIterator.Init(myNumDocs, _qresult->_sortIndex, _qresult->_sortData);
+            *sortDataDocs += myNumDocs;
+        }
+        return true;
+    }
+    return false;
+}
+
+
+FastS_EngineBase *
+FastS_FNET_SearchNode::getPartition(const FastOS_Mutex & dsMutex, bool userow, FastS_FNET_DataSet *dataset)
+{
+    return ((userow)
+        ? dataset->getPartitionMLD(dsMutex, getPartID(), _flags._docsumMld, _docsumRow)
+        : dataset->getPartitionMLD(dsMutex, getPartID(), _flags._docsumMld));
+}
+
+
+void
+FastS_FNET_SearchNode::
+allocGDX(search::docsummary::GetDocsumArgs *args, const search::engine::PropertiesMap &props)
+{
+    FS4Packet_GETDOCSUMSX *gdx = new FS4Packet_GETDOCSUMSX(search::fs4transport::PCODE_GETDOCSUMSX);
+    FastS_assert(gdx != NULL);
+
+    gdx->AllocateDocIDs(_docidCnt);
+    _gdx = gdx;
+    _docsum_offsets.resize(_gdx->_docidCnt);
+    _docsum_offsets_idx = 0;
+    if (args == NULL)
+        return;
+
+    if (args->getRankProfile().size() != 0 || args->GetQueryFlags() != 0) {
+        gdx->_features |= search::fs4transport::GDF_RANKP_QFLAGS;
+        gdx->setRanking(args->getRankProfile());
+        gdx->_qflags = args->GetQueryFlags();
+    }
+    gdx->setTimeout(args->getTimeout());
+
+    if (args->getResultClassName().size() > 0) {
+        gdx->_features |= search::fs4transport::GDF_RESCLASSNAME;
+        gdx->setResultClassName(args->getResultClassName());
+    }
+
+    if (props.size() > 0) {
+        PacketConverter::fillPacketProperties(props, gdx->_propsVector);
+        gdx->_features |= search::fs4transport::GDF_PROPERTIES;
+    }
+
+    if (args->getStackDump().size() > 0) {
+        gdx->_features |= search::fs4transport::GDF_QUERYSTACK;
+        gdx->_stackItems = args->GetStackItems();
+        gdx->setStackDump(args->getStackDump());
+    }
+
+    if (args->GetLocationLen() > 0) {
+        gdx->_features |= search::fs4transport::GDF_LOCATION;
+        gdx->setLocation(args->getLocation());
+    }
+
+    if (args->getFlags() != 0) {
+        gdx->_features |= search::fs4transport::GDF_FLAGS;
+        gdx->_flags = args->getFlags();
+    }
+}
+
+
+void
+FastS_FNET_SearchNode::postGDX(uint32_t *pendingDocsums, uint32_t *docsumNodes)
+{
+    FS4Packet_GETDOCSUMSX *gdx = _gdx;
+    FastS_assert(gdx->_docidCnt == _docsum_offsets_idx);
+    if (_flags._docsumMld) {
+        gdx->_features |= search::fs4transport::GDF_MLD;
+    }
+    gdx->UpdateCompatPCODE();
+    if (PostPacket(gdx)) {
+        _pendingDocsums = _docsum_offsets_idx;
+        *pendingDocsums += _pendingDocsums;
+        (*docsumNodes)++;
+    }
+    _gdx = NULL; // packet hand-over
+    _docsum_offsets_idx = 0;
+}
+
+
+FNET_IPacketHandler::HP_RetCode
+FastS_FNET_SearchNode::HandlePacket(FNET_Packet *packet, FNET_Context context)
+{
+    uint32_t pcode = packet->GetPCODE();
+    if (LOG_WOULD_LOG(spam)) {
+        LOG(spam, "handling packet %p\npacket=%s", packet, packet->Print().c_str());
+        context.Print();
+    }
+    if (packet->IsChannelLostCMD()) {
+        _search->LostSearchNode(this);
+    } else if (pcode == search::fs4transport::PCODE_QUERYRESULTX) {
+        _search->GotQueryResult(this, (FS4Packet_QUERYRESULTX *) packet);
+    } else if (pcode == search::fs4transport::PCODE_DOCSUM) {
+        _search->GotDocsum(this, (FS4Packet_DOCSUM *) packet);
+    } else if (pcode == search::fs4transport::PCODE_ERROR) {
+        _search->GotError(this, static_cast<FS4Packet_ERROR *>(packet));
+    } else {
+        if (pcode == search::fs4transport::PCODE_EOL) {
+            _search->GotEOL(this);
+        }
+        packet->Free();
+    }
+    return FNET_KEEP_CHANNEL;
+}
+
+
+FastS_FNET_SearchNode *
+FastS_FNET_SearchNode::
+allocExtraDocsumNode(bool mld, uint32_t rowid, uint32_t rowbits)
+{
+    if (_extraDocsumNodes.empty()) {
+        size_t sz = (1 << (rowbits + 1));
+        _extraDocsumNodes.resize(sz);
+    }
+
+    uint32_t idx = (rowid << 1) + (mld ? 1 : 0);
+
+    if (_extraDocsumNodes[idx].get() == NULL) {
+        UP eNode(new FastS_FNET_SearchNode(_search, getPartID()));
+        eNode->_docsumRow = rowid;
+        eNode->_flags._docsumMld = mld;
+
+        eNode->_nextExtraDocsumNode = this;
+        eNode->_prevExtraDocsumNode = _prevExtraDocsumNode;
+        _prevExtraDocsumNode->_nextExtraDocsumNode = eNode.get();
+        _prevExtraDocsumNode = eNode.get();
+        _extraDocsumNodes[idx] = std::move(eNode);
+    }
+    return _extraDocsumNodes[idx].get();
+}
+
+
+//---------------------------------------------------------------------
+
+void
+FastS_FNET_Search::Timeout::PerformTask()
+{
+    _search->HandleTimeout();
+}
+
+//---------------------------------------------------------------------
+
+void
+FastS_FNET_Search::AllocNodes()
+{
+    FastS_assert(_nodes.empty());
+
+    _nodes.reserve(_dataset->GetPartitions());
+
+    for (uint32_t i = 0; i < _nodes.capacity(); i++) {
+        _nodes.emplace_back(this, i);
+    }
+}
+
+namespace {
+volatile std::atomic<uint64_t> _G_prevFixedRow(0);
+} //anonymous namespace
+
+uint32_t
+FastS_FNET_Search::getFixedRowCandidate()
+{
+    uint32_t rowId(_dataset->useRoundRobinForFixedRow()
+                   ? (_G_prevFixedRow++)
+                   : _dataset->getRandomWeightedRow());
+    return rowId % _dataset->getNumRows();
+}
+
+uint32_t
+FastS_FNET_Search::getNextFixedRow()
+{
+    size_t numTries(0);
+    uint32_t fixedRow(0);
+    size_t maxTries(_dataset->getNumRows());
+    if ( ! _dataset->useRoundRobinForFixedRow()) {
+       maxTries *= 10;
+    }
+    for(;numTries < maxTries; numTries++) {
+        fixedRow = getFixedRowCandidate();
+        if (_dataset->isGoodRow(fixedRow)) {
+            break;
+        }
+    }
+    if (numTries == maxTries) {
+        fixedRow = getFixedRowCandidate(); // Will roundrobin/random if all rows are incomplete.
+    }
+    LOG(debug, "FixedRow: selected=%d, numRows=%d, numTries=%ld, _G_prevFixedRow=%ld", fixedRow, _dataset->getNumRows(), numTries, _G_prevFixedRow.load());
+    return fixedRow;
+}
+
+void
+FastS_FNET_Search::connectNodes(const EngineNodeMap & engines)
+{
+    for (const auto & pair : engines) {
+        if ( ! pair.second->IsConnected() ) {
+            // Here we are connecting without having the DataSet lock.
+            // This might give a race when nodes go up or down, or there is a config change.
+            // However none has ever been detected for as long as the race has existed.
+            // The correct fix would be to make the DataSet be constant and be replaced upon changes.
+            // And using shared_ptr to them. That would avoid the big global lock all together.
+            pair.second->Connect_HasDSLock(pair.first->GetFNETEngine());
+        } else {
+            pair.first->SubCost();
+        }
+    }
+    _nodesConnected = true;
+}
+
+void
+FastS_FNET_Search::ConnectQueryNodes()
+{
+    FastS_assert( ! _nodes.empty() );
+    FastS_assert(!_nodesConnected);
+
+    uint32_t fixedRow(0);
+    if (_dataset->useFixedRowDistribution()) {
+        fixedRow = getNextFixedRow();
+        _fixedRow = fixedRow;
+    }
+    EngineNodeMap engines;
+    engines.reserve(_nodes.size());
+    FastOS_Mutex & dsLock = _dataset->getMutex();
+    dsLock.Lock();
+    for (uint32_t i = 0; i < _nodes.size(); i++) {
+        FastS_EngineBase *engine = NULL;
+        if (_dataset->useFixedRowDistribution()) {
+            engine = _dataset->getPartition(dsLock, i, fixedRow);
+            LOG(debug, "FixedRow: getPartition(part=%u, row=%u) -> engine(%s)", i, fixedRow, (engine != nullptr ? engine->GetName() : "null"));
+        } else {
+            engine = _dataset->getPartition(dsLock, i);
+        }
+        if (engine != nullptr) {
+            LOG(debug, "Wanted part=%d, engine={name=%s, row=%d, partid=%d}", i, engine->GetName(), engine->GetConfRowID(), engine->GetPartID());
+            if (engine != nullptr) {
+                engines.emplace_back(engine, getNode(i));
+            }
+        } else {
+            LOG(debug, "No engine for part %d", i);
+        }
+    }
+    dsLock.Unlock();
+    connectNodes(engines);
+}
+
+
+void
+FastS_FNET_Search::ConnectEstimateNodes()
+{
+    FastS_assert( ! _nodes.empty() );
+    FastS_assert(!_nodesConnected);
+
+    uint32_t partid  = _util.GetQuery().StackDumpHashKey() % _estPartCutoff;
+    uint32_t trycnt  = 0;
+    uint32_t partcnt = 0;
+
+    EngineNodeMap engines;
+    FastOS_Mutex & dsLock = _dataset->getMutex();
+    dsLock.Lock();
+    while (partcnt < _dataset->GetEstimateParts() && trycnt < _estPartCutoff) {
+        FastS_EngineBase *engine = _dataset->getPartition(dsLock, partid);
+        if (engine != NULL) {
+            engines.emplace_back(engine, getNode(partid));
+            partcnt++;
+        }
+        trycnt++;
+        partid = (partid + 1) % _estPartCutoff;
+    }
+    _estParts = partcnt;
+    dsLock.Unlock();
+    connectNodes(engines);
+}
+
+
+void FastS_FNET_SearchNode::Connect(FastS_FNET_Engine *engine)
+{
+    FastS_assert(_engine == NULL);
+    FastS_assert(_channel == NULL);
+
+    _engine = engine;
+    _flags._needSubCost = true;
+    _engine->LockDataSet();
+    _channel = _engine->OpenChannel_HasDSLock(this);
+    _rowid   = _engine->GetConfRowID();
+    _stamp   = _engine->GetTimeStamp();
+    _engine->UnlockDataSet();
+}
+
+void FastS_FNET_SearchNode::Connect_HasDSLock(FastS_FNET_Engine *engine)
+{
+    _engine = engine;
+    _flags._needSubCost = true;
+    _channel = _engine->OpenChannel_HasDSLock(this);
+    _rowid   = _engine->GetConfRowID();
+    _stamp   = _engine->GetTimeStamp();
+}
+
+
+void FastS_FNET_Search::connectSearchPath(const vespalib::string &spec)
+{
+    FastS_assert( ! _nodes.empty());
+    FastS_assert(!_nodesConnected);
+
+    SearchPath searchPath(spec, _nodes.size());
+    uint32_t dispatchLevel = _dsc->GetAppContext()->getDispatchLevel();
+    LOG(debug, "Looking up searchpath element for dispatch level %u in searchpath '%s' (size=%zu)",
+        dispatchLevel, spec.c_str(), searchPath.elements().size());
+    if (dispatchLevel < searchPath.elements().size()) {
+        connectSearchPath(searchPath.elements()[dispatchLevel], spec, dispatchLevel);
+    } else {
+        LOG(warning, "Did not find searchpath element for dispatch level "
+            "%u in searchpath '%s' (size=%zu). No search nodes will be queried.",
+            dispatchLevel, spec.c_str(), searchPath.elements().size());
+    }
+}
+
+void FastS_FNET_Search::connectSearchPath(const SearchPath::Element &elem,
+                                          const vespalib::string &spec,
+                                          uint32_t dispatchLevel)
+{
+    EngineNodeMap engines;
+    FastOS_Mutex & dsLock = _dataset->getMutex();
+    dsLock.Lock();
+    if (!elem.hasRow()) {
+        for (size_t partId : elem.nodes()) {
+            if (partId < _nodes.size()) {
+                FastS_EngineBase *engine = _dataset->getPartition(dsLock, partId);
+                LOG(debug, "searchpath='%s', partId=%ld, dispatchLevel=%u", spec.c_str(), partId, dispatchLevel);
+                if (engine != NULL) {
+                    engines.emplace_back(engine, getNode(partId));
+                }
+            }
+        }
+    } else {
+        for (size_t partId : elem.nodes()) {
+            if (partId < _nodes.size()) {
+                FastS_EngineBase *engine = _dataset->getPartition(dsLock, partId, elem.row());
+                LOG(debug, "searchpath='%s', partId=%ld, row=%ld, dispatchLevel=%u", spec.c_str(), partId, elem.row(), dispatchLevel);
+                if (engine != NULL) {
+                    engines.emplace_back(engine, getNode(partId));
+                }
+            }
+        }
+    }
+    dsLock.Unlock();
+    connectNodes(engines);
+}
+
+void
+FastS_FNET_Search::ConnectDocsumNodes(bool ignoreRow)
+{
+    FastS_assert( ! _nodes.empty());
+    if (_nodesConnected)
+        return;
+
+    bool userow = (_dataset->GetRowBits() > 0) && !ignoreRow;
+
+    EngineNodeMap engines;
+    FastOS_Mutex & dsLock = _dataset->getMutex();
+    dsLock.Lock();
+    for (auto & node : _nodes) {
+        if (node._gdx != NULL) {
+            FastS_EngineBase *engine = node.getPartition(dsLock, userow, _dataset);
+            if (engine != nullptr) {
+                engines.emplace_back(engine, &node);
+            }
+        }
+        for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
+            FastS_FNET_SearchNode *eNode = *iter;
+            if (eNode->_gdx != NULL) {
+                FastS_EngineBase *engine = eNode->getPartition(dsLock, userow, _dataset);
+                if (engine != nullptr) {
+                    engines.emplace_back(engine, eNode);
+                }
+            }
+        }
+    }
+    dsLock.Unlock();
+    connectNodes(engines);
+}
+
+void
+FastS_FNET_Search::EncodePartIDs(uint32_t partid, uint32_t rowid, bool mld,
+                                 FS4Packet_QUERYRESULTX::FS4_hit *pt,
+                                 FS4Packet_QUERYRESULTX::FS4_hit *end)
+{
+    uint32_t rowbits  = _dataset->GetRowBits();
+    uint32_t partbits = _dataset->GetPartBits();
+
+    if (rowbits > 0) {
+        if (mld) {
+            for (; pt < end; pt++) {
+                pt->_partid = ((ENCODE_MLD_PART(pt->_partid) + partid) << rowbits) + rowid;
+            }
+        } else {
+            for (; pt < end; pt++) {
+                pt->_partid = (partid << rowbits) + rowid;
+            }
+        }
+
+    } else { // rowbits == 0
+
+        if (mld) {
+            for (; pt < end; pt++) {
+                pt->_partid = ENCODE_MLD_PART(pt->_partid) + partid;
+            }
+        } else {
+            for (; pt < end; pt++) {
+                pt->_partid = partid;
+            }
+        }
+    }
+}
+
+
+FastS_FNET_Search::FastS_FNET_Search(FastS_DataSetCollection *dsc,
+                                     FastS_FNET_DataSet *dataset,
+				     FastS_TimeKeeper *timeKeeper)
+    : FastS_AsyncSearch(dataset->GetID()),
+      _lock(),
+      _timeKeeper(timeKeeper),
+      _startTime(timeKeeper->GetTime()),
+      _timeout(dataset->GetAppContext()->GetFNETScheduler(), this),
+      _util(),
+      _dsc(dsc),
+      _dataset(dataset),
+      _datasetActiveCostRef(true),
+      _nodes(),
+      _nodesConnected(false),
+      _estParts(0),
+      _estPartCutoff(dataset->GetEstimatePartCutoff()),
+      _FNET_mode(FNET_NONE),
+      _pendingQueries(0),
+      _goodQueries(0),
+      _pendingDocsums(0),
+      _pendingDocsumNodes(0),
+      _requestedDocsums(0),
+      _goodDocsums(0),
+      _queryNodes(0),
+      _queryNodesTimedOut(0),
+      _docsumNodes(0),
+      _docsumNodesTimedOut(0),
+      _docsumsTimedOut(0),
+      _queryTimeout(false),
+      _docsumTimeout(false),
+      _queryStartTime(0.0),
+      _queryMinWait(0.0),
+      _queryMaxWait(0.0),
+      _queryWaitCalculated(false),
+      _adjustedQueryTimeOut(0.0),
+      _docSumStartTime(0.0),
+      _adjustedDocSumTimeOut(0.0),
+      _fixedRow(0),
+      _resbuf()
+{
+    _util.GetQuery().SetDataSet(dataset->GetID());
+    _util.SetStartTime(GetTimeKeeper()->GetTime());
+    AllocNodes();
+}
+
+
+FastS_FNET_Search::~FastS_FNET_Search()
+{
+    _timeout.Kill();
+    _nodes.clear();
+    _util.DropResult();
+    dropDatasetActiveCostRef();
+}
+
+
+void
+FastS_FNET_Search::dropDatasetActiveCostRef(void)
+{
+    if (_datasetActiveCostRef) {
+        _dataset->SubCost();
+        _dataset->ClearActiveQuery(GetTimeKeeper());
+        _datasetActiveCostRef = false;
+    }
+}
+
+
+void
+FastS_FNET_Search::GotQueryResult(FastS_FNET_SearchNode *node,
+                                  FS4Packet_QUERYRESULTX *qrx)
+{
+    if (!BeginFNETWork()) {
+        qrx->Free();
+        return;
+    }
+
+    if (_FNET_mode == FNET_QUERY &&
+        node->_flags._pendingQuery) {
+        FastS_assert(node->_qresult == NULL);
+        node->_qresult = qrx;
+        EncodePartIDs(node->getPartID(), node->GetRowID(),
+                      (qrx->_features & search::fs4transport::QRF_MLD) != 0,
+                      qrx->_hits, qrx->_hits + qrx->_numDocs);
+        LOG(spam, "Got result from row(%d), part(%d) = hits(%d), numDocs(%" PRIu64 ")", node->GetRowID(), node->getPartID(), qrx->_numDocs, qrx->_totNumDocs);
+        node->_flags._pendingQuery = false;
+        _pendingQueries--;
+        _goodQueries++;
+        double tnow = GetTimeKeeper()->GetTime();
+        node->_queryTime = tnow - _startTime;
+        node->GetEngine()->UpdateSearchTime(tnow, node->_queryTime, false);
+        adjustQueryTimeout();
+        node->dropCost();
+    } else {
+        qrx->Free();
+    }
+    EndFNETWork();
+}
+
+void
+FastS_FNET_Search::GotDocsum(FastS_FNET_SearchNode *node,
+                             FS4Packet_DOCSUM *docsum)
+{
+    if (!BeginFNETWork()) {
+        docsum->Free();
+        return;
+    }
+
+    if (_FNET_mode == FNET_DOCSUMS &&
+        node->_pendingDocsums > 0) {
+        LOG(spam, "Got docsum from row(%d), part(%d) = docsumidx(%d)", node->GetRowID(), node->getPartID(), node->_docsum_offsets_idx);
+        uint32_t offset = node->_docsum_offsets[node->_docsum_offsets_idx++];
+        docsum->swapBuf(_resbuf[offset]._buf);
+        node->_pendingDocsums--;
+        _pendingDocsums--;
+        if ( ! _resbuf[offset]._buf.empty())
+            _goodDocsums++;	// Only nonempty docsum is considered good
+        if (node->_pendingDocsums == 0) {
+            node->_docsumTime = (GetTimeKeeper()->GetTime() - _startTime - node->_queryTime);
+            _pendingDocsumNodes--;
+        }
+        adjustDocsumTimeout();
+    }
+    docsum->Free();
+    EndFNETWork();
+}
+
+void
+FastS_FNET_Search::LostSearchNode(FastS_FNET_SearchNode *node)
+{
+    if (!BeginFNETWork())
+        return;
+
+    if (_FNET_mode == FNET_QUERY && node->_flags._pendingQuery) {
+        FastS_assert(_pendingQueries > 0);
+        _pendingQueries--;
+        node->_flags._pendingQuery = false;
+        adjustQueryTimeout();
+        node->dropCost();
+    } else if (_FNET_mode == FNET_DOCSUMS && node->_pendingDocsums > 0) {
+        uint32_t nodePendingDocsums = node->_pendingDocsums;
+        FastS_assert(_pendingDocsums >= nodePendingDocsums);
+        _pendingDocsums -= nodePendingDocsums;
+        node->_pendingDocsums = 0;
+        _pendingDocsumNodes--;
+        adjustDocsumTimeout();
+    }
+    EndFNETWork();
+}
+
+
+void
+FastS_FNET_Search::GotEOL(FastS_FNET_SearchNode *node)
+{
+    if (!BeginFNETWork())
+        return;
+
+    LOG(spam, "Got EOL from row(%d), part(%d) = pendingQ(%d) pendingDocsum(%d)", node->GetRowID(), node->getPartID(), node->_flags._pendingQuery, node->_pendingDocsums);
+    if (_FNET_mode == FNET_QUERY && node->_flags._pendingQuery) {
+        FastS_assert(_pendingQueries > 0);
+        _pendingQueries--;
+        node->_flags._pendingQuery = false;
+        adjustQueryTimeout();
+        node->dropCost();
+    } else if (_FNET_mode == FNET_DOCSUMS && node->_pendingDocsums > 0) {
+        uint32_t nodePendingDocsums = node->_pendingDocsums;
+        FastS_assert(_pendingDocsums >= nodePendingDocsums);
+        _pendingDocsums -= nodePendingDocsums;
+        node->_pendingDocsums = 0;
+        _pendingDocsumNodes--;
+        adjustDocsumTimeout();
+    }
+    EndFNETWork();
+}
+
+
+void
+FastS_FNET_Search::GotError(FastS_FNET_SearchNode *node,
+                            FS4Packet_ERROR *error)
+{
+    if (!BeginFNETWork()) {
+        error->Free();
+        return;
+    }
+
+    LOG(spam,
+        "Got Error from row(%d), part(%d) = pendingQ(%d) pendingDocsum(%d)",
+        node->GetRowID(),
+        node->getPartID(),
+        node->_flags._pendingQuery,
+        node->_pendingDocsums);
+
+    if (_FNET_mode == FNET_QUERY && node->_flags._pendingQuery) {
+        FastS_assert(_pendingQueries > 0);
+        _pendingQueries--;
+        node->_flags._pendingQuery = false;
+        if (error->_errorCode == search::engine::ECODE_TIMEOUT) {
+            node->_flags._queryTimeout = true;
+            _queryNodesTimedOut++;
+        }
+        adjustQueryTimeout();
+    } else if (_FNET_mode == FNET_DOCSUMS && node->_pendingDocsums > 0) {
+        uint32_t nodePendingDocsums = node->_pendingDocsums;
+        FastS_assert(_pendingDocsums >= nodePendingDocsums);
+        _pendingDocsums -= nodePendingDocsums;
+        node->_pendingDocsums = 0;
+        _pendingDocsumNodes--;
+        if (error->_errorCode == search::engine::ECODE_TIMEOUT) {
+            node->_flags._docsumTimeout = true;
+            _docsumNodesTimedOut++;
+            _docsumsTimedOut += nodePendingDocsums;
+        }
+        adjustDocsumTimeout();
+    }
+    error->Free();
+    EndFNETWork();
+}
+
+
+void
+FastS_FNET_Search::HandleTimeout()
+{
+    if (!BeginFNETWork())
+        return;
+
+    if (_FNET_mode == FNET_QUERY) {
+        for (FastS_FNET_SearchNode & node : _nodes) {
+            if (node._flags._pendingQuery) {
+                FastS_assert(_pendingQueries > 0);
+                _pendingQueries--;
+                node._flags._pendingQuery = false;
+                node._flags._queryTimeout = true;
+                _queryNodesTimedOut++;
+                double tnow = GetTimeKeeper()->GetTime();
+                node._queryTime = tnow - _startTime;
+                node.GetEngine()->UpdateSearchTime(tnow, node._queryTime, true);
+            }
+        }
+        _queryTimeout = true;
+    } else if (_FNET_mode == FNET_DOCSUMS) {
+        for (FastS_FNET_SearchNode & node : _nodes) {
+            if (node._pendingDocsums > 0) {
+                uint32_t nodePendingDocsums = node._pendingDocsums;
+                FastS_assert(_pendingDocsums >= nodePendingDocsums);
+                _pendingDocsums -= nodePendingDocsums;
+                _docsumsTimedOut += nodePendingDocsums;
+                node._pendingDocsums = 0;
+                node._flags._docsumTimeout = true;
+                _docsumNodesTimedOut++;
+                _pendingDocsumNodes--;
+            }
+            for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
+                FastS_FNET_SearchNode *eNode = *iter;
+                if (eNode->_pendingDocsums > 0) {
+                    uint32_t nodePendingDocsums = eNode->_pendingDocsums;
+                    FastS_assert(_pendingDocsums >= nodePendingDocsums);
+                    _pendingDocsums -= nodePendingDocsums;
+                    _docsumsTimedOut += nodePendingDocsums;
+                    eNode->_pendingDocsums = 0;
+                    eNode->_flags._docsumTimeout = true;
+                    _docsumNodesTimedOut++;
+                    _pendingDocsumNodes--;
+                }
+            }
+        }
+        _docsumTimeout = true;
+    }
+    EndFNETWork();
+}
+
+bool
+FastS_FNET_Search::BeginFNETWork()
+{
+    Lock();
+    if (_FNET_mode != FNET_NONE)
+        return true;
+    Unlock();
+    return false;
+}
+
+void
+FastS_FNET_Search::EndFNETWork()
+{
+    if (_FNET_mode == FNET_QUERY && _pendingQueries == 0) {
+        _FNET_mode = FNET_NONE;
+        Unlock();
+        _searchOwner->DoneQuery(this, _searchContext);
+    } else if (_FNET_mode == FNET_DOCSUMS && _pendingDocsums == 0) {
+        _FNET_mode = FNET_NONE;
+        Unlock();
+        _searchOwner->DoneDocsums(this, _searchContext);
+    } else {
+        Unlock();
+    }
+}
+
+bool
+FastS_FNET_Search::ShouldLimitHitsPerNode() const
+{
+    return (_util.GetAlignedMaxHits() > _dataset->GetMaxHitsPerNode());
+}
+
+
+void
+FastS_FNET_Search::MergeHits()
+{
+    FastS_HitMerger<FastS_FNETMerge> merger(this);
+    merger.MergeHits();
+
+    if (_util.IsEstimate())
+        return;
+
+    if (ShouldLimitHitsPerNode())
+        _dataset->UpdateMaxHitsPerNodeLog(merger.WasIncomplete(), merger.WasFuzzy());
+
+    if (!_queryArgs->groupSpec.empty()) {
+        _groupMerger.reset(new search::grouping::MergingManager(_dataset->GetPartBits(), _dataset->GetRowBits()));
+        for (const FastS_FNET_SearchNode & node : _nodes) {
+            if (node._qresult != NULL) {
+                _groupMerger->addResult(node.getPartID(), node.GetRowID(),
+                                        ((node._qresult->_features & search::fs4transport::QRF_MLD) != 0),
+                                        node._qresult->_groupData, node._qresult->_groupDataLen);
+            }
+        }
+        _groupMerger->merge();
+        _util.SetGroupResultLen(_groupMerger->getGroupResultLen());
+        _util.SetGroupResult(_groupMerger->getGroupResult());
+    }
+}
+
+void
+FastS_FNET_Search::CheckCoverage()
+{
+    uint64_t covDocs  = 0;
+    uint64_t activeDocs  = 0;
+    size_t cntNone(0);
+
+    for (const FastS_FNET_SearchNode & node : _nodes) {
+        if (node._qresult != NULL) {
+            covDocs  += node._qresult->_coverageDocs;
+            activeDocs  += node._qresult->_activeDocs;
+        } else {
+            cntNone++;
+        }
+    }
+    if ((cntNone > 0) && (cntNone != _nodes.size())) {
+        activeDocs += cntNone * activeDocs/(_nodes.size() - cntNone);
+    }
+    _util.SetCoverage(covDocs, activeDocs);
+}
+
+
+void
+FastS_FNET_Search::CheckQueryTimes()
+{
+    double   factor    = _dataset->GetSlowQueryLimitFactor();
+    double   bias      = _dataset->GetSlowQueryLimitBias();
+    double   queryTime = 0.0;
+    int      queryCnt  = 0;
+
+    for (const FastS_FNET_SearchNode & node : _nodes) {
+        if (node.IsConnected() && node._queryTime > 0.0) {
+            queryTime += node._queryTime;
+            queryCnt++;
+        }
+    }
+
+    if (queryCnt == 0)
+        return;
+
+    queryTime = queryTime / (double)queryCnt;
+    double maxQueryTime = queryTime * factor + bias;
+
+    for (const FastS_FNET_SearchNode & node : _nodes) {
+        if (node.IsConnected() && node._queryTime > maxQueryTime) {
+            node.GetEngine()->SlowQuery(maxQueryTime, node._queryTime - maxQueryTime, false);
+        }
+    }
+}
+
+
+void
+FastS_FNET_Search::CheckDocsumTimes()
+{
+    double   factor     = _dataset->GetSlowDocsumLimitFactor();
+    double   bias       = _dataset->GetSlowDocsumLimitBias();
+    double   docsumTime = 0.0;
+    int      docsumCnt  = 0;
+
+    for (const FastS_FNET_SearchNode & node : _nodes) {
+        if (node.IsConnected() && node._docsumTime > 0.0) {
+            docsumTime += node._docsumTime;
+            docsumCnt++;
+        }
+    }
+    if (docsumCnt == 0)
+        return;
+    docsumTime = docsumTime / (double)docsumCnt;
+    double maxDocsumTime = docsumTime * factor + bias;
+
+    for (const FastS_FNET_SearchNode & node : _nodes) {
+        if (node.IsConnected() && node._docsumTime > maxDocsumTime) {
+            node.GetEngine()->SlowDocsum(maxDocsumTime, node._docsumTime - maxDocsumTime);
+        }
+        for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
+            FastS_FNET_SearchNode *eNode = *iter;
+            if (eNode->IsConnected() && eNode->_docsumTime > maxDocsumTime) {
+                eNode->GetEngine()->SlowDocsum(maxDocsumTime, eNode->_docsumTime - maxDocsumTime);
+            }
+        }
+    }
+}
+
+
+void
+FastS_FNET_Search::CheckQueryTimeout()
+{
+    if (_queryNodes != 0 && _queryNodesTimedOut >= _queryNodes)
+        SetError(search::engine::ECODE_TIMEOUT, NULL);
+    if (!_queryTimeout)
+        return;
+
+    vespalib::string nodeList;
+    uint32_t nodeCnt = 0;
+    uint32_t printNodes = 10;
+    for (const FastS_FNET_SearchNode & node : _nodes) {
+        if (node._flags._queryTimeout) {
+            if (nodeCnt < printNodes) {
+                if (nodeCnt > 0) {
+                    nodeList.append(", ");
+                }
+                nodeList.append(node.GetEngine()->GetName());
+            }
+            ++nodeCnt;
+        }
+    }
+    if (nodeCnt > printNodes) {
+        nodeList.append(", ...");
+    }
+    vespalib::string query = _util.GetQuery().getPrintableQuery();
+    LOG(warning, "%u nodes(%s) timed out during query execution (%s)",
+        nodeCnt, nodeList.c_str(), query.c_str());
+}
+
+
+void
+FastS_FNET_Search::CheckDocsumTimeout()
+{
+    if (_docsumNodes != 0 && _docsumNodesTimedOut >= _docsumNodes)
+        SetError(search::engine::ECODE_TIMEOUT, NULL);
+    if (!_docsumTimeout)
+        return;
+
+    vespalib::string nodeList;
+    uint32_t nodeCnt = 0;
+    uint32_t printNodes = 10;
+    for (const FastS_FNET_SearchNode & node : _nodes) {
+        if (node._flags._docsumTimeout) {
+            if (nodeCnt < printNodes) {
+                if (nodeCnt > 0) {
+                    nodeList.append(", ");
+                }
+                nodeList.append(node.GetEngine()->GetName());
+            }
+            ++nodeCnt;
+        }
+        for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
+            FastS_FNET_SearchNode *eNode = *iter;
+            if (eNode->_flags._docsumTimeout) {
+                if (nodeCnt < printNodes) {
+                    if (nodeCnt > 0) {
+                        nodeList.append(", ");
+                    }
+                    nodeList.append(eNode->GetEngine()->GetName());
+                }
+                ++nodeCnt;
+            }
+        }
+    }
+    if (nodeCnt > printNodes) {
+        nodeList.append(", ...");
+    }
+    double elapsed = GetTimeKeeper()->GetTime() - _docSumStartTime;
+    LOG(warning, "%u nodes given %1.6f seconds timeout timed out during docsum fetching after %1.6f seconds (%s)",
+        nodeCnt, _adjustedDocSumTimeOut, elapsed, nodeList.c_str());
+}
+
+
+FastS_ISearch::RetCode
+FastS_FNET_Search::Search(uint32_t searchOffset,
+                          uint32_t maxhits, uint32_t minhits)
+{
+    // minhits is never sent down from dispatch...
+    (void) minhits; // ignore
+
+    _util.setSearchRequest(_queryArgs);
+    _util.SetupQuery(maxhits, searchOffset);
+    if (_util.IsEstimate())
+        _util.InitEstimateMode();
+    _util.AdjustSearchParameters(_nodes.size());
+    _util.AdjustSearchParametersFinal(_nodes.size());
+
+    vespalib::string searchPath;
+    const search::fef::Properties & model = _queryArgs->propertiesMap.modelOverrides();
+    search::fef::Property searchPathProperty = model.lookup("searchpath");
+    if (searchPathProperty.found()) {
+        searchPath = searchPathProperty.get();
+    }
+    _adjustedQueryTimeOut = static_cast<double>(_queryArgs->getTimeLeft().ms()) / 1000.0;
+    if ( ! searchPath.empty()) {
+        connectSearchPath(searchPath);
+    } else if (_util.IsEstimate()) {
+        ConnectEstimateNodes();
+    } else {
+        ConnectQueryNodes();
+    }
+
+    // we support error packets
+    uint32_t qflags = _util.GetQuery().GetQueryFlags() | search::fs4transport::QFLAG_ALLOW_ERRORPACKET;
+
+    // propagate drop-sortdata flag only if we have single sub-node
+    if (_nodes.size() != 1)
+        qflags &= ~search::fs4transport::QFLAG_DROP_SORTDATA;
+
+    uint32_t hitsPerNode = ShouldLimitHitsPerNode()
+                           ? _dataset->GetMaxHitsPerNode()
+                           : _util.GetAlignedMaxHits();
+
+    // set up expected _queryNodes, _pendingQueries and node->_flags._pendingQuery state
+    for (FastS_FNET_SearchNode & node : _nodes) {
+        if (node.IsConnected()) {
+            node._flags._pendingQuery = true;
+            _pendingQueries++;
+            _queryNodes++;
+        }
+    }
+    size_t num_send_ok = 0; // number of partitions where packet send succeeded
+    std::vector<uint32_t> send_failed; // partitions where packet send failed
+
+    // allow FNET responses while requests are being sent
+    Lock();
+    ++_pendingQueries; // add Elephant query node to avoid early query done
+    ++_queryNodes;     // add Elephant query node to avoid early query done
+    _FNET_mode = FNET_QUERY;
+    _queryStartTime = GetTimeKeeper()->GetTime();
+    _timeout.Schedule(_adjustedQueryTimeOut);
+    Unlock();
+    FNET_Packet::SP shared(new FS4Packet_PreSerialized(*setupQueryPacket(hitsPerNode, qflags, _queryArgs->propertiesMap)));
+    for (uint32_t i = 0; i < _nodes.size(); i++) {
+        FastS_FNET_SearchNode & node = _nodes[i];
+        if (node.IsConnected()) {
+            FNET_Packet::UP qx(new FS4Packet_Shared(shared));
+            LOG(spam, "posting packet to node %d='%s'\npacket=%s", i, node.toString().c_str(), qx->Print(0).c_str());
+            if (node.PostPacket(qx.release())) {
+                ++num_send_ok;
+            } else {
+                send_failed.push_back(i);
+                LOG(debug, "FAILED posting packet to node %d='%s'\npacket=%s", i, node.toString().c_str(), qx->Print(0).c_str());
+            }
+        }
+    }
+
+    // finalize setup and check if query is still in progress
+    Lock();
+    assert(_queryNodes >= _pendingQueries);
+    for (uint32_t i: send_failed) {
+        // conditional revert of state for failed nodes
+        if (_nodes[i]._flags._pendingQuery) {
+            _nodes[i]._flags._pendingQuery = false;
+            assert(_pendingQueries > 0);
+            --_pendingQueries;
+            --_queryNodes;
+        }
+    }
+    // revert Elephant query node to allow search to complete
+    assert(_pendingQueries > 0);
+    --_pendingQueries;
+    --_queryNodes;
+    bool done = (_pendingQueries == 0);
+    bool all_down = (num_send_ok == 0);
+    if (done) {
+        _FNET_mode = FNET_NONE;
+        if (all_down) {
+            SetError(search::engine::ECODE_ALL_PARTITIONS_DOWN, NULL);
+        }
+    }
+    Unlock();
+
+    return (done) ? RET_OK : RET_INPROGRESS;
+}
+
+vespalib::string
+FastS_FNET_SearchNode::toString() const
+{
+    vespalib::string s;
+    s += vespalib::make_string("{ channel=%p={%d, c=%p='%s'}, partId = %d, rowid=%d }",
+                               _channel, _channel->GetID(),
+                               _channel->GetConnection(), _channel->GetConnection()->GetSpec(),
+                               _partid, _rowid);
+    return s;
+}
+
+
+FNET_Packet::UP
+FastS_FNET_Search::setupQueryPacket(uint32_t hitsPerNode, uint32_t qflags,
+                                    const search::engine::PropertiesMap &properties)
+{
+    FNET_Packet::UP ret(new FS4Packet_QUERYX(search::fs4transport::PCODE_QUERYX));
+    FS4Packet_QUERYX & qx = static_cast<FS4Packet_QUERYX &>(*ret);
+    qx._features      = search::fs4transport::QF_PARSEDQUERY | search::fs4transport::QF_RANKP;
+    qx._offset        = _util.GetAlignedSearchOffset();
+    qx._maxhits       = hitsPerNode; // capped maxhits
+    qx._qflags        = qflags; // filtered query flags
+    qx.setTimeout(_queryArgs->getTimeLeft());
+
+    qx.setRanking(_queryArgs->ranking);
+
+    if (!_queryArgs->sortSpec.empty()) {
+        qx._features |= search::fs4transport::QF_SORTSPEC;
+        qx.setSortSpec(_queryArgs->sortSpec);
+    }
+
+    if (!_queryArgs->groupSpec.empty()) {
+        qx._features |= search::fs4transport::QF_GROUPSPEC;
+        qx.setGroupSpec(vespalib::stringref(&_queryArgs->groupSpec[0], _queryArgs->groupSpec.size()));
+    }
+
+    if (!_queryArgs->sessionId.empty()) {
+        qx._features |= search::fs4transport::QF_SESSIONID;
+        qx.setSessionId(vespalib::stringref(&_queryArgs->sessionId[0], _queryArgs->sessionId.size()));
+    }
+
+    if (!_queryArgs->location.empty()) {
+        qx._features |= search::fs4transport::QF_LOCATION;
+        qx.setLocation(_queryArgs->location);
+    }
+
+    if (properties.size() > 0) {
+        PacketConverter::fillPacketProperties(properties, qx._propsVector);
+        qx._features |= search::fs4transport::QF_PROPERTIES;
+    }
+
+    qx._numStackItems = _queryArgs->stackItems;
+    qx.setStackDump(_queryArgs->getStackRef());
+    qx.UpdateCompatPCODE();
+    return ret;
+}
+
+
+FastS_ISearch::RetCode
+FastS_FNET_Search::ProcessQueryDone()
+{
+    if (_util.IsQueryFlagSet(search::fs4transport::QFLAG_REPORT_COVERAGE))
+        CheckCoverage();
+
+    if (_errorCode == search::engine::ECODE_NO_ERROR) {
+        MergeHits();
+    }
+    memcpy(&_queryResult, _util.GetQueryResult(), sizeof(FastS_QueryResult));
+    double tnow = GetTimeKeeper()->GetTime();
+    _queryResult._queryResultTime = tnow - _startTime;
+    if (_errorCode == search::engine::ECODE_NO_ERROR) {
+        if (_util.IsEstimate()) {
+            _dataset->UpdateEstimateCount();
+        } else {
+            _dataset->UpdateSearchTime(tnow, _queryResult._queryResultTime, _queryNodesTimedOut != 0);
+        }
+        if ( _dataset->useFixedRowDistribution() ) {
+            _dataset->updateSearchTime(_queryResult._queryResultTime, _fixedRow);
+        }
+    }
+    CheckQueryTimes();
+    CheckQueryTimeout();
+    dropDatasetActiveCostRef();
+    return RET_OK;
+}
+
+
+FastS_ISearch::RetCode
+FastS_FNET_Search::GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt)
+{
+    if (hitcnt > 0) {
+        _resbuf.resize(hitcnt);
+    }
+
+    // copy values from query result
+
+    uint32_t i;
+    for (i = 0; i < hitcnt; i++) {
+        _resbuf[i]._docid     = 0;
+        _resbuf[i]._gid       = hits[i]._gid;
+        _resbuf[i]._metric    = hits[i]._metric;
+        _resbuf[i]._partition = hits[i]._partition;
+    }
+
+    // determine docsum distribution among nodes
+
+    const FastS_hitresult *p = hits;
+    uint32_t rowbits = _dataset->GetRowBits();
+    uint32_t partbits = _dataset->GetPartBits();
+    uint32_t mldpartidmask = (1 << partbits) - 1;
+    bool ignoreRow = (_docsumArgs->getFlags() &
+                      search::fs4transport::GDFLAG_IGNORE_ROW) != 0;
+    if (rowbits > 0) {
+        uint32_t rowmask = (1 << rowbits) - 1;
+        for (i = 0; i < hitcnt; i++, p++) {
+            FastS_FNET_SearchNode *node;
+            uint32_t partid0 = p->_partition >> rowbits;
+            uint32_t row = ignoreRow ? 0u : p->_partition & rowmask;
+            if (IS_MLD_PART(partid0)) {
+                uint32_t partid = MLD_PART_TO_PARTID(partid0);
+                if (partid < _nodes.size()) {
+                    node = getNode(partid);
+                    if (node->_docidCnt == 0) {
+                        node->_flags._docsumMld = true;// Only accept MLD from now on
+                        node->_docsumRow = row;
+                    } else if (!node->_flags._docsumMld || row != node->_docsumRow) {
+                        if (_nodesConnected)
+                            continue;			// Drop (inconsistent)
+                        node = node->allocExtraDocsumNode(true, row, rowbits);
+                    }
+                    node->_docidCnt++;
+                }
+            } else { // !MLD
+                if (partid0 < _nodes.size()) {
+                    node = getNode(partid0);
+                    if (node->_docidCnt == 0) {
+                        node->_docsumRow = row;
+                    } else if (node->_flags._docsumMld || row != node->_docsumRow) {
+                        if (_nodesConnected)
+                            continue;			// Drop (inconsistent)
+                        node = node->allocExtraDocsumNode(false, row, rowbits);
+                    }
+                    node->_docidCnt++;
+                }
+            }
+        }
+    } else { // rowbits == 0
+        for (i = 0; i < hitcnt; i++, p++) {
+            FastS_FNET_SearchNode *node;
+            if (IS_MLD_PART(p->_partition)) {
+                uint32_t partid = MLD_PART_TO_PARTID(p->_partition);
+                if (partid < _nodes.size()) {
+                    node = getNode(partid);
+                    if (node->_docidCnt == 0) {
+                        node->_flags._docsumMld = true;// Only accept MLD from now on
+                    } else if (!node->_flags._docsumMld) {
+                        if (_nodesConnected)
+                            continue;			// Drop (inconsistent)
+                        node = node->allocExtraDocsumNode(true, 0, 0);
+                    }
+                    node->_docidCnt++;
+                }
+            } else { // !MLD
+                if (p->_partition < _nodes.size()) {
+                    node = getNode(p->_partition);
+                    if (node->_docidCnt == 0) {
+                    } else if (node->_flags._docsumMld) {
+                        if (_nodesConnected)
+                            continue;			// Drop (inconsistent)
+                        node = node->allocExtraDocsumNode(false, 0, 0);
+                    }
+                    node->_docidCnt++;
+                }
+            }
+        }
+    }
+    FastS_assert(p == hits + hitcnt);
+
+    // allocate docsum requests and insert features
+
+    search::docsummary::GetDocsumArgs *args = _docsumArgs;
+    for (FastS_FNET_SearchNode & node : _nodes) {
+        if (node._docidCnt != 0) {
+            node.allocGDX(args, args->propertiesMap());
+        }
+        for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
+            FastS_FNET_SearchNode *eNode = *iter;
+            if (eNode->_docidCnt != 0)
+                eNode->allocGDX(args, args->propertiesMap());
+        }
+    }
+
+    // fill docid(/partid/stamp) data into docsum requests
+
+    p = hits;
+    if (rowbits > 0) {
+        uint32_t rowmask = (1 << rowbits) - 1;
+        for (i = 0; i < hitcnt; i++, p++) {
+            FastS_FNET_SearchNode *node;
+            uint32_t partid0 = p->_partition >> rowbits;
+            uint32_t row = ignoreRow ? 0u : p->_partition & rowmask;
+            if (IS_MLD_PART(partid0)) {
+                uint32_t partid = MLD_PART_TO_PARTID(partid0);
+                if (partid < _nodes.size()) {
+                    node = getNode(partid);
+                    if (!node->_flags._docsumMld || row != node->_docsumRow) {
+                        if (_nodesConnected)
+                            continue;			// Drop (inconsistent)
+                        node = node->allocExtraDocsumNode(true, row, rowbits);
+                    }
+
+                    FS4Packet_GETDOCSUMSX::FS4_docid &q = node->_gdx->_docid[node->_docsum_offsets_idx];
+                    q._gid      = p->_gid;
+                    q._partid   = DECODE_MLD_PART(partid0);
+                    node->_docsum_offsets[node->_docsum_offsets_idx++] = i;
+                }
+            } else { // !MLD
+                if (partid0 < _nodes.size()) {
+                    node = getNode(partid0);
+                    if (node->_flags._docsumMld || row != node->_docsumRow) {
+                        if (_nodesConnected)
+                            continue;			// Drop (inconsistent)
+                        node = node->allocExtraDocsumNode(false, row, rowbits);
+                    }
+
+                    FS4Packet_GETDOCSUMSX::FS4_docid &q = node->_gdx->_docid[node->_docsum_offsets_idx];
+                    q._gid   = p->_gid;
+                    node->_docsum_offsets[node->_docsum_offsets_idx++] = i;
+                }
+            }
+        }
+    } else { // rowbits == 0
+        for (i = 0; i < hitcnt; i++, p++) {
+            FastS_FNET_SearchNode *node;
+            if (IS_MLD_PART(p->_partition)) {
+                uint32_t partid = MLD_PART_TO_PARTID(p->_partition);
+                if (partid < _nodes.size()) {
+                    node = getNode(partid);
+                    if (!node->_flags._docsumMld) {
+                        if (_nodesConnected)
+                            continue;			// Drop (inconsistent)
+                        node = node->allocExtraDocsumNode(true, 0, 0);
+                    }
+
+                    FS4Packet_GETDOCSUMSX::FS4_docid &q = node->_gdx->_docid[node->_docsum_offsets_idx];
+                    q._gid      = p->_gid;
+                    q._partid   = DECODE_MLD_PART(p->_partition);
+                    node->_docsum_offsets[node->_docsum_offsets_idx++] = i;
+                }
+            } else { // !MLD
+                if (p->_partition < _nodes.size()) {
+                    node = getNode(p->_partition);
+                    if (node->_flags._docsumMld) {
+                        if (_nodesConnected)
+                            continue;			// Drop (inconsistent)
+                        node = node->allocExtraDocsumNode(false, 0, 0);
+                    }
+
+                    FS4Packet_GETDOCSUMSX::FS4_docid &q = node->_gdx->_docid[node->_docsum_offsets_idx];
+                    q._gid   = p->_gid;
+                    node->_docsum_offsets[node->_docsum_offsets_idx++] = i;
+                }
+            }
+        }
+    }
+    FastS_assert(p == hits + hitcnt);
+
+    ConnectDocsumNodes(ignoreRow);
+    Lock();
+
+    // patch in engine dependent features and send docsum requests
+
+    for (FastS_FNET_SearchNode & node : _nodes) {
+        if (node._gdx != NULL)
+            node.postGDX(&_pendingDocsums, &_docsumNodes);
+        for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
+            FastS_FNET_SearchNode *eNode = *iter;
+            if (eNode->_gdx != NULL)
+                eNode->postGDX(&_pendingDocsums, &_docsumNodes);
+        }
+    }
+    _pendingDocsumNodes = _docsumNodes;
+    _requestedDocsums = _pendingDocsums;
+
+    bool done = (_pendingDocsums == 0);
+    if (!done) {
+        _FNET_mode = FNET_DOCSUMS; // FNET; do your thing
+
+        _adjustedDocSumTimeOut = args->getTimeout().sec();
+        _docSumStartTime = GetTimeKeeper()->GetTime();
+        _timeout.Schedule(_adjustedDocSumTimeOut);
+    }
+    Unlock();
+
+    return (done) ? RET_OK : RET_INPROGRESS;
+}
+
+
+FastS_ISearch::RetCode
+FastS_FNET_Search::ProcessDocsumsDone()
+{
+    _docsumsResult._fullresult      = &_resbuf[0];
+    _docsumsResult._fullResultCount = _resbuf.size();
+    _docsumsResult._queryDocSumTime = GetTimeKeeper()->GetTime() - _startTime;
+    CheckDocsumTimes();
+    CheckDocsumTimeout();
+    dropDatasetActiveCostRef();
+    return RET_OK;
+}
+
+
+void
+FastS_FNET_Search::adjustQueryTimeout()
+{
+    uint32_t pendingQueries = getPendingQueries();
+
+    if (pendingQueries == 0 || _util.IsQueryFlagSet(search::fs4transport::QFLAG_DUMP_FEATURES)) {
+        return;
+    }
+
+    double mincoverage = _dataset->getMinimalSearchCoverage();
+    uint32_t wantedAnswers = getRequestedQueries();
+    if (mincoverage < 100.0) {
+        wantedAnswers *= mincoverage / 100.0;
+        LOG(spam, "Adjusting wanted answers from %u to %u", getRequestedQueries(), wantedAnswers);
+    }
+    if (getDoneQueries() < wantedAnswers) {
+        return;
+    }
+    if (!_queryWaitCalculated) {
+        double timeLeft = _queryArgs->getTimeLeft().sec();
+        _queryMinWait = timeLeft * _dataset->getHigherCoverageMinSearchWait();
+        _queryMaxWait = timeLeft * _dataset->getHigherCoverageMaxSearchWait();
+        _queryWaitCalculated = true;
+    }
+
+    double basewait = 0.0;
+    double minwait = _queryMinWait;
+    double maxwait = _queryMaxWait;
+
+    double elapsed = GetTimeKeeper()->GetTime() - _queryStartTime;
+
+    double missWidth = ((100.0 - mincoverage) * getRequestedQueries()) / 100.0 - 1.0;
+
+    double slopedwait = minwait;
+
+    if (pendingQueries > 1 && missWidth > 0.0)
+        slopedwait += ((maxwait - minwait) * (pendingQueries - 1)) / missWidth;
+
+    double newTimeOut = std::max(elapsed, basewait) + slopedwait;
+
+
+    if (newTimeOut >= _adjustedQueryTimeOut)
+        return;
+
+    _adjustedQueryTimeOut = newTimeOut;
+    if (newTimeOut > elapsed)
+        _timeout.Schedule(newTimeOut - elapsed);
+    else
+        _timeout.ScheduleNow();
+}
+
+
+void
+FastS_FNET_Search::adjustDocsumTimeout(void)
+{
+    uint32_t pendingDocsums = getPendingDocsums();
+
+    if (pendingDocsums == 0 || _util.IsQueryFlagSet(search::fs4transport::QFLAG_DUMP_FEATURES)) {
+        return;
+    }
+
+    double coverage = static_cast<double>(getDoneDocsums() * 100) / getRequestedDocsums();
+
+    double mincoverage = _dataset->getMinimalDocSumCoverage();
+
+    if (coverage < mincoverage)
+        return;
+
+    double basewait = _dataset->getHigherCoverageBaseDocSumWait();
+    double minwait = _dataset->getHigherCoverageMinDocSumWait();
+    double maxwait = _dataset->getHigherCoverageMaxDocSumWait();
+
+    double elapsed = GetTimeKeeper()->GetTime() - _docSumStartTime;
+
+    double missWidth = ((100.0 - mincoverage) * getRequestedDocsums()) / 100.0 - 1.0;
+
+    double slopedwait = minwait;
+
+    if (pendingDocsums > 1 && missWidth > 0.0)
+        slopedwait += ((maxwait - minwait) * (pendingDocsums - 1)) / missWidth;
+
+    double newTimeOut = std::max(elapsed, basewait) + slopedwait;
+
+    if (newTimeOut >= _adjustedDocSumTimeOut)
+        return;
+
+    _adjustedDocSumTimeOut = newTimeOut;
+    if (newTimeOut > elapsed)
+        _timeout.Schedule(newTimeOut - elapsed);
+    else
+        _timeout.ScheduleNow();
+}
+
+
+FastS_Sync_FNET_Search::~FastS_Sync_FNET_Search() {}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h
new file mode 100644
index 00000000000..e87ce2a5670
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h
@@ -0,0 +1,394 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/fnet/fnet.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+#include <vespa/searchlib/common/sortdata.h>
+#include <vespa/searchcore/grouping/mergingmanager.h>
+#include <vespa/searchcore/fdispatch/search/search_path.h>
+#include <vespa/searchcore/fdispatch/search/querycacheutil.h>
+#include <vespa/searchcore/fdispatch/search/fnet_engine.h>
+
+class FastS_FNET_Engine;
+class FastS_FNET_Search;
+
+using search::fs4transport::FS4Packet_QUERYRESULTX;
+using search::fs4transport::FS4Packet_GETDOCSUMSX;
+using search::fs4transport::FS4Packet_DOCSUM;
+using search::fs4transport::FS4Packet_TRACEREPLY;
+
+//-----------------------------------------------------------------
+
+class FastS_FNET_SearchNode : public FNET_IPacketHandler
+{
+public:
+    class ExtraDocsumNodesIter;
+    typedef std::unique_ptr<FastS_FNET_SearchNode> UP;
+private:
+    friend class ExtraDocsumNodesIter;
+
+    FastS_FNET_Search       *_search;   // we are part of this search
+    FastS_FNET_Engine       *_engine;   // we use this search engine
+    FNET_Channel            *_channel;  // connection with search engine
+    uint32_t                 _subds;    // engine sub dataset
+    uint32_t                 _partid;   // engine partition id
+    uint32_t                 _rowid;    // engine row id
+    uint32_t                 _stamp;    // engine timestamp
+
+public:
+
+    FS4Packet_QUERYRESULTX *_qresult; // query result packet
+    double                  _queryTime;
+    struct Flags {
+        Flags() :
+            _pendingQuery(false),
+            _docsumMld(false),
+            _queryTimeout(false),
+            _docsumTimeout(false),
+            _needSubCost(false)
+        { }
+        bool  _pendingQuery;   // is query pending ?
+        bool  _docsumMld;
+        bool  _queryTimeout;
+        bool  _docsumTimeout;
+        bool  _needSubCost;
+    };
+
+    Flags       _flags;
+
+// Docsum related stuff.
+    uint32_t    _docidCnt;
+    uint32_t    _pendingDocsums; // how many docsums pending ?
+    uint32_t    _docsumRow;
+    uint32_t    _docsumStamp;
+    uint32_t    _docsum_offsets_idx;
+    double      _docsumTime;
+
+    FS4Packet_GETDOCSUMSX  *_gdx;
+    std::vector<uint32_t>   _docsum_offsets;
+private:
+    std::vector<FastS_FNET_SearchNode::UP> _extraDocsumNodes;
+    FastS_FNET_SearchNode *_nextExtraDocsumNode;
+    FastS_FNET_SearchNode *_prevExtraDocsumNode;
+public:
+
+// Query processing stuff.
+    FS4Packet_QUERYRESULTX::FS4_hit *_hit_beg; // hit array start
+    FS4Packet_QUERYRESULTX::FS4_hit *_hit_cur; // current hit
+    FS4Packet_QUERYRESULTX::FS4_hit *_hit_end; // end boundary
+
+    search::common::SortDataIterator _sortDataIterator;
+
+public:
+    FastS_FNET_SearchNode(FastS_FNET_Search *search, uint32_t partid);
+    // These objects are referenced everywhere and must never be either copied nor moved,
+    // but std::vector requires this to exist. If called it will assert.
+    FastS_FNET_SearchNode(FastS_FNET_SearchNode && rhs);
+    FastS_FNET_SearchNode(const FastS_FNET_SearchNode &) = delete;
+    FastS_FNET_SearchNode& operator=(const FastS_FNET_SearchNode &) = delete;
+
+    virtual ~FastS_FNET_SearchNode();
+
+    // Methods needed by mergehits
+    bool NT_InitMerge(uint32_t *numDocs, uint64_t *totalHits, search::HitRank *maxRank, uint32_t *sortDataDocs);
+    search::common::SortDataIterator *NT_GetSortDataIterator() { return &_sortDataIterator; }
+    FS4Packet_QUERYRESULTX::FS4_hit *NT_GetHit() const { return _hit_cur; }
+    uint32_t NT_GetNumHitsUsed() const { return (_hit_cur - _hit_beg); }
+    uint32_t NT_GetNumHitsLeft() const { return (_hit_end - _hit_cur); }
+    uint64_t NT_GetTotalHits() const { return (_qresult != NULL) ? _qresult->_totNumDocs : 0; }
+    uint32_t NT_GetNumHits() const { return (_hit_end - _hit_beg); }
+    void NT_NextHit() { _hit_cur++; }
+
+    uint32_t getPartID(void) const     { return _partid; }
+    uint32_t GetRowID(void) const     { return _rowid; }
+    uint32_t GetTimeStamp(void) const { return _stamp; }
+
+    FastS_FNET_SearchNode * allocExtraDocsumNode(bool mld, uint32_t rowid, uint32_t rowbits);
+
+    FastS_FNET_Engine *GetEngine(void) const { return _engine; }
+
+    bool IsConnected(void) const { return _channel != NULL; }
+    void Connect(FastS_FNET_Engine *engine);
+    void Connect_HasDSLock(FastS_FNET_Engine *engine);
+    FastS_EngineBase * getPartition(const FastOS_Mutex & dsMutex, bool userow, FastS_FNET_DataSet *dataset);
+    void allocGDX(search::docsummary::GetDocsumArgs *args, const search::engine::PropertiesMap &properties);
+    void postGDX(uint32_t *pendingDocsums, uint32_t *pendingDocsumNodes);
+    vespalib::string toString() const;
+
+    const char *getHostName() const {
+        return (_engine == NULL ? "localhost" : _engine->getHostName());
+    }
+    int getPortNumber() const {
+        return (_engine == NULL ? 0 : _engine->getPortNumber());
+    }
+
+    void dropCost(void) {
+        if (_engine != NULL && _flags._needSubCost) {
+            _engine->SubCost();
+            _flags._needSubCost = false;
+        }
+    }
+
+
+    void Disconnect()
+    {
+        if (_channel != NULL) {
+            _channel->CloseAndFree();
+            _channel = NULL;
+        }
+        if (_engine != NULL) {
+            if (_flags._needSubCost) {
+                _engine->SubCost();
+                _flags._needSubCost = false;
+            }
+            _engine = NULL;
+        }
+    }
+
+
+    bool PostPacket(FNET_Packet *packet) {
+        return (_channel == NULL) ?  packet->Free(), false : _channel->Send(packet);
+    }
+
+    virtual HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context context);
+};
+
+
+class FastS_FNET_SearchNode::ExtraDocsumNodesIter
+{
+private:
+    ExtraDocsumNodesIter(const ExtraDocsumNodesIter &other);
+    ExtraDocsumNodesIter& operator=(const ExtraDocsumNodesIter &other);
+
+    FastS_FNET_SearchNode       *_cur;
+    const FastS_FNET_SearchNode *_head;
+
+public:
+    ExtraDocsumNodesIter(const FastS_FNET_SearchNode *head)
+        : _cur(head->_nextExtraDocsumNode),
+          _head(head)
+    {
+    }
+
+    ExtraDocsumNodesIter & operator++(void) {
+        _cur = _cur->_nextExtraDocsumNode;
+        return *this;
+    }
+
+    bool valid(void) const { return _cur != _head; }
+    FastS_FNET_SearchNode *operator*(void) const { return _cur; }
+};
+
+
+//-----------------------------------------------------------------
+
+class FastS_FNET_Search : public FastS_AsyncSearch
+{
+private:
+    FastS_FNET_Search(const FastS_FNET_Search &);
+    FastS_FNET_Search& operator=(const FastS_FNET_Search &);
+
+public:
+
+    class Timeout : public FNET_Task
+    {
+    private:
+        Timeout(const Timeout &);
+        Timeout& operator=(const Timeout &);
+
+        FastS_FNET_Search *_search;
+
+    public:
+        Timeout(FNET_Scheduler *scheduler, FastS_FNET_Search *search)
+            : FNET_Task(scheduler),
+              _search(search) {}
+        void PerformTask();
+    };
+
+    enum FNETMode {
+        FNET_NONE    = 0x00,
+        FNET_QUERY   = 0x01,
+        FNET_DOCSUMS = 0x02
+    };
+
+private:
+    FastOS_Mutex             _lock;
+    FastS_TimeKeeper        *_timeKeeper;
+    double                   _startTime;
+    Timeout                  _timeout;
+    FastS_QueryCacheUtil     _util;
+    std::unique_ptr<search::grouping::MergingManager> _groupMerger;
+    FastS_DataSetCollection *_dsc;	// owner keeps this alive
+    FastS_FNET_DataSet      *_dataset;
+    bool		     _datasetActiveCostRef;
+    std::vector<FastS_FNET_SearchNode> _nodes;
+    bool                     _nodesConnected;
+
+    uint32_t                 _estParts;
+    uint32_t                 _estPartCutoff;
+
+    FNETMode                 _FNET_mode;
+
+    uint32_t                 _pendingQueries;	// # nodes with query left
+    uint32_t		     _goodQueries;	// # queries good
+    uint32_t                 _pendingDocsums;	// # docsums left
+    uint32_t                 _pendingDocsumNodes; // # nodes with docsums left
+    uint32_t		     _requestedDocsums; // # docsums requested
+    uint32_t                 _goodDocsums;	// # docsums good
+    uint32_t                 _queryNodes;	  // #nodes with query
+    uint32_t                 _queryNodesTimedOut;  // #nodes with query timeout
+    uint32_t                 _docsumNodes;	  // #nodes with docsums
+    uint32_t                 _docsumNodesTimedOut; // #nodes with docsum timeout
+    uint32_t                 _docsumsTimedOut;
+    bool                     _queryTimeout;
+    bool                     _docsumTimeout;
+
+    double		     _queryStartTime;
+    double                   _queryMinWait;
+    double                   _queryMaxWait;
+    bool                     _queryWaitCalculated;
+    double		     _adjustedQueryTimeOut;
+    double		     _docSumStartTime;
+    double		     _adjustedDocSumTimeOut;
+    uint32_t                 _fixedRow;
+
+    std::vector<FastS_fullresult>  _resbuf;
+
+    void dropDatasetActiveCostRef(void);
+
+    typedef std::vector<std::pair<FastS_EngineBase *, FastS_FNET_SearchNode *>> EngineNodeMap;
+    void connectNodes(const EngineNodeMap & engines);
+    void AllocNodes();
+    void ConnectQueryNodes();
+    void ConnectEstimateNodes();
+    void connectSearchPath(const vespalib::string &spec);
+    void connectSearchPath(const fdispatch::SearchPath::Element &elem,
+                           const vespalib::string &spec,
+                           uint32_t dispatchLevel);
+    void ConnectDocsumNodes(bool ignoreRow);
+    uint32_t getNextFixedRow();
+    uint32_t getFixedRowCandidate();
+
+    void Lock()   { _lock.Lock();   }
+    void Unlock() { _lock.Unlock(); }
+
+    bool BeginFNETWork();
+    void EndFNETWork();
+
+    void EncodePartIDs(uint32_t partid, uint32_t rowid, bool mld,
+                       FS4Packet_QUERYRESULTX::FS4_hit *pt,
+                       FS4Packet_QUERYRESULTX::FS4_hit *end);
+
+    FastS_TimeKeeper *GetTimeKeeper(void) const { return _timeKeeper; }
+
+    FastS_FNET_SearchNode * getNode(size_t i) { return &_nodes[i]; }
+public:
+    FastS_FNET_Search(FastS_DataSetCollection *dsc,
+                      FastS_FNET_DataSet *dataset,
+                      FastS_TimeKeeper *timeKeeper);
+    virtual ~FastS_FNET_Search();
+
+    void GotQueryResult(FastS_FNET_SearchNode *node, FS4Packet_QUERYRESULTX *qrx);
+    void GotDocsum(FastS_FNET_SearchNode *node, FS4Packet_DOCSUM *docsum);
+    void LostSearchNode(FastS_FNET_SearchNode *node);
+    void GotEOL(FastS_FNET_SearchNode *node);
+    void GotError(FastS_FNET_SearchNode *node, search::fs4transport::FS4Packet_ERROR *error);
+
+    void HandleTimeout();
+
+    bool ShouldLimitHitsPerNode() const;
+    void MergeHits();
+    void CheckCoverage();
+    void CheckQueryTimes();
+    void CheckDocsumTimes();
+    void CheckQueryTimeout();
+    void CheckDocsumTimeout();
+
+    // *** API methods -- BEGIN ***
+
+    virtual FastS_SearchInfo *GetSearchInfo() { return _util.GetSearchInfo(); }
+
+    virtual RetCode Search(uint32_t searchOffset, uint32_t maxhits, uint32_t minhits = 0);
+    virtual RetCode ProcessQueryDone();
+    virtual RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt);
+    virtual RetCode ProcessDocsumsDone();
+
+    // *** API methods -- END ***
+
+    // Hit merging methods
+
+    FastS_FNET_SearchNode *ST_GetNode(size_t i) { return getNode(i); }
+    uint32_t ST_GetNumNodes() const { return _nodes.size(); }
+    bool ST_IsEstimate() const { return _util.IsEstimate(); }
+    uint32_t ST_GetEstParts() const { return _estParts; }
+    uint32_t ST_GetEstPartCutoff() const { return _estPartCutoff; }
+    bool ST_ShouldDropSortData() const { return _util.ShouldDropSortData(); }
+    bool ST_ShouldLimitHitsPerNode() const { return ShouldLimitHitsPerNode(); }
+    void ST_SetNumHits(uint32_t numHits) {
+        _util.SetAlignedHitCount(numHits);
+        _util.CalcHitCount();
+        _util.AllocAlignedHitBuf();
+    }
+    uint32_t ST_GetAlignedSearchOffset() const { return _util.GetAlignedSearchOffset(); }
+    uint32_t ST_GetAlignedMaxHits() const { return _util.GetAlignedMaxHits(); }
+    uint32_t ST_GetAlignedHitCount() const { return _util.GetAlignedHitCount(); }
+    FastS_hitresult *ST_GetAlignedHitBuf() { return _util.GetAlignedHitBuf(); }
+    FastS_hitresult *ST_GetAlignedHitBufEnd() { return _util.GetAlignedHitBufEnd(); }
+    void ST_AllocSortData(uint32_t len) { _util.AllocSortData(len); }
+    uint32_t *ST_GetSortIndex() { return _util.GetSortIndex(); }
+    char *ST_GetSortData() { return _util.GetSortData(); }
+    FastS_QueryResult *ST_GetQueryResult() { return _util.GetQueryResult(); }
+
+    void adjustQueryTimeout();
+    void adjustDocsumTimeout(void);
+    uint32_t getGoodQueries(void) const { return _goodQueries; }
+    uint32_t getRequestedQueries(void) const { return _queryNodes; }
+    uint32_t getPendingQueries(void) const { return _pendingQueries; }
+    uint32_t getDoneQueries(void) const {
+        return getRequestedQueries() - getPendingQueries();
+    }
+    uint32_t getBadQueries(void) const {
+        return getDoneQueries() - getGoodQueries();
+    }
+    uint32_t getGoodDocsums(void) const { return _goodDocsums; }
+    uint32_t getRequestedDocsums(void) const { return _requestedDocsums; }
+    uint32_t getPendingDocsums(void) const { return _pendingDocsums; }
+    uint32_t getDoneDocsums(void) const {
+        return getRequestedDocsums() - getPendingDocsums();
+    }
+    uint32_t getBadDocsums(void) const {
+        return getDoneDocsums() - getGoodDocsums();
+    }
+
+    FNET_Packet::UP
+    setupQueryPacket(uint32_t hitsPerNode, uint32_t qflags,
+                     const search::engine::PropertiesMap &properties);
+};
+
+//-----------------------------------------------------------------------------
+
+class FastS_Sync_FNET_Search : public FastS_SyncSearchAdapter
+{
+private:
+    FastS_FNET_Search _search;
+
+public:
+    FastS_Sync_FNET_Search(FastS_DataSetCollection *dsc,
+                           FastS_FNET_DataSet *dataset,
+                           FastS_TimeKeeper *timeKeeper) :
+        FastS_SyncSearchAdapter(&_search),
+        _search(dsc, dataset, timeKeeper)
+    {
+        _search.SetAsyncArgs(this, FastS_SearchContext());
+    }
+    virtual ~FastS_Sync_FNET_Search();
+    virtual void Free() { delete this; }
+};
+
+//-----------------------------------------------------------------
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp
new file mode 100644
index 00000000000..94977b4af34
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp
@@ -0,0 +1,272 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("");
+#include <vespa/fnet/fnet.h>
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/util/stlishheap.h>
+#include <vespa/searchlib/common/packets.h>
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+#include <vespa/searchcore/fdispatch/search/querycacheutil.h>
+#include <vespa/searchcore/fdispatch/search/fnet_dataset.h>
+#include <vespa/searchcore/fdispatch/search/fnet_engine.h>
+#include <vespa/searchcore/fdispatch/search/fnet_search.h>
+#include <vespa/searchcore/fdispatch/search/mergehits.h>
+
+using search::common::SortData;
+using search::common::SortDataIterator;
+
+//-----------------------------------------------------------------------------
+
+template <bool SORTDATA, bool DROP>
+struct FastS_MergeFeatures
+{
+    static bool UseSortData()  { return SORTDATA; }
+    static bool DropSortData() {     return DROP; }
+};
+
+
+template <typename T, typename F>
+inline bool
+FastS_MergeCompare(typename T::NodeType *a,
+                   typename T::NodeType *b)
+{
+    bool prefer_b = (b->NT_GetNumHitsUsed() < a->NT_GetNumHitsUsed());
+    if (F::UseSortData()) {
+        return b->NT_GetSortDataIterator()->Before(a->NT_GetSortDataIterator(), prefer_b);
+    } else {
+        search::HitRank rank_a = a->NT_GetHit()->HT_GetMetric();
+        search::HitRank rank_b = b->NT_GetHit()->HT_GetMetric();
+        return ((rank_b > rank_a) || ((rank_b == rank_a) && prefer_b));
+    }
+}
+
+
+template <typename T>
+inline void
+FastS_MergeCopySortData(typename T::NodeType *node,
+                        SortData::Ref *dst,
+                        uint32_t &sortDataLen)
+{
+    if (dst == NULL)
+        return;
+
+    SortDataIterator *src = node->NT_GetSortDataIterator();
+    dst->_buf = src->GetBuf();
+    dst->_len = src->GetLen();
+    sortDataLen += dst->_len;
+}
+
+
+template <typename T>
+inline void
+FastS_MergeCopyHit(typename T::HitType *src,
+                   FastS_hitresult *dst)
+{
+    dst->HT_SetGlobalID(src->HT_GetGlobalID());
+    dst->HT_SetMetric(src->HT_GetMetric());
+    dst->HT_SetPartID(src->HT_GetPartID());
+    dst->setDistributionKey(src->getDistributionKey());
+}
+
+
+
+template <typename T, typename F>
+void
+FastS_InternalMergeHits(FastS_HitMerger<T> *merger)
+{
+    typename T::SearchType *search   = merger->GetSearch();
+    typename T::NodeType  **heap     = merger->GetHeap();
+    uint32_t                heapSize = merger->GetHeapSize();
+    typename T::NodeType   *node     = NULL;
+    FastS_hitresult *beg = search->ST_GetAlignedHitBuf();
+    FastS_hitresult *end = search->ST_GetAlignedHitBufEnd();
+    FastS_hitresult *pt  = beg;
+
+    // multi-level sorting related variables
+    SortData::Ref *sortRef = NULL;
+    SortData::Ref *sortItr = NULL;
+    uint32_t         sortDataLen = 0;
+
+    if (F::UseSortData() && !F::DropSortData()) {
+        sortRef = merger->AllocSortRef(end - beg);
+        sortItr = sortRef;
+    }
+
+    FastS_make_heap(heap, heapSize, FastS_MergeCompare<T, F>);
+
+    while (pt < end) {
+        node = *heap;
+        FastS_assert(heapSize > 0);
+        if (F::UseSortData()) {
+            if (!F::DropSortData()) {
+                FastS_MergeCopySortData<T>(node, sortItr++, sortDataLen);
+            }
+            node->NT_GetSortDataIterator()->Next();
+        }
+        FastS_MergeCopyHit<T>(node->NT_GetHit(), pt++);
+        node->NT_NextHit();
+        if (node->NT_GetNumHitsLeft() > 0) {
+            FastS_pop_push_heap(heap, heapSize, node, FastS_MergeCompare<T, F>);
+        } else {
+            FastS_pop_heap(heap, heapSize--, FastS_MergeCompare<T, F>);
+        }
+    }
+    merger->SetLastNode(node); // source of last hit
+    if (F::UseSortData()) {
+        FastS_assert(F::DropSortData() || sortItr == sortRef + (end - beg));
+    }
+
+    // generate merged sort data
+    if (F::UseSortData() && sortDataLen > 0) {
+
+        FastS_assert(!F::DropSortData());
+        search->ST_AllocSortData(sortDataLen);
+
+        uint32_t  offset   = 0;
+        uint32_t *sortIdx  = search->ST_GetSortIndex();
+        char     *sortData = search->ST_GetSortData();
+
+        sortItr = sortRef;
+        for (uint32_t residue = (end - beg); residue > 0; residue--) {
+            *sortIdx++ = offset;
+            memcpy(sortData + offset, sortItr->_buf, sortItr->_len);
+            offset += sortItr->_len;
+            sortItr++;
+        }
+        *sortIdx = offset;
+        FastS_assert(sortItr == sortRef + (end - beg));
+        FastS_assert(offset == sortDataLen);
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+template <typename T>
+typename FastS_HitMerger<T>::NODE **
+FastS_HitMerger<T>::AllocHeap(uint32_t maxNodes)
+{
+    FastS_assert(_heap == NULL);
+    _heap = new NODE*[maxNodes];
+    _heapSize = 0;
+    _heapMax = maxNodes;
+    return _heap;
+}
+
+
+template <typename T>
+SortData::Ref *
+FastS_HitMerger<T>::AllocSortRef(uint32_t size)
+{
+    FastS_assert(_sortRef == NULL);
+    _sortRef = new SortData::Ref[size];
+    return _sortRef;
+}
+
+
+template <typename T>
+void
+FastS_HitMerger<T>::MergeHits()
+{
+    uint32_t numNodes     = _search->ST_GetNumNodes();
+    bool     dropSortData = _search->ST_ShouldDropSortData();
+    bool     useSortData  = false;
+    uint32_t numDocs      = 0;
+    uint64_t totalHits    = 0;
+    search::HitRank maxRank =
+        std::numeric_limits<search::HitRank>::is_integer ?
+        std::numeric_limits<search::HitRank>::min() :
+        - std::numeric_limits<search::HitRank>::max();
+    uint32_t sortDataDocs = 0;
+
+    FastS_QueryResult *result = _search->ST_GetQueryResult();
+
+    // just set totalHitCount for estimates
+    if (_search->ST_IsEstimate()) {
+        for (uint32_t i = 0; i < numNodes; i++) {
+            _search->ST_GetNode(i)->NT_InitMerge(&numDocs, &totalHits,
+                                  &maxRank, &sortDataDocs);
+        }
+        result->_totalHitCount = (_search->ST_GetEstParts() == 0) ? 0
+                                 : (uint64_t) (((double)totalHits
+                                                * (double)_search->ST_GetEstPartCutoff())
+                                               / (double)_search->ST_GetEstParts());
+        return;
+    }
+
+    // prepare nodes for merging
+    NODE **heap = AllocHeap(numNodes);
+    for (uint32_t i = 0; i < numNodes; i++) {
+        if (_search->ST_GetNode(i)->NT_InitMerge(&numDocs, &totalHits,
+                                  &maxRank, &sortDataDocs))
+        {
+            heap[_heapSize++] = _search->ST_GetNode(i);
+        }
+    }
+
+    // check if we should use sort data for sorting
+    if (sortDataDocs > 0) {
+        if (sortDataDocs == numDocs) {
+            useSortData = true;
+        } else {
+            LOG(warning,
+                "Some results are missing sort data, sorting by rank instead");
+        }
+    }
+
+    // set some result variables
+    result->_totalHitCount = totalHits;
+    result->_maxRank = maxRank;
+
+    // allocate some needed structures
+    _search->ST_SetNumHits(numDocs); // NB: allocs result buffer
+
+    // do actual merging by invoking templated function
+    if (useSortData) {
+        if (dropSortData) {
+            FastS_InternalMergeHits
+                <T, FastS_MergeFeatures<true, true> >(this);
+        } else {
+            FastS_InternalMergeHits
+                <T, FastS_MergeFeatures<true, false> >(this);
+        }
+    } else {
+        FastS_InternalMergeHits
+            <T, FastS_MergeFeatures<false, false> >(this);
+    }
+
+    // detect incomplete/fuzzy results
+    if (_search->ST_ShouldLimitHitsPerNode()) {
+        if (_search->ST_GetAlignedHitCount() < _search->ST_GetAlignedMaxHits() &&
+            result->_totalHitCount > (_search->ST_GetAlignedSearchOffset() +
+                                      _search->ST_GetAlignedHitCount()))
+        {
+            _incomplete = true;
+        }
+
+        NODE *lastNode = GetLastNode();
+        for (size_t i(0), m(_search->ST_GetNumNodes()); i < m; i++) {
+            NODE *node(_search->ST_GetNode(i));
+            if (node == lastNode ||
+                node->NT_GetTotalHits() == 0)
+                continue;
+            if (node->NT_GetNumHitsLeft() == 0 &&
+                node->NT_GetTotalHits() > (_search->ST_GetAlignedSearchOffset() +
+                                           node->NT_GetNumHits()))
+            {
+                _fuzzy = true;
+                break;
+            }
+        }
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+template class FastS_HitMerger<FastS_MergeHits_DummyMerge>; // for API check
+template class FastS_HitMerger<FastS_FNETMerge>;
+
+//-----------------------------------------------------------------------------
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h
new file mode 100644
index 00000000000..004f3cf5e1a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/globalid.h>
+
+//-----------------------------------------------------------------------------
+
+// T::HitType API
+
+struct FastS_MergeHits_DummyHit
+{
+    document::GlobalId _emptyGid;
+    uint32_t HT_GetDocID()                {   return 0; }
+    const document::GlobalId & HT_GetGlobalID() { return _emptyGid; }
+    search::HitRank HT_GetMetric()        {   return 0; }
+    uint32_t HT_GetPartID()               {   return 0; }
+    uint32_t getDistributionKey()             {   return 0; }
+    void     HT_SetDocID(uint32_t val)    { (void) val; }
+    void     HT_SetGlobalID(const document::GlobalId & val) { (void) val; }
+    void     HT_SetMetric(search::HitRank val)   { (void) val; }
+    void     HT_SetPartID(uint32_t val)   { (void) val; }
+    void     setDistributionKey(uint32_t val) { (void) val; }
+};
+
+// T::NodeType API
+
+struct FastS_MergeHits_DummyNode
+{
+    bool                      NT_InitMerge(uint32_t *numDocs,
+                                           uint64_t *totalHits,
+                                           search::HitRank *maxRank,
+                                           uint32_t *sortDataDocs)
+    {
+        (void) numDocs;
+        (void) totalHits;
+        (void) maxRank;
+        (void) sortDataDocs;
+        return false;
+    }
+    search::common::SortDataIterator   *NT_GetSortDataIterator() { return NULL; }
+    FastS_MergeHits_DummyHit *NT_GetHit()              { return NULL; }
+    uint32_t                  NT_GetNumHitsUsed()      {    return 0; }
+    uint32_t                  NT_GetNumHitsLeft()      {    return 0; }
+    uint64_t                  NT_GetTotalHits()        {    return 0; }
+    uint32_t                  NT_GetNumHits()          {    return 0; }
+    void                      NT_NextHit()             {              }
+};
+
+// T::SearchType API
+
+struct FastS_MergeHits_DummySearch
+{
+    FastS_MergeHits_DummyNode *ST_GetNode(size_t i)    { (void) i;    return NULL; }
+    uint32_t           ST_GetNumNodes()                {       return 0; }
+    bool               ST_IsEstimate()                 {   return false; }
+    uint32_t           ST_GetEstParts()                {       return 0; }
+    uint32_t           ST_GetEstPartCutoff()           {       return 0; }
+    bool               ST_ShouldDropSortData()         {   return false; }
+    bool               ST_ShouldLimitHitsPerNode()     {   return false; }
+    void               ST_SetNumHits(uint32_t numHits) { (void) numHits; }
+    uint32_t           ST_GetAlignedSearchOffset()     {       return 0; }
+    uint32_t           ST_GetAlignedMaxHits()          {       return 0; }
+    uint32_t           ST_GetAlignedHitCount()         {       return 0; }
+    FastS_hitresult   *ST_GetAlignedHitBuf()           {    return NULL; }
+    FastS_hitresult   *ST_GetAlignedHitBufEnd()        {    return NULL; }
+    void               ST_AllocSortData(uint32_t len)  {     (void) len; }
+    uint32_t          *ST_GetSortIndex()               {    return NULL; }
+    char              *ST_GetSortData()                {    return NULL; }
+    FastS_QueryResult *ST_GetQueryResult()             {    return NULL; }
+};
+
+// T (Merge Type) API
+
+struct FastS_MergeHits_DummyMerge
+{
+    typedef FastS_MergeHits_DummyHit     HitType;
+    typedef FastS_MergeHits_DummyNode    NodeType;
+    typedef FastS_MergeHits_DummySearch  SearchType;
+};
+
+//-----------------------------------------------------------------------------
+
+struct FastS_FNETMerge
+{
+    typedef FS4Packet_QUERYRESULTX::FS4_hit HitType;
+    typedef FastS_FNET_SearchNode           NodeType;
+    typedef FastS_FNET_Search               SearchType;
+};
+
+//-----------------------------------------------------------------------------
+
+template <typename T>
+class FastS_HitMerger
+{
+private:
+    FastS_HitMerger(const FastS_HitMerger &);
+    FastS_HitMerger& operator=(const FastS_HitMerger &);
+
+
+    typedef typename T::HitType    HIT;
+    typedef typename T::NodeType   NODE;
+    typedef typename T::SearchType SEARCH;
+
+    // owning search object
+    SEARCH               *_search;
+
+    // sorting heap
+    NODE                **_heap;
+    uint32_t              _heapSize;
+    uint32_t              _heapMax;
+
+    // temporary array for merging sortdata
+    search::common::SortData::Ref  *_sortRef;
+
+    // limit hits per node effect variables
+    NODE                 *_lastNode;
+    bool                  _incomplete;
+    bool                  _fuzzy;
+
+public:
+    FastS_HitMerger(SEARCH *search) : _search(search),
+                                      _heap(NULL),
+                                      _heapSize(0),
+                                      _heapMax(0),
+                                      _sortRef(NULL),
+                                      _lastNode(NULL),
+                                      _incomplete(false),
+                                      _fuzzy(false)
+    {}
+
+    ~FastS_HitMerger()
+    {
+        delete [] _heap;
+        delete [] _sortRef;
+    }
+
+    NODE **AllocHeap(uint32_t maxNodes);
+    search::common::SortData::Ref *AllocSortRef(uint32_t size);
+    void SetLastNode(NODE *lastNode) { _lastNode = lastNode; }
+
+    SEARCH *GetSearch() const { return _search; }
+    NODE **GetHeap() const { return _heap; }
+    uint32_t GetHeapSize() const { return _heapSize; }
+    uint32_t GetHeapMax() const { return _heapMax; }
+    NODE *GetLastNode() const { return _lastNode; }
+    bool WasIncomplete() const { return _incomplete; }
+    bool WasFuzzy() const { return _fuzzy; }
+
+    search::common::SortData::Ref *GetSortRef() const { return _sortRef; }
+
+    void MergeHits();
+};
+
+//-----------------------------------------------------------------------------
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp
new file mode 100644
index 00000000000..9435df7fa62
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp
@@ -0,0 +1,484 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+
+#include <vespa/searchcore/fdispatch/common/appcontext.h>
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+#include <vespa/searchcore/fdispatch/search/nodemanager.h>
+
+#include <vespa/searchcore/fdispatch/search/plain_dataset.h>
+#include "engine_base.h"
+
+LOG_SETUP(".search.nodemanager");
+
+void
+FastS_NodeManager::configure(std::unique_ptr<PartitionsConfig> cfg)
+{
+    LOG(config, "configuring datasetcollection from '%s'",
+        _configUri.getConfigId().c_str());
+    SetPartMap(*cfg, 2000);
+    _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("fdispatch.nodemanager",
+                                       _fetcher->getGeneration(),
+                                       "will not update generation unless config has changed"));
+}
+
+
+class AdminBadEngines
+{
+    std::set<vespalib::string> _bad;
+public:
+    void
+    addAdminBad(const vespalib::string &name)
+    {
+        _bad.insert(name);
+    }
+
+    bool
+    isAdminBad(const vespalib::string &name) const
+    {
+        return _bad.find(name) != _bad.end();
+    }
+};
+
+class CollectAdminBadEngines
+{
+    AdminBadEngines &_adminBadEngines;
+
+public:
+
+    CollectAdminBadEngines(AdminBadEngines &adminBadEngines)
+        : _adminBadEngines(adminBadEngines)
+    {
+    }
+    
+    void operator()(FastS_EngineBase* engine)
+    {
+        if (engine->isAdminBad()) {
+            _adminBadEngines.addAdminBad(engine->GetName());
+        }
+    }
+};
+
+
+class PropagateAdminBadEngines
+{
+    const AdminBadEngines &_adminBadEngines;
+
+public:
+
+    PropagateAdminBadEngines(const AdminBadEngines &adminBadEngines)
+        : _adminBadEngines(adminBadEngines)
+    {
+    }
+    
+    void operator()(FastS_EngineBase* engine)
+    {
+        if (_adminBadEngines.isAdminBad(engine->GetName())) {
+            engine->MarkBad(FastS_EngineBase::BAD_ADMIN);
+        }
+    }
+};
+
+
+FastS_NodeManager::FastS_NodeManager(vespalib::SimpleComponentConfigProducer &componentConfig,
+                                     FastS_AppContext *appCtx,
+                                     uint32_t partition)
+    : _componentConfig(componentConfig),
+      _managerLock(),
+      _configLock(),
+      _stampLock(),
+      _appCtx(appCtx),
+      _mldPartit(partition),
+      _mldDocStamp(0),
+      _mldDocStampMin(0),
+      _gencnt(0),
+      _queryPerf(),
+      _fetcher(),
+      _configUri(config::ConfigUri::createEmpty()),
+      _lastPartMap(NULL),
+      _datasetCollection(NULL),
+      _oldDSCList(NULL),
+      _tempFail(false),
+      _failed(false),
+      _hasDsc(false),
+      _checkTempFailScheduled(false),
+      _shutdown(false)
+{
+    _datasetCollection = new FastS_DataSetCollection(_appCtx);
+    FastS_assert(_datasetCollection != NULL);
+    _datasetCollection->Configure(NULL, 0);
+    FastOS_Time now;
+    now.SetNow();
+    _mldDocStamp = now.GetSeconds();
+    _mldDocStampMin = _mldDocStamp;
+}
+
+
+FastS_NodeManager::~FastS_NodeManager()
+{
+    free(_lastPartMap);
+    FastS_assert(_datasetCollection != NULL);
+    _datasetCollection->subRef();
+}
+
+void
+FastS_NodeManager::CheckTempFail()
+{
+    bool tempfail;
+
+    _checkTempFailScheduled = false;
+    tempfail = false;
+    LockManager();
+    FastS_DataSetCollection *dsc = PeekDataSetCollection();
+    for (unsigned int i = 0; i < dsc->GetMaxNumDataSets(); i++) {
+        FastS_DataSetBase *ds;
+        FastS_PlainDataSet *ds_plain;
+        if ((ds = dsc->PeekDataSet(i)) != NULL &&
+            (ds_plain = ds->GetPlainDataSet()) != NULL &&
+            ds_plain->GetTempFail()) {
+            tempfail = true;
+            break;
+        }
+    }
+    UnlockManager();
+    _tempFail = tempfail;
+}
+
+void
+FastS_NodeManager::SubscribePartMap(const config::ConfigUri & configUri)
+{
+    vespalib::string configId(configUri.getConfigId());
+    LOG(debug, "loading new datasetcollection from %s", configId.c_str());
+    try {
+        _configUri = configUri;
+        _fetcher.reset(new config::ConfigFetcher(_configUri.getContext()));
+        _fetcher->subscribe<PartitionsConfig>(configId, this);
+        _fetcher->start();
+        if (_gencnt == 0) {
+            throw new config::InvalidConfigException("failure during initial configuration: bad partition map");
+        }
+    } catch (std::exception &ex) {
+        LOG(error, "Runtime exception: %s", (const char *) ex.what());
+        EV_STOPPING("", "bad partitions config");
+        exit(1);
+    }
+}
+
+
+uint32_t
+FastS_NodeManager::SetPartMap(const PartitionsConfig& partmap,
+			      unsigned int waitms)
+{
+    LockConfig();
+    FastS_DataSetCollDesc *configDesc = new FastS_DataSetCollDesc();
+    if (!configDesc->ReadConfig(partmap)) {
+        LOG(error, "NodeManager::SetPartMap: Failed to load configuration");
+        delete configDesc;
+        UnlockConfig();
+        return 0;
+    }
+    int retval = SetCollDesc(configDesc, waitms);
+    UnlockConfig();
+    return retval;
+}
+
+
+uint32_t
+FastS_NodeManager::SetCollDesc(FastS_DataSetCollDesc *configDesc,
+                               unsigned int waitms)
+{
+    FastS_DataSetCollection *newCollection;
+    uint32_t gencnt;
+
+    if (_shutdown) return 0;
+
+    AdminBadEngines adminBad;
+
+    {
+        CollectAdminBadEngines adminBadCollect(adminBad);
+        FastS_DataSetCollection *dsc = GetDataSetCollection();
+        for (uint32_t i = 0; i < dsc->GetMaxNumDataSets(); i++) {
+            FastS_DataSetBase *ds;
+            FastS_PlainDataSet *ds_plain;
+            if ((ds = dsc->PeekDataSet(i)) == NULL ||
+                (ds_plain = ds->GetPlainDataSet()) == NULL)
+                continue;
+            
+            ds_plain->ForEachEngine(adminBadCollect);
+        }
+        dsc->subRef();
+    }
+
+
+    newCollection = new FastS_DataSetCollection(_appCtx);
+    if (!newCollection->Configure(configDesc, _gencnt + 1)) {
+        LOG(error, "NodeManager::SetPartMap: Inconsistent configuration");
+        newCollection->subRef();
+        return 0;
+    }
+
+    {
+        PropagateAdminBadEngines adminBadPropagate(adminBad);
+        for (uint32_t i = 0; i < newCollection->GetMaxNumDataSets(); i++) {
+            FastS_DataSetBase *ds;
+            FastS_PlainDataSet *ds_plain;
+            if ((ds = newCollection->PeekDataSet(i)) == NULL ||
+                (ds_plain = ds->GetPlainDataSet()) == NULL)
+                continue;
+            
+            ds_plain->ForEachEngine(adminBadPropagate);
+        }
+    }
+
+    if (waitms > 0) {
+        FastOS_Time last;
+        unsigned int rwait;
+        bool allup;
+        last.SetNow();
+        while (1) {
+            allup = newCollection->AreEnginesReady();
+            rwait = (unsigned int) last.MilliSecsToNow();
+            if (rwait >= waitms || allup)
+                break;
+            FastOS_Thread::Sleep(100);
+        };
+        if (allup)
+            LOG(debug, "All new engines up after %d ms", rwait);
+        else
+            LOG(debug, "Some new engines still down after %d ms", rwait);
+    }
+
+    gencnt = SetDataSetCollection(newCollection);
+
+    ScheduleCheckTempFail(FastS_NoID32());
+    return gencnt;
+}
+
+
+
+/**
+ * When calling this method, a single reference on the 'dsc' parameter
+ * is passed to the monitor object.
+ *
+ * @return generation count, or 0 on fail.
+ * @param dsc new dataset collection. A single reference is passed
+ *            to the monitor when this method is invoked.
+ **/
+uint32_t
+FastS_NodeManager::SetDataSetCollection(FastS_DataSetCollection *dsc)
+{
+    if (dsc == NULL)
+        return 0;
+
+    uint32_t                 gencnt  = 0;
+    FastS_DataSetCollection *old_dsc = NULL;
+
+    if (!dsc->IsValid()) {
+        LOG(error, "NodeManager::SetDataSetCollection: Inconsistent configuration");
+        dsc->subRef();
+
+    } else {
+
+        LockManager();
+        _gencnt++;
+        gencnt = _gencnt;
+
+        old_dsc = _datasetCollection;
+        _datasetCollection = dsc;
+
+        // put old config on service list
+        FastS_assert(old_dsc != NULL);
+        if (!old_dsc->IsLastRef()) {
+            old_dsc->_nextOld = _oldDSCList;
+            _oldDSCList = old_dsc;
+            old_dsc = NULL;
+        }
+        _hasDsc = true;
+        UnlockManager();
+
+        if (old_dsc != NULL)
+            old_dsc->subRef();
+    }
+    return gencnt;
+}
+
+
+FastS_DataSetCollection *
+FastS_NodeManager::GetDataSetCollection()
+{
+    FastS_DataSetCollection *ret;
+
+    LockManager();
+    ret = _datasetCollection;
+    FastS_assert(ret != NULL);
+    ret->addRef();
+    UnlockManager();
+
+    return ret;
+}
+
+
+void
+FastS_NodeManager::ShutdownConfig()
+{
+    FastS_DataSetCollection *dsc;
+    FastS_DataSetCollection *old_dsc;
+
+    LockConfig();
+    LockManager();
+    _shutdown = true;           // disallow SetPartMap
+    dsc = _datasetCollection;
+    _datasetCollection = new FastS_DataSetCollection(_appCtx);
+    _datasetCollection->Configure(NULL, 0);
+    old_dsc = _oldDSCList;
+    _oldDSCList = NULL;
+    UnlockManager();
+    UnlockConfig();
+    dsc->AbortQueryQueues();
+    dsc->subRef();
+    while (old_dsc != NULL) {
+        dsc = old_dsc;
+        old_dsc = old_dsc->_nextOld;
+        dsc->_nextOld = NULL;
+        dsc->AbortQueryQueues();
+        dsc->subRef();
+    }
+}
+
+
+uint32_t
+FastS_NodeManager::GetTotalPartitions()
+{
+    uint32_t ret;
+
+    ret = 0;
+    LockManager();
+    FastS_DataSetCollection *dsc = PeekDataSetCollection();
+    for (unsigned int i = 0; i < dsc->GetMaxNumDataSets(); i++) {
+        FastS_DataSetBase *ds;
+        FastS_PlainDataSet *ds_plain;
+        if ((ds = dsc->PeekDataSet(i)) != NULL &&
+            (ds_plain = ds->GetPlainDataSet()) != NULL)
+            ret += ds_plain->GetPartitions();
+    }
+    UnlockManager();
+    return ret;
+}
+
+
+ChildInfo
+FastS_NodeManager::getChildInfo()
+{
+    ChildInfo r;
+    r.activeDocs.valid = true;
+    FastS_DataSetCollection *dsc = GetDataSetCollection();
+
+    for (unsigned int i = 0; i < dsc->GetMaxNumDataSets(); i++) {
+        FastS_DataSetBase *ds;
+        FastS_PlainDataSet *ds_plain;
+        if ((ds = dsc->PeekDataSet(i)) == NULL ||
+            (ds_plain = ds->GetPlainDataSet()) == NULL)
+            continue;
+        r.maxNodes    += ds_plain->_partMap._childmaxnodesSinceReload;
+        r.activeNodes += ds_plain->_partMap._childnodes;
+        r.maxParts    += ds_plain->_partMap._childmaxpartsSinceReload;
+        r.activeParts += ds_plain->_partMap._childparts;
+        PossCount rowActive = ds_plain->getActiveDocs();
+        if (rowActive.valid) {
+            r.activeDocs.count += rowActive.count;
+        } else {
+            r.activeDocs.valid = false;
+        }
+    }
+
+    dsc->subRef();
+    return r;
+}
+
+
+void
+FastS_NodeManager::logPerformance()
+{
+    _queryPerf.reset();
+    FastS_DataSetCollection *dsc = GetDataSetCollection();
+
+    for (unsigned int i = 0; i < dsc->GetMaxNumDataSets(); i++) {
+        if (dsc->PeekDataSet(i) != NULL) {
+            dsc->PeekDataSet(i)->addPerformance(_queryPerf);
+        }
+    }
+
+    dsc->subRef();
+    _queryPerf.log();
+}
+
+
+void
+FastS_NodeManager::CheckEvents(FastS_TimeKeeper *timeKeeper)
+{
+    // CHECK SCHEDULED OPERATIONS
+
+    if (_checkTempFailScheduled)
+        CheckTempFail();
+
+    // CHECK QUERY QUEUES
+
+    FastS_DataSetCollection *dsc = GetDataSetCollection();
+
+    dsc->CheckQueryQueues(timeKeeper);
+    dsc->subRef();
+
+    // check old query queues and discard old configs
+
+    FastS_DataSetCollection *old_dsc;
+    FastS_DataSetCollection *prev = NULL;
+    FastS_DataSetCollection *tmp;
+
+    LockManager();
+    old_dsc = _oldDSCList;
+    UnlockManager();
+
+    while (old_dsc != NULL) {
+        if (old_dsc->IsLastRef()) {
+            if (prev == NULL) {
+                LockManager();
+                if (_oldDSCList == old_dsc) {
+
+                    _oldDSCList = old_dsc->_nextOld;
+                    UnlockManager();
+
+                } else {
+
+                    prev = _oldDSCList;
+                    UnlockManager();
+                    while (prev->_nextOld != old_dsc)
+                        prev = prev->_nextOld;
+
+                    prev->_nextOld = old_dsc->_nextOld;
+                }
+            } else {
+                prev->_nextOld = old_dsc->_nextOld;
+            }
+            tmp = old_dsc;
+            old_dsc = old_dsc->_nextOld;
+            tmp->subRef();
+
+        } else {
+
+            old_dsc->CheckQueryQueues(timeKeeper);
+            prev = old_dsc;
+            old_dsc = old_dsc->_nextOld;
+        }
+    }
+}
+
+uint32_t
+FastS_NodeManager::GetMldDocstamp()
+{
+    if (!_hasDsc)
+        return 0;
+    return _mldDocStamp;
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h
new file mode 100644
index 00000000000..c51ec6b1e09
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h
@@ -0,0 +1,106 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include "child_info.h"
+#include "configdesc.h"
+#include <vespa/config/helper/configfetcher.h>
+#include <vespa/searchcore/fdispatch/common/queryperf.h>
+#include <vespa/vespalib/net/simple_component_config_producer.h>
+
+using vespa::config::search::core::PartitionsConfig;
+
+class FastS_DataSetBase;
+
+class FastS_NodeManager : public config::IFetcherCallback<PartitionsConfig>
+{
+private:
+    FastS_NodeManager(const FastS_NodeManager &);
+    FastS_NodeManager& operator=(const FastS_NodeManager &);
+
+    vespalib::SimpleComponentConfigProducer &_componentConfig;
+
+    FastOS_Mutex       _managerLock;
+    FastOS_Mutex       _configLock;
+    FastOS_Mutex       _stampLock;
+    FastS_AppContext *_appCtx;
+    uint32_t          _mldPartit;
+    uint32_t          _mldDocStamp;	// Bumped for all cache flushes
+    uint32_t	      _mldDocStampMin;	// Bumped for global cache flush
+    uint32_t          _gencnt;
+
+    FastS_QueryPerf   _queryPerf;
+
+
+    std::unique_ptr<config::ConfigFetcher> _fetcher;
+    config::ConfigUri _configUri;
+
+    char                    *_lastPartMap;
+    FastS_DataSetCollection *_datasetCollection; // current node config
+    FastS_DataSetCollection *_oldDSCList;  // list of old node configs
+
+    bool              _tempFail;
+    bool              _failed;
+    bool              _hasDsc;
+
+    volatile bool     _checkTempFailScheduled;
+    volatile bool     _shutdown;
+
+protected:
+    void SetFailed() { _failed = true; }
+
+    void configure(std::unique_ptr<PartitionsConfig> cfg);
+
+public:
+    FastS_NodeManager(vespalib::SimpleComponentConfigProducer &componentConfig,
+                      FastS_AppContext *appCtx,
+                      uint32_t partition);
+    ~FastS_NodeManager();
+
+    void SubscribePartMap(const config::ConfigUri & configUri);
+
+    void LockManager()   { _managerLock.Lock(); }
+    void UnlockManager() { _managerLock.Unlock(); }
+
+    void LockConfig()   { _configLock.Lock(); }
+    void UnlockConfig() { _configLock.Unlock(); }
+
+    void LockStamp()   { _stampLock.Lock(); }
+    void UnlockStamp() { _stampLock.Unlock(); }
+
+    uint32_t GetMldPartition() const { return _mldPartit; }
+    uint32_t GetMldDocstamp();
+    uint32_t GetGenCnt() const { return _gencnt; }
+
+    bool Failed() const { return _failed; }
+    bool GetTempFail() const { return _tempFail; }
+
+    void ScheduleCheckTempFail(uint32_t datasetid) {
+        (void) datasetid;
+        _checkTempFailScheduled = true;
+    }
+
+    FastS_AppContext *GetAppContext() { return _appCtx; }
+    FastS_DataSetCollection *PeekDataSetCollection()
+    { return _datasetCollection; }
+
+    void CheckTempFail();
+    uint32_t SetPartMap(const PartitionsConfig& partmap, unsigned int waitms);
+    uint32_t SetCollDesc(FastS_DataSetCollDesc *configDesc, unsigned int waitms);
+    uint32_t SetDataSetCollection(FastS_DataSetCollection *dsc);
+    FastS_DataSetCollection *GetDataSetCollection();
+    uint32_t GetTotalPartitions();
+    ChildInfo getChildInfo();
+    void ShutdownConfig();
+
+    /**
+     * log query performance. This method should only be invoked from
+     * the FNET thread.
+     **/
+    void logPerformance();
+
+    void CheckEvents(FastS_TimeKeeper *timeKeeper); // invoked by FNET thread
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/partitioned_array.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/partitioned_array.cpp
new file mode 100644
index 00000000000..a2af3b5c737
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/partitioned_array.cpp
@@ -0,0 +1,167 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <cstring>
+#include <algorithm>
+
+#include <vespa/log/log.h>
+LOG_SETUP("");
+
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/fdispatch/search/partitioned_array.h>
+#include <vespa/searchcore/fdispatch/search/engine_base.h>
+#include <vespa/searchcore/fdispatch/common/stdincl.h>
+
+using namespace FastS_QueryDistribution;
+
+#define PA PartitionedArray
+
+struct PA::PartitionIDLessThan {
+    bool operator()(MeasuredVec& partition,
+                    FastS_EngineBase* engine) const
+    {
+        return (*partition.vec.begin())->GetPartID() <
+          engine->GetPartID();
+    }
+};
+
+namespace {
+struct EngineOrdering {
+    //less than definition
+    bool operator()(FastS_EngineBase* l,
+                    FastS_EngineBase* r) const {
+        return std::strcmp(l->GetName(),
+                           r->GetName()) < 0;
+    }
+};
+
+} //end anonymous namespace
+
+bool PA::EqualPartitionID(PartitionIterator partitionIterator, FastS_EngineBase* engine) {
+    FastS_EngineBase* firstEngine = *partitionIterator->vec.begin();
+    return firstEngine->GetPartID() == engine->GetPartID();
+}
+
+void PA::InsertNewEngine(PartitionIterator partitionIterator,
+        FastS_EngineBase* engine) {
+    EngineIterator engineIterator = std::lower_bound(
+            partitionIterator->vec.begin(),
+            partitionIterator->vec.end(),
+            engine, EngineOrdering());
+
+    partitionIterator->vec.insert(engineIterator, engine);
+
+}
+
+void PA::InsertNewPartition(PartitionIterator partitionIterator,
+        FastS_EngineBase* engine) {
+    if (partitionIterator == _partitions.begin()) {
+        _minPartitionID = engine->GetPartID();
+    }
+
+    MeasuredVec partition;
+    partition.vec.push_back(engine);
+    _partitions.insert(partitionIterator, partition);
+}
+
+void PA::Add(FastS_EngineBase* engine) {
+
+    if (engine->GetPartID() == FastS_NoID32()){
+        AddEngineWithInvalidPartitionID(engine);
+    } else {
+        PartitionIterator partitionIter = std::lower_bound(
+                _partitions.begin(),
+                _partitions.end(),
+                engine,
+                PartitionIDLessThan());
+
+        if( partitionIter == _partitions.end() ||
+           !EqualPartitionID(partitionIter, engine) )
+        {
+            InsertNewPartition( partitionIter, engine);
+        } else {
+            InsertNewEngine( partitionIter, engine );
+        }
+    }
+    ++_numEngines;
+}
+
+void PA::AddEngineWithInvalidPartitionID(FastS_EngineBase* engine) {
+    _invalidPartitionEngines.push_back(engine);
+}
+
+FastS_EngineBase* PA::Extract() {
+    if (_partitions.empty() ) {
+        if (_invalidPartitionEngines.empty())
+            return 0;
+        else {
+            FastS_EngineBase* res = _invalidPartitionEngines.back();
+            _invalidPartitionEngines.pop_back();
+            --_numEngines;
+            return res;
+        }
+    }
+    else {
+        FastS_EngineBase* res = _partitions.back().vec.back();
+        _partitions.back().vec.pop_back();
+        if( _partitions.back().vec.empty() ) {
+            _partitions.pop_back();
+        }
+        --_numEngines;
+        return res;
+    }
+
+}
+
+void PA::EnginePartitionIDChanged(FastS_EngineBase* engine, uint32_t oldID) {
+    if (oldID == FastS_NoID32()) {
+        RemoveFromInvalidList(engine);
+    } else {
+        RemoveFromPartitionedArray(engine,oldID);
+    }
+    Add(engine);
+}
+
+void PA::RemoveFromInvalidList(FastS_EngineBase* engine) {
+    if ( _invalidPartitionEngines.end() !=
+        std::remove(_invalidPartitionEngines.begin(), _invalidPartitionEngines.end(),
+                    engine) )
+    {
+        --_numEngines;
+        _invalidPartitionEngines.pop_back();
+    } else {
+        LOG(error, "RemoveFromInvalidList: Engine not found");
+    }
+}
+
+void PA::RemoveFromPartitionedArray(FastS_EngineBase* engine, uint32_t oldID) {
+    size_t index = oldID - _minPartitionID;
+    if (_partitions[index].vec.end() !=
+        std::remove(_partitions[index].vec.begin(), _partitions[index].vec.end(),
+                    engine) )
+    {
+        --_numEngines;
+        _partitions[index].vec.pop_back();
+    } else {
+        LOG(error, "RemoveFromPartitionedArray: Engine with oldID %d not found", oldID);
+    }
+
+    if (_partitions[index].vec.empty()) {
+        _partitions.erase(_partitions.begin() + index);
+    }
+}
+
+uint32_t PA::totalMeasure() const{
+    uint32_t result = 0;
+
+    for (const_PartitionIterator pi = _partitions.begin();
+         pi != _partitions.end();
+         ++pi) {
+        result += pi->count;
+    }
+    return result;
+}
+
+uint32_t PA::Partition::ID() const {
+    FastS_assert(!Empty());
+    return (*this)[0]->GetPartID();
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/partitioned_array.h b/searchcore/src/vespa/searchcore/fdispatch/search/partitioned_array.h
new file mode 100644
index 00000000000..c719aa773a2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/partitioned_array.h
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+
+#include <vector>
+
+class FastS_EngineBase;
+
+namespace FastS_QueryDistribution {
+
+//assumes partitions are numbered sequentially
+class PartitionedArray {
+private:
+    typedef std::vector<FastS_EngineBase*> Vec;
+
+    struct MeasuredVec {
+        Vec vec;
+        //counts number of deterministically distributed queries
+        mutable uint32_t count;
+        MeasuredVec()
+            :count(0)
+        {}
+    };
+
+    typedef std::vector<MeasuredVec> Vec2d;
+
+    typedef Vec2d::iterator PartitionIterator;
+    typedef Vec2d::const_iterator const_PartitionIterator;
+
+    typedef Vec::iterator EngineIterator;
+    typedef Vec::const_iterator const_EngineIterator;
+
+    Vec2d _partitions;
+    size_t _minPartitionID;
+
+    Vec _invalidPartitionEngines;
+    size_t _numEngines;
+
+public:
+    class Partition {
+        const MeasuredVec& _partition;
+    public:
+        Partition(const PartitionedArray& pc, size_t index)
+            : _partition( pc._partitions[index] )
+        {}
+
+        FastS_EngineBase* operator[](size_t index) const {
+            return _partition.vec[index];
+        }
+
+        size_t Size() const{
+            return _partition.vec.size();
+        }
+
+        uint32_t QueryCount() const {
+            return _partition.count;
+        }
+
+        void IncQueryCount() const {
+            ++_partition.count;
+        }
+
+        bool Empty() const {
+            return _partition.vec.empty();
+        }
+
+        uint32_t ID() const;
+
+    };
+
+private:
+    bool EqualPartitionID(PartitionIterator partitionIterator, FastS_EngineBase* engine);
+    void InsertNewEngine(PartitionIterator partitionIterator, FastS_EngineBase* engine);
+    void InsertNewPartition(PartitionIterator partitionIterator, FastS_EngineBase* engine);
+    void AddEngineWithInvalidPartitionID(FastS_EngineBase* engine);
+    void RemoveFromInvalidList(FastS_EngineBase* engine);
+    void RemoveFromPartitionedArray(FastS_EngineBase* engine, uint32_t oldID);
+    struct PartitionIDLessThan;
+
+
+public:
+    //should only be used after building modus is finished
+    inline Partition operator[](size_t partitionIndex) const {
+        return Partition(*this, partitionIndex);
+    }
+
+    inline Partition operator()(size_t partitionID) const {
+        return Partition(*this, partitionID - _minPartitionID);
+    }
+
+    FastS_EngineBase* Extract();
+    void Add(FastS_EngineBase* engine);
+    void EnginePartitionIDChanged(FastS_EngineBase* engine, uint32_t oldID);
+
+    template <class FUN>
+    FUN ForEachPartition(FUN fun) const {
+        for (size_t i=0; i<_partitions.size(); i++) {
+            fun(Partition(*this, i));
+        }
+        return fun;
+    }
+
+    template <class FUN>
+    FUN ForEach(FUN fun) const {
+        for(const_EngineIterator ei = _invalidPartitionEngines.begin();
+            ei != _invalidPartitionEngines.end();
+            ++ei)
+        {
+            fun(*ei);
+        }
+
+        for(const_PartitionIterator pi=_partitions.begin();
+            pi != _partitions.end();
+            ++pi)
+        {
+            for(const_EngineIterator ei = pi->vec.begin();
+                ei != pi->vec.end();
+                ++ei)
+            {
+                fun(*ei);
+            }
+        }
+        return fun;
+    }
+
+    uint32_t totalMeasure() const;
+
+    PartitionedArray() :
+        _minPartitionID(0)
+    {}
+
+    size_t NumPartitions() {
+        return _partitions.size();
+    }
+
+};
+
+} //namespace FastS_QueryDistribution
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp
new file mode 100644
index 00000000000..285e37abee7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp
@@ -0,0 +1,604 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <functional>
+#include <iomanip>
+#include <sstream>
+
+#include <vespa/log/log.h>
+#include <vespa/fnet/fnet.h>
+#include <vespa/vespalib/util/host_name.h>
+#include <vespa/searchcore/fdispatch/common/stdincl.h>
+#include <vespa/searchlib/common/fslimits.h>
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+#include <vespa/searchcore/fdispatch/common/search.h>
+
+#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
+#include <vespa/searchcore/fdispatch/search/engine_base.h>
+#include <vespa/searchcore/fdispatch/search/plain_dataset.h>
+#include <vespa/searchcore/fdispatch/search/nodemanager.h>
+
+#include <boost/bind.hpp>
+#include <boost/mem_fn.hpp>
+
+LOG_SETUP(".search.plain_dataset");
+
+//--------------------------------------------------------------------------
+
+static inline int imax(int a, int b) { return (a > b) ? a : b; }
+
+//--------------------------------------------------------------------------
+
+FastS_PartitionMap::Partition::Partition()
+    : _engines(NULL),
+      _maxnodesNow(0),
+      _maxnodesSinceReload(0),
+      _nodes(0),
+      _maxpartsNow(0),
+      _maxpartsSinceReload(0),
+      _parts(0)
+{
+}
+
+
+FastS_PartitionMap::Partition::~Partition()
+{
+    FastS_assert(_engines == NULL);
+    FastS_assert(_nodes == 0);
+    FastS_assert(_parts == 0);
+}
+
+//--------------------------------------------------------------------------
+
+FastS_PartitionMap::FastS_PartitionMap(FastS_DataSetDesc *desc)
+    : _partitions(NULL),
+      _partBits(desc->GetPartBits()),
+      _rowBits(desc->GetRowBits()),
+      _num_partitions(desc->GetNumParts()),
+      _first_partition(desc->GetFirstPart()),
+      _minchildparts(desc->GetMinChildParts()),
+      _maxNodesDownPerFixedRow(desc->getMaxNodesDownPerFixedRow()),
+      _useRoundRobinForFixedRow(desc->useRoundRobinForFixedRow()),
+      _childnodes(0),
+      _childmaxnodesNow(0),
+      _childmaxnodesSinceReload(0),
+      _childparts(0),
+      _childmaxpartsNow(0),
+      _childmaxpartsSinceReload(0),
+      _mpp(desc->getMPP()),
+      _maxRows(0)
+{
+    // finalize config settings
+    if (_num_partitions > (1U << _partBits)) {
+        LOG(error,
+            "Too many partitions %d constrained by partbits %d",
+            _num_partitions, _partBits);
+        _num_partitions = (1U << _partBits);
+    }
+
+    if (_num_partitions > 0) {
+        _partitions = new Partition[_num_partitions];
+        FastS_assert(_partitions != NULL);
+    }
+    for (FastS_EngineDesc *curr = desc->GetEngineList(); curr != NULL; curr = curr->GetNext()) {
+        _maxRows = std::max(_maxRows, curr->GetConfRowID());
+    }
+    _numPartitions = std::vector<uint32_t>(getNumRows(), 0);
+    for (FastS_EngineDesc *curr = desc->GetEngineList(); curr != NULL; curr = curr->GetNext()) {
+        size_t rowId(curr->GetConfRowID());
+        _numPartitions[rowId] = std::max(_numPartitions[rowId], curr->GetConfPartID()+1);
+    }
+}
+
+
+FastS_PartitionMap::~FastS_PartitionMap()
+{
+    delete [] _partitions;
+}
+
+
+void
+FastS_PartitionMap::RecalcPartCnt(uint32_t partid)
+{
+    uint32_t  maxparts = 0;
+    uint32_t  parts = 0;
+    for (FastS_EngineBase *  engine = _partitions[partid]._engines;
+         engine != NULL; engine = engine->_nextpart) {
+        maxparts = imax(engine->_reported._maxParts, maxparts);
+        parts = imax(engine->_reported._actParts, parts);
+    }
+    if (_partitions[partid]._maxpartsNow != maxparts) {
+        _childmaxpartsNow += maxparts - _partitions[partid]._maxpartsNow;
+        _partitions[partid]._maxpartsNow = maxparts;
+        if (_childmaxpartsNow > _childmaxpartsSinceReload)
+            _childmaxpartsSinceReload = _childmaxpartsNow;
+    }
+    if (_partitions[partid]._parts != parts) {
+        _childparts += parts - _partitions[partid]._parts;
+        _partitions[partid]._parts = parts;
+    }
+}
+
+
+void
+FastS_PartitionMap::LinkIn(FastS_EngineBase *engine)
+{
+    uint32_t partid = engine->_partid - _first_partition;
+
+    FastS_assert(partid < GetSize());
+    FastS_assert(engine->_nextpart == NULL);
+    FastS_assert(engine->_prevpart == NULL);
+    FastS_PartitionMap::Partition & part = _partitions[partid];
+    engine->_nextpart = part._engines;
+    if (engine->_nextpart != NULL)
+        engine->_nextpart->_prevpart = engine;
+    part._engines = engine;
+    part._maxnodesNow += engine->_reported._maxNodes;
+    part._maxnodesSinceReload = std::max(part._maxnodesSinceReload, part._maxnodesNow);
+    part._nodes += engine->_reported._actNodes;
+    _childmaxnodesNow += engine->_reported._maxNodes;
+    _childmaxnodesSinceReload = std::max(_childmaxnodesSinceReload, _childmaxnodesNow);
+    _childnodes += engine->_reported._actNodes;
+    if (part._maxpartsNow <= engine->_reported._maxParts) {
+        _childmaxpartsNow += engine->_reported._maxParts
+                             - part._maxpartsNow;
+        _childmaxpartsSinceReload += std::max(_childmaxpartsSinceReload, _childmaxpartsNow);
+        part._maxpartsNow = engine->_reported._maxParts;
+    }
+    if (part._parts < engine->_reported._actParts) {
+        _childparts += engine->_reported._actParts - part._parts;
+        part._parts = engine->_reported._actParts;
+    }
+}
+
+
+void
+FastS_PartitionMap::LinkOut(FastS_EngineBase *engine)
+{
+    uint32_t partid = engine->_partid - _first_partition;
+
+    FastS_assert(partid < GetSize());
+    if (engine->_nextpart != NULL)
+        engine->_nextpart->_prevpart = engine->_prevpart;
+    if (engine->_prevpart != NULL)
+        engine->_prevpart->_nextpart = engine->_nextpart;
+    if (_partitions[partid]._engines == engine)
+        _partitions[partid]._engines = engine->_nextpart;
+
+    _partitions[partid]._maxnodesNow -= engine->_reported._maxNodes;
+    _partitions[partid]._nodes  -= engine->_reported._actNodes;
+    _childmaxnodesNow -= engine->_reported._maxNodes;
+    _childnodes -= engine->_reported._actNodes;
+    if (_partitions[partid]._maxpartsNow <= engine->_reported._maxParts ||
+        _partitions[partid]._parts <= engine->_reported._actParts)
+        RecalcPartCnt(partid);
+
+    engine->_nextpart = NULL;
+    engine->_prevpart = NULL;
+}
+
+//--------------------------------------------------------------------------
+
+FastS_PlainDataSet::MHPN_log_t::MHPN_log_t()
+    : _cnt(0),
+      _incompleteCnt(0),
+      _fuzzyCnt(0)
+{
+}
+
+//--------------------------------------------------------------------------
+
+void
+FastS_PlainDataSet::InsertEngine(FastS_EngineBase *engine)
+{
+    _enginesArray.Add(engine);
+}
+
+FastS_EngineBase *
+FastS_PlainDataSet::ExtractEngine()
+{
+    return _enginesArray.Extract();
+}
+
+FastS_PlainDataSet::FastS_PlainDataSet(FastS_AppContext *appCtx,
+                                       FastS_DataSetDesc *desc)
+    : FastS_DataSetBase(appCtx, desc),
+      _partMap(desc),
+      _stateOfRows(_partMap.getNumRows(), 1.0, desc->GetQueryDistributionMode().getLatencyDecayRate()),
+      _MHPN_log(),
+      _slowQueryLimitFactor(desc->GetSlowQueryLimitFactor()),
+      _slowQueryLimitBias(desc->GetSlowQueryLimitBias()),
+      _slowDocsumLimitFactor(desc->GetSlowDocsumLimitFactor()),
+      _slowDocsumLimitBias(desc->GetSlowDocsumLimitBias()),
+      _monitorInterval(desc->getMonitorInterval()),
+      _higherCoverageMaxSearchWait(desc->getHigherCoverageMaxSearchWait()),
+      _higherCoverageMinSearchWait(desc->getHigherCoverageMinSearchWait()),
+      _higherCoverageBaseSearchWait(desc->getHigherCoverageBaseSearchWait()),
+      _minimalSearchCoverage(desc->getMinimalSearchCoverage()),
+      _higherCoverageMaxDocSumWait(desc->getHigherCoverageMaxDocSumWait()),
+      _higherCoverageMinDocSumWait(desc->getHigherCoverageMinDocSumWait()),
+      _higherCoverageBaseDocSumWait(desc->getHigherCoverageBaseDocSumWait()),
+      _minimalDocSumCoverage(desc->getMinimalDocSumCoverage()),
+      _maxHitsPerNode(desc->GetMaxHitsPerNode()),
+      _estimateParts(desc->GetEstimateParts()),
+      _estimatePartCutoff(desc->GetEstPartCutoff()),
+      _queryDistributionMode(desc->GetQueryDistributionMode()),
+      _randState()
+{
+    uint32_t seed = 0;
+    const char *hostname = vespalib::HostName::get().c_str();
+    unsigned const char *p = reinterpret_cast<unsigned const char *>(hostname);
+
+    if (p != NULL) {
+        while (*p != '\0') {
+            seed = (seed << 7) + *p + (seed >> 25);
+            p++;
+        }
+    }
+    seed ^= _createtime.GetSeconds();
+    seed ^= _createtime.GetMicroSeconds();
+    _randState.srand48(seed);
+}
+
+
+FastS_PlainDataSet::~FastS_PlainDataSet()
+{
+}
+
+
+void
+FastS_PlainDataSet::UpdateMaxHitsPerNodeLog(bool incomplete, bool fuzzy)
+{
+    LockDataset();
+    _MHPN_log._cnt++;
+    if (incomplete)
+        _MHPN_log._incompleteCnt++;
+    if (fuzzy)
+        _MHPN_log._fuzzyCnt++;
+    UnlockDataset();
+}
+
+
+bool
+FastS_PlainDataSet::RefCostUseNewEngine(FastS_EngineBase *oldEngine,
+                                        FastS_EngineBase *newEngine,
+                                        unsigned int *oldCount)
+{
+    if (oldEngine->_totalrefcost + oldEngine->_config._unitrefcost >
+        newEngine->_totalrefcost + newEngine->_config._unitrefcost) {
+        *oldCount = 1;
+        return true;
+    }
+    if (oldEngine->_totalrefcost + oldEngine->_config._unitrefcost <
+        newEngine->_totalrefcost + newEngine->_config._unitrefcost)
+        return false;
+    /* Use random generator for tie breaker */
+    (*oldCount)++;
+    return ((_randState.lrand48() % *oldCount) == 0);
+}
+
+void
+FastS_PlainDataSet::updateSearchTime(double searchTime, uint32_t rowId)
+{
+    LockDataset();
+    _stateOfRows.updateSearchTime(searchTime, rowId);
+    UnlockDataset();
+}
+
+uint32_t
+FastS_PlainDataSet::getRandomWeightedRow() const
+{
+    return _stateOfRows.getRandomWeightedRow();
+}
+
+
+bool
+FastS_PlainDataSet::UseNewEngine(FastS_EngineBase *oldEngine,
+                                 FastS_EngineBase *newEngine,
+                                 unsigned int *oldCount)
+{
+    /*
+     * If old engine has used _indexSwitchMinSearchGrace seconds
+     * of grace period then select new engine if it has used
+     * less grace period.
+     */
+    if (!EngineDocStampOK(oldEngine->_reported._docstamp) &&
+        (EngineDocStampOK(newEngine->_reported._docstamp)))
+    {
+        *oldCount = 1;
+        return true;
+    }
+
+    /*
+     * If new engine has used _indexSwitchMinSearchGrace seconds
+     * of grace period then select old engine if it has used
+     * less grace period.
+     */
+    if (!EngineDocStampOK(newEngine->_reported._docstamp) &&
+        (EngineDocStampOK(oldEngine->_reported._docstamp)))
+    {
+        return false;
+    }
+
+    return RefCostUseNewEngine(oldEngine, newEngine, oldCount);
+}
+
+using namespace FastS_QueryDistribution;
+
+FastS_EngineBase *
+FastS_PlainDataSet::getPartition(const FastOS_Mutex & dsMutex, uint32_t partindex, uint32_t rowid)
+{
+    (void) dsMutex;
+    FastS_EngineBase*  ret = NULL;
+
+    if (IsValidPartIndex_HasLock(partindex)) {
+        for (FastS_EngineBase* iter = _partMap._partitions[partindex]._engines;
+             iter != NULL && ret == NULL;
+             iter = iter->_nextpart) {
+
+            // NB: cost race condition
+
+            if (!iter->IsRealBad() &&
+                EngineDocStampOK(iter->_reported._docstamp) &&
+                iter->_config._confRowID == rowid) {
+                ret = iter;
+            }
+        }
+    }
+
+    if (ret != NULL) {
+        ret->AddCost();
+    }
+    return ret;
+}
+
+size_t
+FastS_PlainDataSet::countNodesUpInRow_HasLock(uint32_t rowid)
+{
+    size_t count = 0;
+    const size_t numParts = _partMap.GetSize();
+    for (size_t partindex = 0; partindex < numParts; ++partindex) {
+        for (FastS_EngineBase* iter = _partMap._partitions[partindex]._engines;
+             iter != NULL;
+             iter = iter->_nextpart)
+        {
+            if (!iter->IsRealBad() &&
+                EngineDocStampOK(iter->_reported._docstamp) &&
+                iter->_config._confRowID == rowid)
+            {
+                ++count;
+                break;
+            }
+        }
+    }
+    return count;
+}
+
+FastS_EngineBase *
+FastS_PlainDataSet::getPartition(const FastOS_Mutex & dsMutex, uint32_t partindex)
+{
+    (void) dsMutex;
+    FastS_EngineBase*  ret = NULL;
+    unsigned int oldCount = 1;
+    unsigned int engineCount = 0;
+
+    if (IsValidPartIndex_HasLock(partindex)) {
+        for (FastS_EngineBase* iter = _partMap._partitions[partindex]._engines;
+             iter != NULL;
+             iter = iter->_nextpart) {
+
+            // NB: cost race condition
+
+            if (!iter->IsRealBad() &&
+                (iter->_config._unitrefcost > 0) &&
+                EngineDocStampOK(iter->_reported._docstamp))
+            {
+                engineCount++;
+                if (ret == NULL || UseNewEngine(ret, iter, &oldCount))
+                    ret = iter;
+            }
+        }
+    }
+
+    if (engineCount < getMPP()) {
+        ret = NULL;
+    }
+    if (ret != NULL) {
+        ret->AddCost();
+    }
+    return ret;
+}
+
+FastS_EngineBase *
+FastS_PlainDataSet::getPartitionMLD(const FastOS_Mutex & dsMutex, uint32_t partindex, bool mld)
+{
+    (void) dsMutex;
+    FastS_EngineBase*  ret = NULL;
+    unsigned int oldCount = 1;
+    if (partindex < _partMap._num_partitions) {
+        FastS_EngineBase* iter;
+        for (iter = _partMap._partitions[partindex]._engines; iter != NULL; iter = iter->_nextpart) {
+            // NB: cost race condition
+
+            if (!iter->IsRealBad() &&
+                iter->_reported._mld == mld &&
+                (iter->_config._unitrefcost > 0) &&
+                EngineDocStampOK(iter->_reported._docstamp) &&
+                (ret == NULL || UseNewEngine(ret, iter, &oldCount)))
+            {
+                ret = iter;
+            }
+        }
+    } else {
+        LOG(error, "Couldn't fetch partition data: Partition ID too big, partindex=%x _partMap._num_partitions=%x", partindex, _partMap._num_partitions);
+    }
+    if (ret != NULL) {
+        ret->AddCost();
+    }
+    return ret;
+}
+
+FastS_EngineBase *
+FastS_PlainDataSet::getPartitionMLD(const FastOS_Mutex & dsMutex, uint32_t partindex, bool mld, uint32_t rowid)
+{
+    (void) dsMutex;
+    FastS_EngineBase*  ret = NULL;
+    unsigned int oldCount = 1;
+
+    if (partindex < _partMap._num_partitions) {
+        FastS_EngineBase* iter;
+        for (iter = _partMap._partitions[partindex]._engines; iter != NULL; iter = iter->_nextpart) {
+            // NB: cost race condition
+            if (!iter->IsRealBad() &&
+                (iter->_reported._mld == mld) &&
+                (iter->_config._confRowID == rowid) &&
+                EngineDocStampOK(iter->_reported._docstamp) &&
+                (ret == NULL || UseNewEngine(ret, iter, &oldCount)))
+            {
+                ret = iter;
+            }
+        }
+    } else {
+        LOG(error, "Couldn't fetch partition data: Partition ID too big, partindex=%x _partMap._num_partitions=%x", partindex, _partMap._num_partitions);
+    }
+    if (ret != NULL) {
+        ret->AddCost();
+    }
+    return ret;
+}
+
+
+std::vector<FastS_EngineBase *>
+FastS_PlainDataSet::getPartEngines(uint32_t partition)
+{
+    typedef FastS_EngineBase EB;
+    typedef std::vector<EB *> EBV;
+    EBV partEngines;
+    LockDataset();
+    for (FastS_EngineBase *iter = _partMap._partitions[partition]._engines; iter != NULL; iter = iter->_nextpart) {
+        partEngines.push_back(iter);
+    }
+    UnlockDataset();
+    return partEngines;
+}
+
+
+void
+FastS_PlainDataSet::LinkInPart_HasLock(FastS_EngineBase *engine)
+{
+    if (engine->GetPartID() == FastS_NoID32())
+        return;
+
+    _partMap.LinkIn(engine);
+}
+
+
+void
+FastS_PlainDataSet::LinkOutPart_HasLock(FastS_EngineBase *engine)
+{
+    if (engine->GetPartID() == FastS_NoID32())
+        return;
+
+    _partMap.LinkOut(engine);
+}
+
+
+uint32_t
+FastS_PlainDataSet::CalculateQueueLens_HasLock(uint32_t &dispatchnodes)
+{
+    uint32_t partindex;
+    uint32_t equeueLen;
+    uint32_t pqueueLen;
+    FastS_EngineBase *eng;
+    uint32_t pdispatchnodes;
+    uint32_t dupnodes;
+
+    uint32_t queueLen = 0;
+    dispatchnodes = 1;
+    for (partindex = 0; partindex < _partMap._num_partitions ; partindex++) {
+        eng = _partMap._partitions[partindex]._engines;
+        if (eng != NULL) {
+            pqueueLen = eng->GetQueueLen();
+            pdispatchnodes = eng->GetDispatchers();
+            dupnodes = 1;
+            eng = eng->_nextpart;
+            while (eng != NULL) {
+                equeueLen = eng->GetQueueLen();
+                if (equeueLen < pqueueLen)
+                    pqueueLen = equeueLen;
+                pdispatchnodes += eng->GetDispatchers();
+                dupnodes++;
+                eng = eng->_nextpart;
+            }
+            if (pqueueLen > queueLen)
+                queueLen = pqueueLen;
+            if (dispatchnodes * dupnodes < pdispatchnodes)
+                dispatchnodes = pdispatchnodes / dupnodes;
+        }
+    }
+    return queueLen;
+}
+
+namespace {
+struct CheckReady {
+    bool allReady;
+    CheckReady() : allReady(true) {}
+
+    inline void operator()(FastS_EngineBase* engine) {
+        allReady &= engine->IsReady();
+    }
+};
+
+} //anonymous namespace
+
+bool
+FastS_PlainDataSet::AreEnginesReady()
+{
+
+    // We don't need to lock things here, since the engine list
+    // is non-mutable during datasetcollection lifetime.
+     return _enginesArray.ForEach( CheckReady() ).allReady;
+}
+
+void
+FastS_PlainDataSet::Ping()
+{
+#ifdef FNET_SANITY_CHECKS
+    FastOS_Time beforePing;
+    beforePing.SetNow();
+#endif
+    _enginesArray.ForEach(
+            std::mem_fun(&FastS_EngineBase::Ping) );
+#ifdef FNET_SANITY_CHECKS
+    double pingTime = beforePing.MilliSecsToNow();
+    if (pingTime > 100.0) {
+        LOG(warning, "SANITY: Dataset (%d) ping time: %.2f ms", GetID(), pingTime);
+    }
+#endif
+}
+
+
+ChildInfo
+FastS_PlainDataSet::getChildInfo() const
+{
+    ChildInfo r;
+    r.maxNodes    = _partMap._childmaxnodesSinceReload;
+    r.activeNodes = _partMap._childnodes;
+    r.maxParts    = _partMap._childmaxpartsSinceReload;
+    r.activeParts = _partMap._childparts;
+    r.activeDocs  = getActiveDocs();
+    return r;
+}
+
+bool
+FastS_PlainDataSet::IsValidPartIndex_HasLock(uint32_t partindex) {
+    if (partindex < _partMap._num_partitions) {
+        return true;
+    } else {
+        LOG(error, "Couldn't fetch partition data: Partition ID too big, partindex=%x _partMap._num_partitions=%x", partindex, _partMap._num_partitions);
+        return false;
+    }
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h
new file mode 100644
index 00000000000..a8d849469a4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h
@@ -0,0 +1,229 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <list>
+
+#include <vespa/fastos/fastos.h>
+#include "child_info.h"
+#include <vespa/searchcore/fdispatch/search/dataset_base.h>
+#include <vespa/searchlib/util/rand48.h>
+
+#include <vespa/searchcore/fdispatch/search/partitioned_array.h>
+#include <vespa/searchcore/fdispatch/search/configdesc.h>
+
+#include <vespa/searchcore/fdispatch/search/rowstate.h>
+#include <vespa/fnet/task.h>
+
+//----------------------------------------------------------------
+// class holding information about a set of partitions
+//----------------------------------------------------------------
+class FastS_PartitionMap
+{
+public:
+
+    //----------------------------------------------------------------
+    // class holding information about a single partition
+    //----------------------------------------------------------------
+    class Partition
+    {
+
+    public:
+        FastS_EngineBase *_engines;
+        uint32_t          _maxnodesNow;
+        uint32_t          _maxnodesSinceReload;
+        uint32_t          _nodes;
+        uint32_t          _maxpartsNow;
+        uint32_t          _maxpartsSinceReload;
+        uint32_t          _parts;
+
+    public:
+        Partition();
+        ~Partition();
+    private:
+        Partition(const Partition &);
+        Partition& operator=(const Partition &);
+    };
+
+
+public:
+    Partition *_partitions;
+    uint32_t   _partBits;
+    uint32_t   _rowBits;
+    uint32_t   _num_partitions;	 // Number of partitions (active)
+    uint32_t   _first_partition;   // From partitions-file 'firstpart' (active)
+    uint32_t   _minchildparts;	 // Minimum partitions live to avoid tempfail
+    uint32_t   _maxNodesDownPerFixedRow;
+    bool       _useRoundRobinForFixedRow;
+    uint32_t   _childnodes;
+    uint32_t   _childmaxnodesNow;
+    uint32_t   _childmaxnodesSinceReload;
+    uint32_t   _childparts;
+    uint32_t   _childmaxpartsNow;
+    uint32_t   _childmaxpartsSinceReload;
+    uint32_t   _mpp;		// Number of engines needed per partition
+
+    std::vector<uint32_t> _numPartitions;
+
+public:
+    FastS_PartitionMap(FastS_DataSetDesc *desc);
+    ~FastS_PartitionMap();
+
+    void RecalcPartCnt(uint32_t partid);
+    void LinkIn(FastS_EngineBase *engine);
+    void LinkOut(FastS_EngineBase *engine);
+
+    uint32_t GetSize() { return _num_partitions; }
+
+    uint32_t getNumRows() const { return _maxRows + 1; }
+    uint32_t getNumPartitions(size_t rowId) { return _numPartitions[rowId]; }
+private:
+    FastS_PartitionMap(const FastS_PartitionMap &);
+    FastS_PartitionMap& operator=(const FastS_PartitionMap &);
+    uint32_t   _maxRows;
+
+};
+
+//---------------------------------------------------------------------------
+
+class FastS_PlainDataSet : public FastS_DataSetBase
+{
+    friend class FastS_NodeManager;
+private:
+    FastS_PlainDataSet(const FastS_PlainDataSet &);
+    FastS_PlainDataSet& operator=(const FastS_PlainDataSet &);
+
+public:
+
+    //----------------------------------------------------------------
+    // Max Hits Per Node Stats
+    //----------------------------------------------------------------
+
+    class MHPN_log_t
+    {
+    public:
+        uint32_t _cnt;            // # times maxHitsPerNode affected # hits requested
+        uint32_t _incompleteCnt;  // # times maxHitsPerNode caused too few hits
+        uint32_t _fuzzyCnt;       // # times maxHitsPerNode may have caused wrong hits
+
+        MHPN_log_t();
+    };
+
+protected:
+    FastS_PartitionMap     _partMap;
+    fdispatch::StateOfRows _stateOfRows;
+    MHPN_log_t   _MHPN_log;
+    double       _slowQueryLimitFactor;
+    double       _slowQueryLimitBias;
+    double       _slowDocsumLimitFactor;
+    double       _slowDocsumLimitBias;
+    double	 _monitorInterval;
+    double	 _higherCoverageMaxSearchWait;
+    double	 _higherCoverageMinSearchWait;
+    double	 _higherCoverageBaseSearchWait;
+    double	 _minimalSearchCoverage;
+    double	 _higherCoverageMaxDocSumWait;
+    double	 _higherCoverageMinDocSumWait;
+    double	 _higherCoverageBaseDocSumWait;
+    double	 _minimalDocSumCoverage;
+    uint32_t     _maxHitsPerNode;     // Max hits requested from single node
+    uint32_t     _estimateParts;      // number of partitions used for estimate
+    uint32_t     _estimatePartCutoff; // First partition not used for estimate
+
+    FastS_DataSetDesc::QueryDistributionMode  _queryDistributionMode;
+    //all engines in this dataset
+    FastS_QueryDistribution::PartitionedArray _enginesArray;
+    search::Rand48 _randState;
+
+    void InsertEngine(FastS_EngineBase *engine);
+    FastS_EngineBase *ExtractEngine();
+    bool RefCostUseNewEngine(FastS_EngineBase *oldEngine, FastS_EngineBase *newEngine, unsigned int *oldCount);
+    bool UseNewEngine(FastS_EngineBase *oldEngine, FastS_EngineBase *newEngine, unsigned int *oldCount);
+
+    bool IsValidPartIndex_HasLock(uint32_t partindex);
+public:
+    FastS_PlainDataSet(FastS_AppContext *appCtx, FastS_DataSetDesc *desc);
+    virtual ~FastS_PlainDataSet();
+
+    bool useFixedRowDistribution() const {
+        return _queryDistributionMode == FastS_DataSetDesc::QueryDistributionMode::FIXEDROW;
+    }
+    uint32_t getNumRows() const { return _partMap.getNumRows(); }
+    uint32_t getNumPartitions(size_t rowId) { return _partMap.getNumPartitions(rowId); }
+    uint32_t GetRowBits(void) const { return _partMap._rowBits; }
+    uint32_t GetPartBits(void) const { return _partMap._partBits; }
+    uint32_t GetFirstPart(void) const { return _partMap._first_partition; }
+    uint32_t GetLastPart(void) const {
+        return _partMap._first_partition + _partMap._num_partitions;
+    }
+    uint32_t GetPartitions(void) const { return _partMap._num_partitions; }
+    uint32_t GetEstimateParts(void) const { return _estimateParts; }
+    uint32_t GetEstimatePartCutoff(void) const { return _estimatePartCutoff; }
+    uint32_t GetMaxHitsPerNode(void) const { return _maxHitsPerNode; }
+    double GetSlowQueryLimitFactor(void) const { return _slowQueryLimitFactor; }
+    double GetSlowQueryLimitBias(void) const { return _slowQueryLimitBias; }
+    double GetSlowDocsumLimitFactor(void) const { return _slowDocsumLimitFactor; }
+    double GetSlowDocsumLimitBias(void) const { return _slowDocsumLimitBias; }
+    bool GetTempFail(void) const { return _partMap._childparts < _partMap._minchildparts; }
+    void UpdateMaxHitsPerNodeLog(bool incomplete, bool fuzzy);
+    uint32_t getMaxNodesDownPerFixedRow() const { return _partMap._maxNodesDownPerFixedRow; }
+    uint32_t useRoundRobinForFixedRow() const { return _partMap._useRoundRobinForFixedRow; }
+    double getMinGroupCoverage() const { return _queryDistributionMode.getMinGroupCoverage(); }
+    void updateSearchTime(double searchTime, uint32_t rowId);
+    void updateActiveDocs_HasLock(uint32_t rowId, PossCount newVal, PossCount oldVal) {
+        _stateOfRows.updateActiveDocs(rowId, newVal, oldVal);
+    }
+    PossCount getActiveDocs() const { return _stateOfRows.getActiveDocs(); }
+    uint32_t getRandomWeightedRow() const;
+
+    FastS_EngineBase * getPartition(const FastOS_Mutex & lock, uint32_t partid);
+    FastS_EngineBase * getPartition(const FastOS_Mutex & lock, uint32_t partid, uint32_t rowid);
+
+    size_t countNodesUpInRow_HasLock(uint32_t rowid);
+
+    FastS_EngineBase * getPartitionMLD(const FastOS_Mutex & lock, uint32_t partid, bool mld);
+    FastS_EngineBase * getPartitionMLD(const FastOS_Mutex & lock, uint32_t partid, bool mld, uint32_t rowid);
+
+    std::vector<FastS_EngineBase *> getPartEngines(uint32_t partition);
+
+    void LinkInPart_HasLock(FastS_EngineBase *engine);
+    void LinkOutPart_HasLock(FastS_EngineBase *engine);
+
+    virtual ChildInfo getChildInfo() const;
+
+    uint32_t getMPP(void) const { return _partMap._mpp; }
+    double getMonitorInterval(void) const { return _monitorInterval; }
+    double getHigherCoverageMaxSearchWait(void) const { return _higherCoverageMaxSearchWait; }
+    double getHigherCoverageMinSearchWait(void) const { return _higherCoverageMinSearchWait; }
+    double getHigherCoverageBaseSearchWait(void) const { return _higherCoverageBaseSearchWait; }
+    double getMinimalSearchCoverage(void) const { return _minimalSearchCoverage; }
+    double getHigherCoverageMaxDocSumWait(void) const { return _higherCoverageMaxDocSumWait; }
+    double getHigherCoverageMinDocSumWait(void) const { return _higherCoverageMinDocSumWait; }
+    double getHigherCoverageBaseDocSumWait(void) const { return _higherCoverageBaseDocSumWait; }
+    double getMinimalDocSumCoverage(void) const { return _minimalDocSumCoverage; }
+
+    // API
+    //----
+    virtual uint32_t CalculateQueueLens_HasLock(uint32_t &dispatchnodes);
+    virtual bool AreEnginesReady();
+    virtual void Ping();
+
+    // Downcast
+    //---------
+    virtual FastS_PlainDataSet * GetPlainDataSet(void) { return this; }
+
+    template <class FUN>
+    FUN ForEachEngine(FUN fun) {
+        return _enginesArray.ForEach(fun);
+    }
+
+    static bool EngineDocStampOK(time_t haveDocStamp) { return (haveDocStamp != 0); }
+
+    void EnginePartIDChanged_HasLock(FastS_EngineBase* engine, uint32_t oldID) {
+        _enginesArray.EnginePartitionIDChanged(engine, oldID);
+    }
+
+    void UseDeterministicQueryDistribution(bool);
+};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h b/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h
new file mode 100644
index 00000000000..53671e58de3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+
+struct PossCount {
+    bool valid;
+    uint64_t count;
+
+    PossCount() : valid(false), count(0) {}
+
+    bool operator != (const PossCount& other) {
+        return (valid != other.valid) || (count != other.count);
+    }
+};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp
new file mode 100644
index 00000000000..a9462476e78
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp
@@ -0,0 +1,176 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1999-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".query");
+#include <vespa/searchcore/util/log.h>
+#include <vespa/searchcore/fdispatch/search/query.h>
+#include <vespa/searchlib/common/transport.h>
+#include <vespa/searchlib/parsequery/simplequerystack.h>
+
+
+/** Marks as empty
+ */
+FastS_query::FastS_query(void)
+    : _dataset(0),
+      _flags(0),
+      _stackDump(),
+      _sortSpec(NULL),
+      _groupSpec(),
+      _location(NULL),
+      _rankProperties(),
+      _featureOverrides()
+{
+};
+
+FastS_query::FastS_query(const search::docsummary::GetDocsumArgs &docsumArgs)
+    : _dataset(0),                        // not known
+      _flags(docsumArgs.GetQueryFlags()),
+      _stackDump(docsumArgs.getStackDump()),
+      _sortSpec(NULL),                    // not known
+      _groupSpec(),                   // not known
+      _location(NULL),
+      _rankProperties(docsumArgs.rankProperties()),
+      _featureOverrides(docsumArgs.featureOverrides())
+{
+    // _query = search::SimpleQueryStack::StackbufToString(docsumArgs.getStackDump());
+    if (docsumArgs.getLocation().size() > 0) {
+        _location = strdup(docsumArgs.getLocation().c_str());
+    }
+}
+
+
+void
+FastS_query::SetStackDump(const vespalib::stringref &stackRef)
+{
+    _stackDump = stackRef;
+}
+
+const char *
+FastS_query::getPrintableQuery()
+{
+    if (_printableQuery.empty()) {
+        _printableQuery = search::SimpleQueryStack::StackbufToString(_stackDump);
+    }
+    return  _printableQuery.c_str();
+}
+
+FastS_query::~FastS_query(void)
+{
+}
+
+
+void
+FastS_query::SetDataSet(uint32_t dataset)
+{
+    _dataset = dataset;
+}
+
+unsigned int
+FastS_query::StackDumpHashKey() const
+{
+    unsigned int res = 0;
+    const unsigned char *p;
+    const unsigned char *e;
+    p = (const unsigned char *) _stackDump.begin();
+    e = (const unsigned char *) _stackDump.end();
+    while (p != e) {
+        res = (res << 7) + (res >> 25) + *p;
+        p++;
+    }
+    return res;
+}
+
+unsigned int
+FastS_query::HashKey(void) const
+{
+    unsigned int res;
+
+    res = StackDumpHashKey();
+    res += hash_str_check((const unsigned char *) _location.c_str());
+    res += _rankProperties.hashCode();
+    res += _featureOverrides.hashCode();
+    res += (_flags & search::fs4transport::QFLAG_CACHE_MASK);
+    return res;
+}
+
+
+bool
+FastS_query::Similar(const FastS_query &other) const
+{
+    if (((_flags ^ other._flags) &
+         search::fs4transport::QFLAG_CACHE_MASK) != 0 ||
+        _stackDump.size() != other._stackDump.size() ||
+        !cmp_str_ref(_stackDump, other._stackDump) ||
+        !(_location == other._location) ||
+        !(_rankProperties == other._rankProperties) ||
+        !(_featureOverrides == other._featureOverrides))
+        return false;
+    return true;
+}
+
+bool
+FastS_query::Equal(const FastS_query &other) const
+{
+    if (_dataset != other._dataset ||
+        ((_flags ^ other._flags) &
+         search::fs4transport::QFLAG_CACHE_MASK) != 0 ||
+        _stackDump.size() != other._stackDump.size() ||
+        !cmp_str_ref(_stackDump, other._stackDump) ||
+        !(_sortSpec == other._sortSpec) ||
+        _groupSpec != other._groupSpec ||
+        !(_location == other._location) ||
+        !(_rankProperties == other._rankProperties) ||
+        !(_featureOverrides == other._featureOverrides))
+        return false;
+    return true;
+}
+
+
+namespace
+{
+
+// This is ugly, somebody please find a better way.
+
+class SizeCollector : public search::fef::IPropertiesVisitor
+{
+    static const size_t _stringFuzz = 15; // Compensate for malloc() waste
+    static const size_t _vectorFuzz = 15;
+    static const size_t _mapFuzz = 15;
+    size_t _size;
+public:
+    SizeCollector(void)
+        : _size(0)
+    {
+    }
+
+    virtual void
+    visitProperty(const search::fef::Property::Value &key,
+                  const search::fef::Property &values)
+    {
+        // Account for std::map element size
+        _size += _mapFuzz;
+        // Account for key string size
+        _size += key.size() + _stringFuzz;
+        size_t numValues = values.size();
+        // Account for value vector size
+        if (numValues > 0) {
+            _size += numValues * sizeof(search::fef::Property::Value) + _vectorFuzz;
+            for (size_t i = 0; i < numValues; ++i) {
+                // Account for string sizes in value vector
+                const search::fef::Property::Value &str = values.getAt(i);
+                _size += str.size() + _stringFuzz;
+            }
+        }
+    }
+
+    size_t
+    getSize(void) const
+    {
+        return _size;
+    }
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.h b/searchcore/src/vespa/searchcore/fdispatch/search/query.h
new file mode 100644
index 00000000000..d2f5b094a5f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/query.h
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1999-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/searchlib/fef/properties.h>
+#include <vespa/searchsummary/docsummary/getdocsumargs.h>
+
+class FastS_query
+{
+public:
+    uint32_t _dataset;
+    uint32_t _flags;
+    vespalib::string _stackDump;
+    vespalib::string _printableQuery;
+    vespalib::string _sortSpec;
+    std::vector<char> _groupSpec; // this is binary
+    vespalib::string  _location;
+    search::fef::Properties _rankProperties;
+    search::fef::Properties _featureOverrides;
+
+    FastS_query(const FastS_query &other);
+    FastS_query &operator=(const FastS_query &other);
+public:
+    FastS_query(void);
+    FastS_query(const search::docsummary::GetDocsumArgs &docsumArgs);
+    ~FastS_query(void);
+
+    void SetStackDump(const vespalib::stringref& stackDump);
+    void SetSortSpec(const char *spec) { _sortSpec = spec; }
+    void SetLocation(const char *loc) { _location = loc; }
+    void SetRankProperties(const search::fef::Properties &rp) { _rankProperties = rp; }
+    void SetFeatureOverrides(const search::fef::Properties &fo) { _featureOverrides = fo; }
+    void SetDataSet(uint32_t dataset);
+    void SetQueryFlags(uint32_t flags) { _flags = flags; }
+    void SetFlag(uint32_t flag) { _flags |= flag; }
+    void ClearFlag(uint32_t flag) { _flags &= ~flag; }
+    const vespalib::string &getStackDump() const { return _stackDump; }
+    const char *GetSortSpec() const { return _sortSpec.c_str(); }
+    const char *GetLocation() const { return _location.c_str(); }
+    const search::fef::Properties &GetRankProperties() const { return _rankProperties; }
+    const search::fef::Properties &GetFeatureOverrides() const { return _featureOverrides; }
+
+    uint32_t GetQueryFlags(void) const { return _flags; }
+    const char *getPrintableQuery();
+    bool IsFlagSet(uint32_t flag) const { return (_flags & flag) != 0; }
+    bool Similar(const FastS_query &other) const;
+    bool Equal(const FastS_query &other) const;
+
+    unsigned int StackDumpHashKey() const;
+    unsigned int HashKey(void) const;
+
+
+private:
+    static unsigned int hash_str_check(const unsigned char *pt)
+    {
+        if (pt == NULL)
+            return 0;
+
+        unsigned int res = 0;
+        for (; *pt != 0; pt++)
+            res = (res << 7) + (res >> 25) + *pt;
+        return res;
+    }
+    static bool cmp_str_check(const char *a, const char *b)
+    {
+        if (a == NULL && b == NULL)
+            return true;
+        if (a == NULL || b == NULL)
+            return false;
+        return (strcmp(a, b) == 0);
+    }
+    static bool cmp_str_ref(const vespalib::stringref &a,
+                            const vespalib::stringref &b)
+    {
+        return (a.size() == b.size() &&
+                memcmp(a.c_str(), b.c_str(), a.size()) == 0);
+    }
+};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp
new file mode 100644
index 00000000000..938e0605e0e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp
@@ -0,0 +1,169 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".search.querycacheutil");
+#include <vespa/searchcore/util/log.h>
+
+#include <vespa/searchlib/common/transport.h>
+#include <vespa/searchlib/parsequery/simplequerystack.h>
+
+#include <vespa/searchlib/common/sortdata.h>
+
+#include <vespa/searchcore/fdispatch/search/querycacheutil.h>
+
+using search::common::SortData;
+
+uint32_t FastS_QueryCacheUtil::_systemMaxHits;
+uint32_t FastS_QueryCacheUtil::_maxOffset = 4000;
+
+
+FastS_QueryCacheUtil::FastS_QueryCacheUtil()
+    : _startTime(),
+      _userMaxHits(0),
+      _alignedMaxHits(0),
+      _alignedSearchOffset(0),
+      _ranking(),
+      _dateTime(0),
+      _forceStrictLimits(false),
+      _query(),
+      _queryResult(),
+      _docsumsResult(),
+      _searchInfo(),
+      _alignedHitBuf(NULL),
+      _hitbuf_needfree(false),
+      _alignedHitCount(0),
+      _sortIndex(NULL),
+      _sortData(NULL),
+      _sortdata_needfree(false)
+{
+    _searchInfo._maxHits = 10;
+}
+
+
+void
+FastS_QueryCacheUtil::setSearchRequest(const search::engine::SearchRequest * request)
+{
+    _ranking  = request->ranking;
+
+    _query.SetQueryFlags(request->queryFlags);
+
+    _query.SetStackDump(request->getStackRef());
+    _query.SetSortSpec(request->sortSpec.c_str());
+    _query._groupSpec = request->groupSpec;
+    _query.SetLocation(request->location.c_str());
+    _query.SetRankProperties(request->propertiesMap.rankProperties());
+    _query.SetFeatureOverrides(request->propertiesMap.featureOverrides());
+}
+
+
+void
+FastS_QueryCacheUtil::SetupQuery(uint32_t maxhits,
+				 uint32_t offset)
+{
+    FastS_assert(_queryResult._hitbuf == NULL);
+    FastS_assert(_alignedHitBuf == NULL);
+    FastS_assert(!_hitbuf_needfree);
+    FastS_assert(_queryResult._hitCount == 0);
+    FastS_assert(_docsumsResult._fullResultCount == 0);
+    FastS_assert(_alignedHitCount == 0);
+    FastS_assert(_queryResult._totalHitCount == 0);
+    FastS_assert(_alignedMaxHits == 0);
+    FastS_assert(_alignedSearchOffset == 0);
+    FastS_assert(_docsumsResult._fullresult == NULL);
+    _searchInfo._searchOffset = offset;
+    _searchInfo._maxHits = maxhits;
+}
+
+
+void
+FastS_QueryCacheUtil::AdjustSearchParameters(uint32_t partitions)
+{
+    bool     strict = _forceStrictLimits || (partitions > 1);
+
+    if (_searchInfo._maxHits == 0) {
+        _searchInfo._searchOffset = 0;
+    }
+
+    _searchInfo._maxHits = std::min(_searchInfo._maxHits, _maxOffset + _systemMaxHits);
+    if (strict) {
+        _searchInfo._searchOffset = std::min(_searchInfo._searchOffset, _maxOffset);
+        _searchInfo._maxHits = std::min(_searchInfo._maxHits, _maxOffset + _systemMaxHits - _searchInfo._searchOffset);
+    }
+}
+
+
+void
+FastS_QueryCacheUtil::AdjustSearchParametersFinal(uint32_t partitions)
+{
+    if (IsEstimate()) {
+
+        FastS_assert(_searchInfo._searchOffset == 0);
+        FastS_assert(_searchInfo._maxHits  == 0);
+
+        _alignedSearchOffset = 0;
+        _alignedMaxHits      = 0;
+    } else {
+        _alignedSearchOffset = (partitions > 1) ? 0 : _searchInfo._searchOffset;
+        _alignedMaxHits = _searchInfo._maxHits + _searchInfo._searchOffset - _alignedSearchOffset;
+        FastS_assert(_alignedMaxHits <= _maxOffset + _systemMaxHits);
+    }
+}
+
+void
+FastS_QueryCacheUtil::DropResult(void)
+{
+    _queryResult._groupResultLen = 0;
+    _queryResult._groupResult = NULL;
+
+    if (_hitbuf_needfree) {
+        FastS_assert(_alignedHitBuf != NULL);
+        free(_alignedHitBuf);
+    }
+    if (_sortdata_needfree) {
+        FastS_assert(_sortIndex != NULL);
+        free(_sortIndex);
+    }
+    _sortIndex = NULL;
+    _sortData = NULL;
+    _sortdata_needfree = false;
+    _alignedHitBuf = NULL;
+    _queryResult._hitbuf = NULL;
+    _hitbuf_needfree = false;
+    free(_docsumsResult._fullresult);
+    _docsumsResult._fullresult = NULL;
+    _queryResult._hitCount = 0;
+    _docsumsResult._fullResultCount = 0;
+    _queryResult._totalHitCount = 0;
+    _queryResult._maxRank = std::numeric_limits<search::HitRank>::is_integer ?
+                            std::numeric_limits<search::HitRank>::min() :
+                            - std::numeric_limits<search::HitRank>::max();
+
+    _alignedHitCount = 0;
+}
+
+
+bool
+FastS_QueryCacheUtil::IsEstimate(void) const
+{
+    return _query.IsFlagSet(search::fs4transport::QFLAG_ESTIMATE);
+}
+
+
+void
+FastS_QueryCacheUtil::ForceStrictLimits(void)
+{
+    _forceStrictLimits = true;
+}
+
+
+void
+FastS_QueryCacheUtil::InitEstimateMode(void)
+{
+    _searchInfo._searchOffset = 0;
+    _searchInfo._maxHits  = 0;
+    _ranking.clear();
+    _dateTime	= 0;
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h
new file mode 100644
index 00000000000..6e9ecafcdb6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h
@@ -0,0 +1,155 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/searchcore/fdispatch/search/query.h>
+#include <vespa/searchlib/common/transport.h>
+
+#include <vespa/searchcore/fdispatch/common/search.h>
+#include <vespa/searchcore/util/log.h>
+
+class FastS_DataSetCollection;
+
+class FastS_QueryCacheUtil
+{
+private:
+    FastS_QueryCacheUtil(const FastS_QueryCacheUtil &);
+    FastS_QueryCacheUtil& operator=(const FastS_QueryCacheUtil &);
+
+    double _startTime;		// For the query
+
+    uint32_t _userMaxHits;	 // Max hits spec.d by user; NB: see _systemMaxHits
+    uint32_t _alignedMaxHits;	 // Max hits      (forwarded to engine)
+    uint32_t _alignedSearchOffset; // Search offset (forwarded to engine)
+    vespalib::string _ranking;   // ranking profile to be used
+    uint32_t _randomSeed;        // seed for random rank values
+    uint32_t _dateTime;		 // datetime used for freshness boost
+
+    bool _forceStrictLimits;      // use strict limits (offset/maxhits)
+
+    FastS_query _query;		// NB: Here it is!
+
+    FastS_QueryResult _queryResult;
+    FastS_DocsumsResult _docsumsResult;
+    FastS_SearchInfo _searchInfo;
+
+    FastS_hitresult *_alignedHitBuf;	// Hits from engine
+    bool	 _hitbuf_needfree;	// Destructor should free _hitbuf.
+    uint32_t _alignedHitCount;	// # Hits from engine
+
+    uint32_t *_sortIndex;
+    char     *_sortData; // NB: same malloc as _sortIndex
+    bool      _sortdata_needfree;
+
+public:
+    static uint32_t _systemMaxHits;
+    static uint32_t _maxOffset;
+public:
+    FastS_QueryCacheUtil();
+    bool AgeDropCheck(void);
+    void DropResult(void);
+    bool GotNoResultsYet(void) const { return _queryResult._hitbuf == NULL; }
+    uint32_t GetSearchOffset(void) const { return _searchInfo._searchOffset; }
+    uint32_t GetMaxHits(void) const { return _searchInfo._maxHits; }
+    uint32_t GetAlignedMaxHits(void) const { return _alignedMaxHits; }
+    uint32_t GetAlignedSearchOffset(void) const { return _alignedSearchOffset; }
+    const vespalib::string & GetRanking(void) const { return _ranking; }
+    uint32_t GetRandomSeed(void) const { return _randomSeed; }
+    uint32_t GetDateTime(void) const { return _dateTime; }
+    FastS_query &GetQuery(void) { return _query; }
+    const char *GetSortSpec() const { return _query.GetSortSpec(); }
+    const char *GetLocation() const { return _query.GetLocation(); }
+    bool ErrorPacketsAllowed(void) const {
+        return _query.IsFlagSet(search::fs4transport::QFLAG_ALLOW_ERRORPACKET);
+    }
+    bool ShouldDropSortData() const {
+        return _query.IsFlagSet(search::fs4transport::QFLAG_DROP_SORTDATA);
+    }
+    bool IsQueryFlagSet(uint32_t flag) const { return _query.IsFlagSet(flag); }
+    FastS_QueryResult *GetQueryResult(void) {
+        return &_queryResult;
+    }
+    FastS_DocsumsResult *GetDocsumsResult(void) { return &_docsumsResult; }
+    FastS_SearchInfo *GetSearchInfo(void) { return &_searchInfo; }
+    void SetStartTime(double timeref) { _startTime = timeref; }
+    void AdjustSearchParameters(uint32_t partitions);
+    void AdjustSearchParametersFinal(uint32_t partitions);
+    void SetupQuery(uint32_t maxhits, uint32_t offset);
+    bool IsEstimate(void) const;
+    void ForceStrictLimits(void);
+    void InitEstimateMode(void);
+    double ElapsedSecs(double now) const {
+        double ret = now - _startTime;
+        if (ret < 0.0)
+            ret = 0.0;
+        return ret;
+    }
+    void SetCoverage(uint64_t coverageDocs,
+                     uint64_t activeDocs)
+    {
+        _searchInfo._coverageDocs  = coverageDocs;
+        _searchInfo._activeDocs  = activeDocs;
+    }
+    void SetAlignedHitCount(uint32_t alignedHitCount) {
+        if (alignedHitCount > _alignedMaxHits)
+            alignedHitCount = _alignedMaxHits;
+        _alignedHitCount = alignedHitCount;
+    }
+    void CalcHitCount(void) {
+        if (_alignedHitCount + _alignedSearchOffset >
+            _searchInfo._searchOffset)
+            _queryResult._hitCount = _alignedHitCount + _alignedSearchOffset -
+                                     _searchInfo._searchOffset;
+        else
+            _queryResult._hitCount = 0;
+        if (_queryResult._hitCount > _searchInfo._maxHits)
+            _queryResult._hitCount = _searchInfo._maxHits;
+    }
+    void AllocAlignedHitBuf(void) {
+        FastS_assert(_alignedHitBuf == NULL);
+        if (_alignedHitCount != 0) {
+            _alignedHitBuf =
+                (FastS_hitresult*)malloc(sizeof(FastS_hitresult) *
+                                         _alignedHitCount);
+            _hitbuf_needfree = true;
+            _queryResult._hitbuf =
+                _alignedHitBuf + _searchInfo._searchOffset - _alignedSearchOffset;
+        }
+    }
+    void AllocSortData(uint32_t sortDataLen)
+    {
+        FastS_assert(_sortIndex == NULL && _sortData == NULL);
+        uint32_t hitcnt = _alignedHitCount;
+        if (hitcnt == 0) {
+            FastS_assert(sortDataLen == 0);
+            return;
+        }
+        void *pt = malloc((hitcnt + 1) * sizeof(uint32_t) + sortDataLen);
+        FastS_assert(pt != NULL);
+        _sortIndex = (uint32_t *) pt;
+        _sortData  = (char *)(void *)(_sortIndex + hitcnt + 1);
+        _sortdata_needfree = true;
+        if (hitcnt > _searchInfo._searchOffset) {
+            _queryResult._sortIndex =
+                _sortIndex + _searchInfo._searchOffset - _alignedSearchOffset;
+            _queryResult._sortData = _sortData;
+        }
+    }
+    uint32_t *GetSortIndex() const { return _sortIndex; }
+    char *GetSortData() const { return _sortData; }
+    FastS_hitresult *GetAlignedHitBuf(void) const { return _alignedHitBuf; }
+    FastS_hitresult *GetAlignedHitBufEnd(void) const {
+        return _alignedHitBuf + _alignedHitCount;
+    }
+    uint32_t GetAlignedHitCount(void) const { return _alignedHitCount; }
+    void SetGroupResult(const char *groupResult) {
+        _queryResult._groupResult = groupResult;
+    }
+    void SetGroupResultLen(uint32_t groupResultLen) {
+        _queryResult._groupResultLen = groupResultLen;
+    }
+    void setSearchRequest(const search::engine::SearchRequest * request);
+};
+
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp
new file mode 100644
index 00000000000..25780bd3b2a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/searchcore/fdispatch/search/rowstate.h>
+
+namespace fdispatch {
+
+void RowState::updateSearchTime(double searchTime)
+{
+    _avgSearchTime = (searchTime + (_decayRate-1)*_avgSearchTime)/_decayRate;
+}
+
+StateOfRows::StateOfRows(size_t numRows, double initialValue, double decayRate) :
+   _rows(numRows, RowState(initialValue, decayRate)),
+   _sumActiveDocs(0), _invalidActiveDocsCounter(0)
+{
+   srand48(1);
+}
+
+void
+StateOfRows::updateSearchTime(double searchTime, uint32_t rowId)
+{
+    _rows[rowId].updateSearchTime(searchTime);
+}
+
+uint32_t
+StateOfRows::getRandomWeightedRow() const
+{
+    return getWeightedNode(drand48());
+}
+
+uint32_t
+StateOfRows::getWeightedNode(double cand) const
+{
+    double sum = 0;
+    for (const RowState & rs : _rows) {
+        sum += rs.getAverageSearchTimeInverse();
+    }
+    double accum(0.0);
+    for (size_t rowId(0); (rowId + 1) < _rows.size(); rowId++) {
+        accum += _rows[rowId].getAverageSearchTimeInverse();
+        if (cand < accum/sum) {
+            return rowId;
+        }
+    }
+    return _rows.size() - 1;
+}
+
+void
+StateOfRows::updateActiveDocs(uint32_t rowId, PossCount newVal, PossCount oldVal)
+{
+    uint64_t tmp = _sumActiveDocs + newVal.count - oldVal.count;
+    _sumActiveDocs = tmp;
+    _rows[rowId].updateActiveDocs(newVal.count, oldVal.count);
+    if (newVal.valid != oldVal.valid) {
+        if (oldVal.valid) {
+            ++_invalidActiveDocsCounter;
+        } else {
+            --_invalidActiveDocsCounter;
+        }
+    }
+}
+
+PossCount
+StateOfRows::getActiveDocs() const
+{
+    PossCount r;
+    if (activeDocsValid()) {
+        r.valid = true;
+        r.count = 0;
+        for (const RowState &row : _rows) {
+            r.count = std::max(r.count, row.activeDocs());
+        }
+    }
+    return r;
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h
new file mode 100644
index 00000000000..8f2ab40ad9e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vector>
+#include <stdint.h>
+#include <stdlib.h>
+#include "poss_count.h"
+
+namespace fdispatch {
+
+/**
+ * RowState keeps track of state per row or rather group.
+ * Currently it just keeps the average searchtime as exponential decay.
+ **/
+class RowState {
+public:
+    RowState(double initialValue, double decayRate) :
+        _avgSearchTime(initialValue),
+        _decayRate(decayRate),
+        _sumActiveDocs(0)
+    { }
+    double getAverageSearchTime() const { return _avgSearchTime; }
+    double getAverageSearchTimeInverse() const { return 1.0/_avgSearchTime; }
+    void updateSearchTime(double searchTime);
+    void setAverageSearchTime(double avgSearchTime) { _avgSearchTime = avgSearchTime; }
+    uint64_t activeDocs() const { return _sumActiveDocs; }
+    void updateActiveDocs(uint64_t newVal, uint64_t oldVal) {
+        uint64_t tmp = _sumActiveDocs + newVal - oldVal;
+        _sumActiveDocs = tmp;
+    }
+private:
+    double _avgSearchTime;
+    double _decayRate;
+    uint64_t _sumActiveDocs;
+};
+
+/**
+ * StateOfRows keeps track of the state of all rows/groups.
+ * Currently used for tracking latency in groups. This latency
+ * can be used for selecting a random node with weighted probability
+ * with the intention to favor load on fast groups.
+ **/
+class StateOfRows {
+public:
+    StateOfRows(size_t numRows, double initial, double decayRate);
+    void updateSearchTime(double searchTime, uint32_t rowId);
+    const RowState & getRowState(uint32_t rowId) const { return _rows[rowId]; }
+    RowState & getRowState(uint32_t rowId) { return _rows[rowId]; }
+    uint32_t getRandomWeightedRow() const;
+    uint32_t getWeightedNode(double rnd) const;
+    void updateActiveDocs(uint32_t rowId, PossCount newVal, PossCount oldVal);
+    uint32_t numRowStates() const { return _rows.size(); }
+    uint64_t sumActiveDocs() const { return _sumActiveDocs; }
+    PossCount getActiveDocs() const;
+    bool activeDocsValid() const { return _invalidActiveDocsCounter == 0; }
+private:
+    std::vector<RowState> _rows;
+    uint64_t _sumActiveDocs;
+    size_t _invalidActiveDocsCounter;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp
new file mode 100644
index 00000000000..b8e0417678e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".fdispatch.search_path");
+#include "search_path.h"
+
+#include <iostream>
+
+namespace fdispatch {
+
+SearchPath::Element::Element()
+    : _nodes(),
+      _row(std::numeric_limits<size_t>::max())
+{
+}
+
+vespalib::stringref
+SearchPath::parseElement(const vespalib::stringref &spec, size_t numNodes)
+{
+    _elements.push_back(Element());
+    vespalib::string::size_type specSepPos(spec.find('/'));
+    parsePartList(spec.substr(0, specSepPos), numNodes);
+
+    vespalib::stringref remaining = spec.substr(specSepPos + 1);
+    vespalib::string::size_type elementSepPos = remaining.find(';');
+    parseRow(remaining.substr(0, elementSepPos));
+
+    if (elementSepPos != vespalib::string::npos) {
+        return remaining.substr(elementSepPos + 1);
+    }
+    return vespalib::stringref();
+}
+
+void
+SearchPath::parsePartList(const vespalib::stringref &partSpec, size_t numNodes)
+{
+    try {
+        if (!partSpec.empty() && (partSpec[0] != '*')) {
+            vespalib::asciistream is(partSpec);
+            is.eatWhite();
+            parsePartList(is, numNodes);
+        } else {
+            for (size_t i(0); i < numNodes; i++) {
+                _elements.back().addPart(i);
+            }
+        }
+    } catch (const std::exception & e) {
+        LOG(warning, "Failed parsing part of searchpath='%s' with error '%s'. Result might be mumbo jumbo.",
+            partSpec.c_str(), e.what());
+    }
+}
+
+void
+SearchPath::parsePartList(vespalib::asciistream &spec, size_t numNodes)
+{
+    spec.eatWhite();
+    if ( !spec.empty() ) {
+        char c(spec.c_str()[0]);
+        if (c == '[') {
+            parsePartRange(spec, numNodes);
+        } else {
+            size_t num(0);
+            spec >> num;
+            _elements.back().addPart(num);
+        }
+        if ( ! spec.eof() ) {
+            spec >> c;
+            if (c == ',') {
+                parsePartList(spec, numNodes);
+            }
+        }
+    } else {
+        throw std::runtime_error("Expected either '[' or a number, got EOF");
+    }
+}
+
+void
+SearchPath::parsePartRange(vespalib::asciistream &spec, size_t numNodes)
+{
+    size_t from(0);
+    size_t to(numNodes);
+    char s(0), c(0), e(0);
+    spec >> s >> from >> c >> to >> e;
+    if (c != ',') {
+        throw std::runtime_error("Expected ','");
+    }
+    if (e != '>') {
+        throw std::runtime_error("Expected '>'");
+    }
+    to = std::min(numNodes, to);
+    for (size_t i(from); i < to; i++) {
+        _elements.back().addPart(i);
+    }
+}
+
+void
+SearchPath::parseRow(const vespalib::stringref &rowSpec)
+{
+    if (!rowSpec.empty()) {
+        _elements.back().setRow(strtoul(rowSpec.c_str(), NULL, 0));
+    }
+}
+
+SearchPath::SearchPath(const vespalib::string &spec, size_t numNodes)
+    : _elements()
+{
+    vespalib::stringref specBuf = spec;
+    while (!specBuf.empty()) {
+        specBuf = parseElement(specBuf, numNodes);
+    }
+}
+
+} // namespace fdispatch
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.h b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.h
new file mode 100644
index 00000000000..706bb1bc1b9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <set>
+
+namespace fdispatch {
+
+class SearchPath
+{
+public:
+    typedef std::set<size_t> NodeList;
+
+    class Element
+    {
+    private:
+        NodeList _nodes;
+        size_t   _row;
+
+    public:
+        Element();
+        Element &addPart(size_t part) {
+            _nodes.insert(part);
+            return *this;
+        }
+        Element &setRow(size_t row_) {
+            _row = row_;
+            return *this;
+        }
+        bool hasRow() const { return _row != std::numeric_limits<size_t>::max(); }
+        size_t row() const { return _row; }
+        const NodeList &nodes() const { return _nodes; }
+    };
+
+    typedef std::vector<Element> ElementVector;
+
+private:
+    ElementVector _elements;
+
+    vespalib::stringref parseElement(const vespalib::stringref &spec, size_t numNodes);
+    void parsePartList(const vespalib::stringref &partSpec, size_t numNodes);
+    void parsePartList(vespalib::asciistream &spec, size_t numNodes);
+    void parsePartRange(vespalib::asciistream &spec, size_t numNodes);
+    void parseRow(const vespalib::stringref &rowSpec);
+
+public:
+    SearchPath(const vespalib::string &spec, size_t numNodes);
+    const ElementVector &elements() const { return _elements; }
+};
+
+} // namespace fdispatch
+
diff --git a/searchcore/src/vespa/searchcore/grouping/.gitignore b/searchcore/src/vespa/searchcore/grouping/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/grouping/CMakeLists.txt b/searchcore/src/vespa/searchcore/grouping/CMakeLists.txt
new file mode 100644
index 00000000000..9496b5be634
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_grouping STATIC
+    SOURCES
+    groupingcontext.cpp
+    groupingmanager.cpp
+    groupingsession.cpp
+    mergingmanager.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/grouping/OWNERS b/searchcore/src/vespa/searchcore/grouping/OWNERS
new file mode 100644
index 00000000000..9bbc3a73836
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/OWNERS
@@ -0,0 +1,2 @@
+balder
+havardpe
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp
new file mode 100644
index 00000000000..3167491461c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".groupingcontext");
+#include "groupingcontext.h"
+#include <vector>
+#include <vespa/searchlib/aggregation/grouping.h>
+#include <vespa/searchlib/aggregation/predicates.h>
+
+namespace search {
+
+using aggregation::CountFS4Hits;
+using aggregation::FS4HitSetDistributionKey;
+
+namespace grouping {
+
+void
+GroupingContext::deserialize(const char *groupSpec, uint32_t groupSpecLen)
+{
+    if ((groupSpec != NULL) && (groupSpecLen > 4)) {
+        vespalib::nbostream is(groupSpec, groupSpecLen);
+        vespalib::NBOSerializer nis(is);
+        uint32_t numGroupings = 0;
+        nis >> numGroupings;
+        for (size_t i = 0; i < numGroupings; i++) {
+            GroupingPtr grouping(new search::aggregation::Grouping);
+            grouping->deserialize(nis);
+            grouping->setClock(&_clock);
+            grouping->setTimeOfDoom(_timeOfDoom);
+            _groupingList.push_back(grouping);
+        }
+    }
+}
+
+size_t
+GroupingContext::countFS4Hits()
+{
+    size_t numFs4Hits(0);
+    for (GroupingPtr & g : _groupingList) {
+        CountFS4Hits counter;
+        g->select(counter, counter);
+        numFs4Hits += counter.getHitCount();
+    }
+    return numFs4Hits;
+}
+
+void
+GroupingContext::setDistributionKey(uint32_t distributionKey)
+{
+    for (GroupingPtr & g : _groupingList) {
+        FS4HitSetDistributionKey updater(distributionKey);
+        g->select(updater, updater);
+    }
+}
+
+GroupingContext::GroupingContext(const vespalib::Clock & clock, fastos::TimeStamp timeOfDoom, const char *groupSpec, uint32_t groupSpecLen) :
+    _clock(clock),
+    _timeOfDoom(timeOfDoom),
+    _os(),
+    _groupingList()
+{
+    deserialize(groupSpec, groupSpecLen);
+}
+
+GroupingContext::GroupingContext(const vespalib::Clock & clock, fastos::TimeStamp timeOfDoom) :
+    _clock(clock),
+    _timeOfDoom(timeOfDoom),
+    _os(),
+    _groupingList()
+{
+}
+
+GroupingContext::GroupingContext(const GroupingContext & rhs) :
+    _clock(rhs._clock),
+    _timeOfDoom(rhs._timeOfDoom),
+    _os(),
+    _groupingList()
+{
+}
+
+void
+GroupingContext::addGrouping(const GroupingPtr & g)
+{
+    _groupingList.push_back(g);
+}
+
+void
+GroupingContext::serialize()
+{
+    vespalib::NBOSerializer nos(_os);
+
+    nos << (uint32_t)_groupingList.size();
+    for (size_t i = 0; i < _groupingList.size(); i++) {
+        search::aggregation::Grouping & grouping(*_groupingList[i]);
+        grouping.serialize(nos);
+    }
+}
+
+bool
+GroupingContext::needRanking() const
+{
+    if (_groupingList.empty()) {
+        return false;
+    }
+    return true;
+}
+
+
+} // namespace search::grouping
+} // namespace search
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h
new file mode 100644
index 00000000000..cd3b69d1bf5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/aggregation/grouping.h>
+#include <vector>
+#include <memory>
+
+namespace search {
+
+namespace grouping {
+
+/**
+ * A Grouping Context contains all grouping expressions that should be evaluated
+ * for a particular pass, together with the ability to serialize and deserialize
+ * the data from/to a byte buffer.
+ **/
+class GroupingContext
+{
+public:
+    typedef std::unique_ptr<GroupingContext>                     UP;
+    typedef std::shared_ptr<search::aggregation::Grouping> GroupingPtr;
+    typedef std::vector<GroupingPtr>                           GroupingList;
+
+private:
+    GroupingContext &operator=(const GroupingContext &);
+
+    const vespalib::Clock     & _clock;
+    fastos::TimeStamp           _timeOfDoom;
+    vespalib::nbostream         _os;
+    GroupingList                _groupingList;
+public:
+
+    /**
+     * Deserialize a grouping spec into this context.
+     *
+     * @param groupSpec The grouping specification to use for initialization.
+     * @param groupSpecLen The length of the grouping specification, in bytes.
+     **/
+    void deserialize(const char *groupSpec, uint32_t groupSpecLen);
+
+    /**
+     * Create a new grouping context from a byte buffer.
+     * @param groupSpec The grouping specification to use for initialization.
+     * @param groupSpecLen The length of the grouping specification, in bytes.
+     **/
+    GroupingContext(const vespalib::Clock & clock, fastos::TimeStamp timeOfDoom, const char *groupSpec, uint32_t groupSpecLen);
+
+    /**
+     * Create a new grouping context from a byte buffer.
+     * @param groupSpec The grouping specification to use for initialization.
+     * @param groupSpecLen The length of the grouping specification, in bytes.
+     **/
+    GroupingContext(const vespalib::Clock & clock, fastos::TimeStamp timeOfDoom);
+
+    /**
+     * Shallow copy of references
+     **/
+    GroupingContext(const GroupingContext & rhs);
+
+    /**
+     * Add another grouping to this context.
+     * @param g Pointer to the grouping object to become part of this context.
+     **/
+    void addGrouping(const GroupingPtr & g);
+
+    /**
+     * Reset the context to an empty state.
+     **/
+    void reset() { _groupingList.clear(); }
+
+    /**
+     * Return the internal list of grouping expressions in this context.
+     * @return a list of groupings.
+     **/
+    GroupingList &getGroupingList() { return _groupingList; }
+
+    /**
+     * Serialize the grouping expressions in this context.
+     **/
+    void serialize();
+
+    /**
+     * Check whether this context contains any groupings.
+     **/
+    bool empty() const { return _groupingList.empty(); }
+
+    /**
+     * Obtain the grouping result.
+     *
+     * @return grouping result
+     **/
+    vespalib::nbostream & getResult() { return _os; }
+
+    /**
+     * Count number of fs4hits
+     *
+     * @return number of fs4 hits.
+     */
+    size_t countFS4Hits();
+    /**
+     * Will inject the distribution key in the FS4Hits aggregated so far.
+     *
+     * @param the distribution key.
+     */
+    void setDistributionKey(uint32_t distributionKey);
+    /**
+     * Obtain the time of doom.
+     */
+    fastos::TimeStamp getTimeOfDoom() const { return _timeOfDoom; }
+    /**
+     * Figure out if ranking is necessary for any of the grouping requests here.
+     * @return true if ranking is required.
+     */
+    bool needRanking() const;
+};
+
+} // namespace search::grouping
+} // namespace search
+
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
new file mode 100644
index 00000000000..fe8f67384b4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
@@ -0,0 +1,136 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".groupingmanager");
+#include "groupingmanager.h"
+#include <vector>
+#include <vespa/searchlib/aggregation/grouping.h>
+#include <vespa/searchlib/aggregation/groupinglevel.h>
+#include <vespa/searchlib/aggregation/fs4hit.h>
+#include <vespa/vespalib/objects/objectpredicate.h>
+#include <vespa/vespalib/objects/objectoperation.h>
+#include <vespa/searchlib/expression/attributenode.h>
+#include <vespa/searchlib/expression/expressiontree.h>
+#include <vespa/searchlib/expression/expressionnode.h>
+#include <vespa/searchcore/grouping/groupingsession.h>
+
+namespace search {
+namespace grouping {
+
+using search::aggregation::Grouping;
+using search::attribute::IAttributeContext;
+
+//-----------------------------------------------------------------------------
+
+GroupingManager::GroupingManager(GroupingContext & groupingContext)
+    : _groupingContext(groupingContext)
+{
+}
+
+GroupingManager::~GroupingManager()
+{
+}
+
+using search::expression::ExpressionNode;
+using search::expression::AttributeNode;
+using search::expression::ConfigureStaticParams;
+using search::aggregation::Grouping;
+using search::aggregation::GroupingLevel;
+
+void
+GroupingManager::init(const IAttributeContext &attrCtx)
+{
+    GroupingContext::GroupingList list;
+    GroupingContext::GroupingList &groupingList(_groupingContext.getGroupingList());
+    for (size_t i = 0; i < groupingList.size(); ++i) {
+        Grouping &grouping = *groupingList[i];
+        try {
+            Grouping::GroupingLevelList &levels = grouping.levels();
+            for (size_t k = grouping.getFirstLevel(); k <= grouping.getLastLevel() &&
+                            k < levels.size(); k++) {
+                GroupingLevel & level(levels[k]);
+                ExpressionNode::LP en = level.getExpression().getRoot();
+
+                if (en->inherits(AttributeNode::classId)) {
+                    AttributeNode & an = static_cast<AttributeNode &>(*en);
+                    an.useEnumOptimization();
+                }
+            }
+            ConfigureStaticParams stuff(&attrCtx, NULL);
+            grouping.configureStaticStuff(stuff);
+            list.push_back(groupingList[i]);
+        } catch (const std::exception & e) {
+            LOG(error, "Could not locate attribute for grouping number %ld : %s. Ignoring grouping '%s'", i, e.what(), grouping.asString().c_str());
+        }
+    }
+    std::swap(list, groupingList);
+}
+
+void
+GroupingManager::groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize)
+{
+    GroupingContext::GroupingList &groupingList(_groupingContext.getGroupingList());
+    for (size_t i = 0; i < groupingList.size(); ++i) {
+        Grouping & g = *groupingList[i];
+        if ( ! g.needResort() ) {
+            g.aggregate(searchResults, binSize);
+            LOG(debug, "groupInRelevanceOrder: %s", g.asString().c_str());
+            g.cleanTemporary();
+            g.cleanupAttributeReferences();
+        }
+    }
+}
+
+void
+GroupingManager::groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow)
+{
+    GroupingContext::GroupingList &groupingList(_groupingContext.getGroupingList());
+    for (size_t i = 0; i < groupingList.size(); ++i) {
+        Grouping & g = *groupingList[i];
+        if ( g.needResort() ) {
+            g.aggregate(searchResults, binSize, overflow);
+            LOG(debug, "groupUnordered: %s", g.asString().c_str());
+            g.cleanTemporary();
+            g.cleanupAttributeReferences();
+        }
+    }
+}
+
+void
+GroupingManager::merge(GroupingContext &ctx)
+{
+    GroupingContext::GroupingList &list_a(_groupingContext.getGroupingList());
+    GroupingContext::GroupingList &list_b(ctx.getGroupingList());
+    LOG_ASSERT(list_a.size() == list_b.size());
+    for (size_t i = 0; i < list_a.size(); ++i) {
+        Grouping &a = *list_a[i];
+        Grouping &b = *list_b[i];
+        LOG_ASSERT(a.getId() == b.getId());
+        a.merge(b);
+    }
+}
+
+void
+GroupingManager::prune()
+{
+    GroupingContext::GroupingList &groupingList(_groupingContext.getGroupingList());
+    for (size_t i = 0; i < groupingList.size(); ++i) {
+        Grouping &g = *groupingList[i];
+        g.postMerge();
+        g.sortById();
+    }
+}
+
+void
+GroupingManager::convertToGlobalId(const search::IDocumentMetaStore &metaStore)
+{
+    GroupingContext::GroupingList & groupingList = _groupingContext.getGroupingList();
+    for (size_t i = 0; i < groupingList.size(); ++i) {
+        Grouping & g = *groupingList[i];
+        g.convertToGlobalId(metaStore);
+        LOG(debug, "convertToGlobalId: %s", g.asString().c_str());
+    }
+}
+
+} // namespace search::grouping
+} // namespace search
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h
new file mode 100644
index 00000000000..4c3ae35a61d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h
@@ -0,0 +1,97 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/common/idocumentmetastore.h>
+#include <vespa/searchlib/aggregation/grouping.h>
+#include <vespa/searchcore/grouping/groupingcontext.h>
+
+namespace search {
+
+namespace grouping {
+
+/**
+ * Wrapper class used to handle actual grouping. All input data is
+ * assumed to be kept alive by the user.
+ **/
+class GroupingManager
+{
+private:
+    GroupingManager(const GroupingManager &);
+    GroupingManager &operator=(const GroupingManager &);
+
+    GroupingContext         &_groupingContext;
+
+public:
+    /**
+     * Create a new grouping manager.
+     *
+     * @param groupingContext Context to use for grouping
+     **/
+    GroupingManager(GroupingContext & groupingContext);
+
+    /**
+     * Release resources
+     **/
+    ~GroupingManager();
+
+    /**
+     * @return true if this manager is holding an empty grouping request.
+     **/
+    bool empty() const { return _groupingContext.getGroupingList().empty(); }
+
+    /**
+     * Initialize underlying context with attribute bindings.
+     *
+     * @param attrCtx attribute context
+     **/
+    void init(const search::attribute::IAttributeContext &attrCtx);
+
+    /**
+     * Perform actual grouping on the given results.
+     * The results must be in relevance sort order.
+     * Will only perform grouping that will not resort.
+     *
+     * @param searchResults the result set in array form
+     * @param binSize size of search result array
+     **/
+    void groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize);
+
+    /**
+     * Perform actual grouping on the given the results.
+     * The results should be in fastest access order which is normally unsorted.
+     * Will only perform grouping that actually will resort.
+     *
+     * @param searchResults the result set in array form
+     * @param binSize size of search result array
+     * @param overflow The unranked hits.
+     **/
+    void groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow);
+
+    /**
+     * Merge another grouping context into the underlying context of
+     * this manager. Both contexts must have the same groupings in the
+     * same order.
+     *
+     * @param ctx context to merge into the underlying context of this manager
+     **/
+    void merge(GroupingContext &ctx);
+
+    /**
+     * Called after merge has been called (possibly multiple times) to
+     * prune unwanted information from the underlying grouping
+     * context.
+     **/
+    void prune();
+
+    /**
+     * Perform converting from local to global document id on all hits
+     * in the underlying grouping trees.
+     *
+     * @param metaStore the attribute used to map from lid to gid.
+     **/
+    void convertToGlobalId(const search::IDocumentMetaStore &metaStore);
+};
+
+} // namespace search::grouping
+} // namespace search
+
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp
new file mode 100644
index 00000000000..d93bfcd748a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".groupingsession");
+#include "groupingsession.h"
+#include <vector>
+#include <vespa/searchlib/aggregation/grouping.h>
+
+namespace search {
+namespace grouping {
+
+using search::aggregation::Group;
+using search::aggregation::Grouping;
+using search::aggregation::GroupingLevel;
+using search::attribute::IAttributeContext;
+
+GroupingSession::GroupingSession(const SessionId &sessionId,
+                                 GroupingContext & groupingContext,
+                                 const IAttributeContext &attrCtx)
+    : _sessionId(sessionId),
+      _mgrContext(groupingContext),
+      _groupingManager(_mgrContext),
+      _timeOfDoom(groupingContext.getTimeOfDoom())
+{
+    init(groupingContext, attrCtx);
+}
+
+GroupingSession::~GroupingSession()
+{
+}
+
+using search::expression::ExpressionNode;
+using search::expression::AttributeNode;
+using search::expression::ConfigureStaticParams;
+using search::aggregation::Grouping;
+using search::aggregation::GroupingLevel;
+
+void
+GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext &attrCtx)
+{
+    GroupingList & sessionList(groupingContext.getGroupingList());
+    for (size_t i = 0; i < sessionList.size(); ++i) {
+        GroupingPtr g(sessionList[i]);
+        // Make internal copy of those we want to keep for another pass
+        if (!_sessionId.empty() && g->getLastLevel() < g->levels().size()) {
+            GroupingPtr gp(new Grouping(*g));
+            gp->setLastLevel(gp->levels().size());
+            _groupingMap[gp->getId()] = gp;
+            g = gp;
+        }
+        _mgrContext.addGrouping(g);
+    }
+    _groupingManager.init(attrCtx);
+}
+
+void
+GroupingSession::prepareThreadContextCreation(size_t num_threads)
+{
+    if (num_threads > 1) {
+        _mgrContext.serialize(); // need copy of internal modified request
+    }
+}
+
+GroupingContext::UP
+GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext &attrCtx)
+{
+    GroupingContext::UP ctx(new GroupingContext(_mgrContext));
+    if (thread_id == 0) {
+        GroupingContext::GroupingList &groupingList = _mgrContext.getGroupingList();
+        for (size_t i = 0; i < groupingList.size(); ++i) {
+            ctx->addGrouping(groupingList[i]);
+        }
+    } else {
+        ctx->deserialize(_mgrContext.getResult().peek(),
+                         _mgrContext.getResult().size());
+        GroupingManager man(*ctx);
+        man.init(attrCtx);
+    }
+    return ctx;
+}
+
+void
+GroupingSession::continueExecution(GroupingContext & groupingContext)
+{
+    GroupingList &orig(groupingContext.getGroupingList());
+    for (GroupingList::iterator it(orig.begin()), mt(orig.end()); it != mt; it++) {
+        Grouping &origGrouping(**it);
+        if (_groupingMap.find((*it)->getId()) != _groupingMap.end()) {
+            Grouping &cachedGrouping(*_groupingMap[(*it)->getId()]);
+            cachedGrouping.prune(origGrouping);
+            origGrouping.mergePartial(cachedGrouping);
+            // No use in keeping it for the next round
+            if (origGrouping.getLastLevel() == cachedGrouping.getLastLevel()) {
+                _groupingMap.erase(origGrouping.getId());
+            }
+        }
+        LOG(debug, "Continue execution result: %s", origGrouping.asString().c_str());
+    }
+    groupingContext.serialize();
+}
+
+} // namespace search::grouping
+} // namespace search
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.h b/searchcore/src/vespa/searchcore/grouping/groupingsession.h
new file mode 100644
index 00000000000..f355f5de418
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.h
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/aggregation/grouping.h>
+#include <map>
+#include <vespa/searchcore/grouping/groupingmanager.h>
+#include <vespa/searchcore/grouping/groupingcontext.h>
+#include <vespa/searchcore/grouping/sessionid.h>
+
+namespace search {
+
+namespace grouping {
+
+/**
+ * A grouping session represents the execution of a grouping expression with one
+ * or more passes. Multiple passes are supported by keeping internal state, and
+ * providing a way to copy parts of this state into a context object for each
+ * pass.
+ **/
+class GroupingSession
+{
+private:
+    GroupingSession(const GroupingSession &);
+    GroupingSession &operator=(const GroupingSession &);
+
+    typedef std::shared_ptr<search::aggregation::Grouping> GroupingPtr;
+    typedef std::map<uint32_t, GroupingPtr>                    GroupingMap;
+    typedef std::vector<GroupingPtr>                           GroupingList;
+
+    SessionId       _sessionId;
+    GroupingContext _mgrContext;
+    GroupingManager _groupingManager;
+    GroupingMap     _groupingMap;
+    fastos::TimeStamp _timeOfDoom;
+
+public:
+    typedef vespalib::LinkedPtr<GroupingSession> LP;
+    typedef std::unique_ptr<GroupingSession> UP;
+
+    /**
+     * Create a new grouping session
+     *
+     * @param sessionId The session id of this session.
+     * @param groupingContext grouping context.
+     * @param attrCtx attribute context.
+     **/
+    GroupingSession(const SessionId & sessionId,
+                    GroupingContext & groupingContext,
+                    const search::attribute::IAttributeContext &attrCtx);
+
+    /**
+     * Release resources
+     **/
+    ~GroupingSession();
+
+    /**
+     * Return our session identifier
+     **/
+    const SessionId & getSessionId() const { return _sessionId; }
+
+    /**
+     * Initialize the session with data from the current context.
+     * @param groupingContext The current grouping context.
+     * @param attrCtx attribute context.
+     **/
+    void init(GroupingContext & groupingContext,
+              const search::attribute::IAttributeContext &attrCtx);
+
+    /**
+     * This function is called to prepare for creation of individual
+     * contexts for separate threads.
+     *
+     * @param num_threads number of threads that will request contexts
+     **/
+    void prepareThreadContextCreation(size_t num_threads);
+
+    /**
+     * Create a grouping context to be used by a single thread when
+     * performing multi-threaded grouping. Thread 0 will get a
+     * grouping context that shares groups with this session while
+     * other threads will get equivalent copies that can later be
+     * merged into the master context after partial grouping is
+     * performed in parallel. Note that this thread may be called by
+     * multiple threads at the same time.
+     *
+     * @param thread_id thread id
+     * @param attrCtx attribute context.
+     **/
+    GroupingContext::UP createThreadContext(size_t thread_id,
+                                            const search::attribute::IAttributeContext &attrCtx);
+
+    /**
+     * Return the GroupingManager to use when performing grouping.
+     **/
+    GroupingManager & getGroupingManager() { return _groupingManager; }
+
+    /**
+     * Continue excuting a query given a context.
+     *
+     * @param context The grouping context which contains information about the
+     *                current pass.
+     **/
+    void continueExecution(GroupingContext & context);
+
+    /**
+     * Checks whether or not the session is finished.
+     **/
+    bool finished() const { return _groupingMap.empty(); }
+
+    /**
+     * Get this sessions timeout.
+     */
+    fastos::TimeStamp getTimeOfDoom() const { return _timeOfDoom; }
+};
+
+} // namespace search::grouping
+} // namespace search
+
diff --git a/searchcore/src/vespa/searchcore/grouping/mergingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/mergingmanager.cpp
new file mode 100644
index 00000000000..32ceb8e7925
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/mergingmanager.cpp
@@ -0,0 +1,169 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".mergingmanager");
+#include "mergingmanager.h"
+#include <map>
+#include <vespa/searchlib/aggregation/grouping.h>
+#include <vespa/searchlib/aggregation/fs4hit.h>
+#include <vespa/vespalib/objects/objectpredicate.h>
+#include <vespa/vespalib/objects/objectoperation.h>
+
+namespace search {
+namespace grouping {
+
+namespace {
+
+class PathMangler : public vespalib::ObjectPredicate,
+                    public vespalib::ObjectOperation
+{
+private:
+    uint32_t _partBits;
+    uint32_t _rowBits;
+    uint32_t _partId;
+    uint32_t _rowId;
+    bool     _mld;
+
+public:
+    typedef search::aggregation::FS4Hit FS4Hit;
+    PathMangler(uint32_t partBits, uint32_t rowBits, uint32_t partId, uint32_t rowId, bool mld)
+        : _partBits(partBits), _rowBits(rowBits), _partId(partId), _rowId(rowId), _mld(mld) {}
+    virtual bool check(const vespalib::Identifiable &obj) const {
+        return (obj.getClass().id() == FS4Hit::classId);
+    }
+    virtual void execute(vespalib::Identifiable &obj) {
+        FS4Hit &hit = static_cast<search::aggregation::FS4Hit&>(obj);
+        hit.setPath(computeNewPath(hit.getPath()));
+    }
+    uint32_t computeNewPath(uint32_t path) const {
+        if (_mld) {
+            path = (path + 1) << _partBits;
+        }
+        path += _partId;
+        if (_rowBits > 0) {
+            path = (path << _rowBits) + _rowId;
+        }
+        return path;
+    }
+};
+
+} // namespace search::grouping::<unnamed>
+
+using search::aggregation::Grouping;
+
+//-----------------------------------------------------------------------------
+
+MergingManager::MergingManager(uint32_t partBits, uint32_t rowBits)
+    : _partBits(partBits),
+      _rowBits(rowBits),
+      _input(),
+      _result(0),
+      _resultLen(0)
+{
+}
+
+MergingManager::~MergingManager()
+{
+    free(_result);
+}
+
+void
+MergingManager::addResult(uint32_t partId, uint32_t rowId, bool mld,
+                          const char *groupResult, size_t groupResultLen)
+{
+    _input.push_back(Entry(partId, rowId, mld, groupResult, groupResultLen));
+}
+
+bool MergingManager::needMerge() const
+{
+    if (_input.size() == 1) {
+        PathMangler pathMangler(_partBits, _rowBits,
+                                _input[0].partId, _input[0].rowId,
+                                _input[0].mld);
+        if (pathMangler.computeNewPath(0) == 0) {
+            return false;
+        }
+    }
+    return true;
+}
+
+void
+MergingManager::merge()
+{
+    if (needMerge()) {
+        fullMerge();
+    } else {
+        _resultLen = _input[0].length;
+        _result = (char *) malloc(_resultLen);
+        memcpy(_result, _input[0].data, _resultLen);
+    }
+}
+
+void
+MergingManager::fullMerge()
+{
+    typedef std::unique_ptr<Grouping>    UP;
+    typedef std::map<uint32_t, UP> MAP;
+    typedef MAP::iterator          ITR;
+
+    MAP map;
+    for (size_t i = 0; i < _input.size(); ++i) {
+        PathMangler pathMangler(_partBits, _rowBits,
+                                _input[i].partId, _input[i].rowId,
+                                _input[i].mld);
+        if ((_input[i].data != NULL) && (_input[i].length > 0)) {
+            vespalib::nbostream is(_input[i].data, _input[i].length);
+            vespalib::NBOSerializer nis(is);
+            uint32_t cnt = 0;
+            nis >> cnt;
+            for (uint32_t j = 0; j < cnt; ++j) {
+                UP g(new Grouping());
+                g->deserialize(nis);
+                g->select(pathMangler, pathMangler);
+                ITR pos = map.find(g->getId());
+                if (pos == map.end()) {
+                    map[g->getId()] = std::move(g);
+                } else {
+                    pos->second->merge(*g);
+                }
+            }
+        }
+    }
+    vespalib::nbostream os;
+    vespalib::NBOSerializer nos(os);
+    nos << (uint32_t)map.size();
+    ITR end = map.end();
+    for (ITR itr = map.begin(); itr != end; ++itr) {
+        itr->second->postMerge();
+        itr->second->sortById();
+        itr->second->serialize(nos);
+    }
+    _resultLen = os.size();
+    _result = (char *) malloc(os.size());
+    memcpy(_result, os.c_str(), os.size());
+}
+
+size_t
+MergingManager::getGroupResultLen() const
+{
+    return _resultLen;
+}
+
+const char *
+MergingManager::getGroupResult() const
+{
+    return _result;
+}
+
+char *
+MergingManager::stealGroupResult()
+{
+    char *tmp = _result;
+    _result = 0;
+    return tmp;
+}
+
+//-----------------------------------------------------------------------------
+
+} // namespace search::grouping
+} // namespace search
diff --git a/searchcore/src/vespa/searchcore/grouping/mergingmanager.h b/searchcore/src/vespa/searchcore/grouping/mergingmanager.h
new file mode 100644
index 00000000000..ba8e21c6954
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/mergingmanager.h
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <memory>
+#include <vector>
+
+namespace search {
+namespace grouping {
+
+/**
+ * Wrapper class used to handle merging of grouping results. All input
+ * data is assumed to be kept alive by the user.
+ **/
+class MergingManager
+{
+private:
+    MergingManager(const MergingManager &);
+    MergingManager &operator=(const MergingManager &);
+    void fullMerge();
+    bool needMerge() const;
+
+    /**
+     * Simple wrapper for all the grouping results from a single
+     * search/fdispatch node.
+     **/
+    struct Entry {
+        uint32_t    partId;
+        uint32_t    rowId;
+        bool        mld;
+        const char *data;
+        size_t      length;
+
+        Entry(uint32_t part, uint32_t row, bool m, const char *pt, size_t len)
+            : partId(part), rowId(row), mld(m), data(pt), length(len) {}
+    };
+
+    uint32_t           _partBits;
+    uint32_t           _rowBits;
+    std::vector<Entry> _input;
+    char              *_result;
+    size_t             _resultLen;
+
+public:
+    /**
+     * Create a new merging manager.
+     *
+     * @param partBits how many bits to be used to encode partId into path
+     * @param rowBits how many bits to be used to encode rowId into path
+     **/
+    MergingManager(uint32_t partBits, uint32_t rowBits);
+
+    /**
+     * Release resources
+     **/
+    ~MergingManager();
+
+    /**
+     * Register an additional grouping result that should be part of
+     * the upcoming merge operation.
+     *
+     * @param partId which partition these results came from
+     * @param rowId which row these results came from
+     * @param mld true if the node below is a dispatch node
+     * @param groupSpec group spec
+     * @param groupSpecLen length of the group spec
+     **/
+    void addResult(uint32_t partId, uint32_t rowId, bool mld,
+                   const char *groupResult, size_t groupResultLen);
+
+    /**
+     * Perform actual merging of all the registered grouping results.
+     **/
+    void merge();
+
+    /**
+     * Obtain the size of the grouping result
+     *
+     * @return grouping result size
+     **/
+    size_t getGroupResultLen() const;
+
+    /**
+     * Obtain the grouping result.
+     *
+     * @return grouping result
+     **/
+    const char *getGroupResult() const;
+
+    /**
+     * Steal the grouping result. Invoking this method will take
+     * overship of the grouping result blob returned by this
+     * method. Use 'free' to release the memory when you are done with
+     * it.
+     *
+     * @return grouping result that have just been stolen
+     **/
+    char *stealGroupResult();
+};
+
+} // namespace search::grouping
+} // namespace search
+
diff --git a/searchcore/src/vespa/searchcore/grouping/sessionid.h b/searchcore/src/vespa/searchcore/grouping/sessionid.h
new file mode 100644
index 00000000000..b0740ee0c7c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/grouping/sessionid.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace search {
+namespace grouping {
+
+/**
+ * Representing the session identifier of a grouping to be used in the
+ * session manager.
+ **/
+typedef vespalib::string SessionId;
+
+} // namespace grouping
+} // namespace search
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/.gitignore b/searchcore/src/vespa/searchcore/proton/attribute/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
new file mode 100644
index 00000000000..57a76366f78
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
@@ -0,0 +1,29 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_attribute STATIC
+    SOURCES
+    address_space_usage_stats.cpp
+    attribute_collection_spec_factory.cpp
+    attribute_factory.cpp
+    attribute_initializer.cpp
+    attribute_manager_explorer.cpp
+    attribute_manager_initializer.cpp
+    attribute_populator.cpp
+    attribute_usage_filter.cpp
+    attribute_usage_sampler_context.cpp
+    attribute_usage_sampler_functor.cpp
+    attribute_usage_stats.cpp
+    attribute_vector_explorer.cpp
+    attribute_writer.cpp
+    attributes_initializer_base.cpp
+    attributedisklayout.cpp
+    attributemanager.cpp
+    attributesconfigscout.cpp
+    document_field_populator.cpp
+    document_field_retriever.cpp
+    exclusive_attribute_read_accessor.cpp
+    filter_attribute_manager.cpp
+    flushableattribute.cpp
+    initialized_attributes_result.cpp
+    sequential_attributes_initializer.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/OWNERS b/searchcore/src/vespa/searchcore/proton/attribute/OWNERS
new file mode 100644
index 00000000000..7ae1acb1be9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/OWNERS
@@ -0,0 +1 @@
+geirst
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp
new file mode 100644
index 00000000000..2dcdc0d3e9c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "address_space_usage_stats.h"
+
+namespace proton {
+
+AddressSpaceUsageStats::AddressSpaceUsageStats(const search::AddressSpace &
+                                               usage)
+    : _usage(usage),
+      _attributeName(),
+      _subDbName()
+{
+}
+
+void
+AddressSpaceUsageStats::merge(const search::AddressSpace &usage,
+                              const vespalib::string &attributeName,
+                              const vespalib::string &subDbName)
+{
+    if (attributeName.empty() || usage.usage() > _usage.usage()) {
+        _usage = usage;
+        _attributeName = attributeName;
+        _subDbName = subDbName;
+    }
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h
new file mode 100644
index 00000000000..96f66793382
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/attribute/address_space.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+/*
+ * class representing usage of a single address space (enum store or
+ * multi value) and the most largest attribute in that respect, relative
+ * to the limit.
+ */
+class AddressSpaceUsageStats
+{
+    search::AddressSpace _usage;
+    vespalib::string _attributeName;
+    vespalib::string _subDbName;
+
+public:
+    AddressSpaceUsageStats(const search::AddressSpace &usage);
+    void merge(const search::AddressSpace &usage,
+               const vespalib::string &attributeName,
+               const vespalib::string &subDbName);
+
+    const search::AddressSpace &getUsage() const { return _usage; }
+    const vespalib::string &getAttributeName() const { return _attributeName; }
+    const vespalib::string &getSubDbName() const { return _subDbName; }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.h
new file mode 100644
index 00000000000..7e1b65da197
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.h
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <memory>
+#include <vector>
+
+namespace proton {
+
+/**
+ * A specification of which attribute vectors an attribute manager should instantiate and manage.
+ */
+class AttributeCollectionSpec
+{
+public:
+    typedef std::unique_ptr<AttributeCollectionSpec> UP;
+
+    class Attribute
+    {
+    private:
+        vespalib::string _name;
+        search::attribute::Config _cfg;
+    public:
+        Attribute(const vespalib::string &name,
+                      const search::attribute::Config &cfg)
+            : _name(name),
+              _cfg(cfg)
+        {
+        }
+        const vespalib::string &getName() const { return _name; }
+        const search::attribute::Config &getConfig() const { return _cfg; }
+    };
+
+    typedef std::vector<Attribute> AttributeList;
+
+private:
+    typedef search::SerialNum SerialNum;
+
+    AttributeList _attributes;
+    uint32_t      _docIdLimit;
+    SerialNum     _currentSerialNum;
+
+public:
+    AttributeCollectionSpec(const AttributeList &attributes,
+                            uint32_t docIdLimit,
+                            SerialNum currentSerialNum)
+        : _attributes(attributes),
+          _docIdLimit(docIdLimit),
+          _currentSerialNum(currentSerialNum)
+    {
+    }
+    const AttributeList &getAttributes() const {
+        return _attributes;
+    }
+    uint32_t getDocIdLimit() const {
+        return _docIdLimit;
+    }
+    SerialNum getCurrentSerialNum() const {
+        return _currentSerialNum;
+    }
+    bool hasAttribute(const vespalib::string &name) const {
+        for (const auto &attr : _attributes) {
+            if (attr.getName() == name) {
+                return true;
+            }
+        }
+        return false;
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp
new file mode 100644
index 00000000000..8fc0b7128fd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.attribute_collection_spec_factory");
+#include "attribute_collection_spec_factory.h"
+#include <vespa/searchlib/attribute/configconverter.h>
+
+using search::attribute::ConfigConverter;
+using search::GrowStrategy;
+
+namespace proton {
+
+AttributeCollectionSpecFactory::AttributeCollectionSpecFactory(
+        const search::GrowStrategy &growStrategy,
+        size_t growNumDocs,
+        bool fastAccessOnly)
+    : _growStrategy(growStrategy),
+      _growNumDocs(growNumDocs),
+      _fastAccessOnly(fastAccessOnly)
+{
+}
+
+AttributeCollectionSpec::UP
+AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg,
+                                       uint32_t docIdLimit,
+                                       search::SerialNum serialNum) const
+{
+    AttributeCollectionSpec::AttributeList attrs;
+    // Amortize memory spike cost over N docs
+    const size_t skew = _growNumDocs/(attrCfg.attribute.size()+1);
+    GrowStrategy grow = _growStrategy;
+    grow.setDocsInitialCapacity(std::max(grow.getDocsInitialCapacity(),
+                                         docIdLimit));
+    for (const auto &attr : attrCfg.attribute) {
+        search::attribute::Config cfg = ConfigConverter::convert(attr);
+        if (_fastAccessOnly && !cfg.fastAccess()) {
+            continue;
+        }
+        grow.setDocsGrowDelta(grow.getDocsGrowDelta() + skew);
+        cfg.setGrowStrategy(grow);
+        attrs.push_back(AttributeCollectionSpec::Attribute(attr.name, cfg));
+    }
+    return AttributeCollectionSpec::UP(new AttributeCollectionSpec(attrs,
+                                                                   docIdLimit,
+                                                                   serialNum));
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h
new file mode 100644
index 00000000000..dc3dfcbd8d4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attribute_collection_spec.h"
+#include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchcommon/common/growstrategy.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/config-attributes.h>
+
+namespace proton {
+
+/**
+ * A factory for generating an AttributeCollectionSpec based on AttributesConfig
+ * from the config server.
+ */
+class AttributeCollectionSpecFactory
+{
+private:
+    typedef vespa::config::search::AttributesConfig AttributesConfig;
+
+    const search::GrowStrategy _growStrategy;
+    const size_t               _growNumDocs;
+    const bool                 _fastAccessOnly;
+
+public:
+    AttributeCollectionSpecFactory(const search::GrowStrategy &growStrategy,
+                                   size_t growNumDocs,
+                                   bool fastAccessOnly);
+
+    AttributeCollectionSpec::UP create(const AttributesConfig &attrCfg,
+                                       uint32_t docIdLimit,
+                                       search::SerialNum serialNum) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp
new file mode 100644
index 00000000000..d0ae670b4ba
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.attribute_factory");
+
+#include "attribute_factory.h"
+#include <vespa/searchlib/attribute/attributefactory.h>
+
+namespace proton {
+
+using search::AttributeVector;
+
+AttributeFactory::AttributeFactory()
+{
+}
+
+AttributeVector::SP
+AttributeFactory::create(const vespalib::string &name,
+                         const search::attribute::Config &cfg) const
+{
+    AttributeVector::SP v(search::AttributeFactory::createAttribute(name, cfg));
+    v->enableEnumeratedSave(true);
+    return v;
+}
+
+void
+AttributeFactory::setupEmpty(const AttributeVector::SP &vec,
+                             search::SerialNum serialNum) const
+{
+    vec->setCreateSerialNum(serialNum);
+    vec->addReservedDoc();
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.h
new file mode 100644
index 00000000000..62eeef3bfeb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_attribute_factory.h"
+#include <set>
+
+namespace proton {
+
+/**
+ * Concrete factory for creating attribute vectors by using the search::AttributeFactory.
+ */
+class AttributeFactory : public IAttributeFactory
+{
+public:
+    typedef std::shared_ptr<AttributeFactory> SP;
+    AttributeFactory();
+
+    // Implements IAttributeFactory
+    virtual search::AttributeVector::SP create(const vespalib::string &name,
+                                               const search::attribute::Config &cfg) const;
+
+    virtual void setupEmpty(const search::AttributeVector::SP &vec,
+                            search::SerialNum serialNum) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp
new file mode 100644
index 00000000000..8c2033882cd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp
@@ -0,0 +1,200 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.attribute_initializer");
+#include "attribute_initializer.h"
+#include "attributedisklayout.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+using search::attribute::Config;
+using search::AttributeVector;
+using search::IndexMetaInfo;
+
+namespace proton {
+
+typedef AttributeInitializer::AttributeHeader AttributeHeader;
+
+namespace {
+
+const vespalib::string dataTypeTag = "datatype";
+const vespalib::string collectionTypeTag = "collectiontype";
+const vespalib::string createSerialNumTag = "createSerialNum";
+
+uint64_t
+extractCreateSerialNum(const vespalib::GenericHeader &header)
+{
+    if (header.hasTag(createSerialNumTag)) {
+        return header.getTag(createSerialNumTag).asInteger();
+    } else {
+        return 0u;
+    }
+}
+
+bool
+extractHeaderTypeOK(const vespalib::GenericHeader &header,
+                    const Config &cfg)
+{
+    return header.hasTag(dataTypeTag) &&
+        header.hasTag(collectionTypeTag) &&
+        header.getTag(dataTypeTag).asString() ==
+        cfg.basicType().asString() &&
+        header.getTag(collectionTypeTag).asString() ==
+        cfg.collectionType().asString();
+}
+
+AttributeHeader
+extractHeader(const AttributeVector::SP &attr,
+              const vespalib::string &attrFileName)
+{
+    std::unique_ptr<Fast_BufferedFile> df;
+    df = search::FileUtil::openFile(attrFileName + ".dat");
+    vespalib::FileHeader datHeader;
+    datHeader.readFile(*df);
+    AttributeHeader retval;
+    retval._createSerialNum = extractCreateSerialNum(datHeader);
+    retval._headerTypeOK = extractHeaderTypeOK(datHeader, attr->getConfig());
+    if (datHeader.hasTag(dataTypeTag)) {
+        retval._btString = datHeader.getTag(dataTypeTag).asString();
+    }
+    if (datHeader.hasTag(collectionTypeTag)) {
+        retval._ctString = datHeader.getTag(collectionTypeTag).asString();
+    }
+    return retval;
+}
+
+void
+logAttributeTooNew(const AttributeVector::SP &attr,
+                   const AttributeHeader &header,
+                   uint64_t serialNum)
+{
+    LOG(info, "Attribute vector '%s' is too new (%" PRIu64 " > %" PRIu64 ")",
+            attr->getBaseFileName().c_str(),
+            header._createSerialNum,
+            serialNum);
+}
+
+void
+logAttributeWrongType(const AttributeVector::SP &attr,
+                      const AttributeHeader &header)
+{
+    const Config &cfg(attr->getConfig());
+    LOG(info, "Attribute vector '%s' is of wrong type (expected %s/%s, got %s/%s)",
+            attr->getBaseFileName().c_str(),
+            cfg.basicType().asString(),
+            cfg.collectionType().asString(),
+            header._btString.c_str(),
+            header._ctString.c_str());
+}
+
+}
+
+AttributeInitializer::AttributeHeader::AttributeHeader()
+    : _createSerialNum(0),
+      _headerTypeOK(false),
+      _btString("unknown"),
+      _ctString("unknown")
+{
+}
+
+AttributeVector::SP
+AttributeInitializer::tryLoadAttribute(const IndexMetaInfo &info) const
+{
+    IndexMetaInfo::Snapshot snap = info.getBestSnapshot();
+    vespalib::string attrFileName =
+            AttributeDiskLayout::getAttributeFileName(_baseDir, _attrName, snap.syncToken);
+    AttributeVector::SP attr = _factory.create(attrFileName, _cfg);
+    if (snap.valid) {
+        AttributeHeader header = extractHeader(attr, attrFileName);
+        if (header._createSerialNum > _currentSerialNum || !header._headerTypeOK) {
+            setupEmptyAttribute(attr, snap, header);
+            return attr;
+        }
+        if (!loadAttribute(attr, snap)) {
+            return AttributeVector::SP();
+        }
+    } else {
+        _factory.setupEmpty(attr, _currentSerialNum);
+    }
+    return attr;
+}
+
+bool
+AttributeInitializer::loadAttribute(const AttributeVector::SP &attr,
+                                    const IndexMetaInfo::Snapshot &snap) const
+{
+    assert(attr->hasLoadData());
+    fastos::TimeStamp startTime = fastos::ClockSystem::now();
+    EventLogger::loadAttributeStart(_documentSubDbName, attr->getName());
+    if (!attr->load()) {
+        LOG(warning, "Could not load attribute vector '%s' from disk. "
+                "Returning empty attribute vector",
+                attr->getBaseFileName().c_str());
+        return false;
+    } else {
+        attr->commit(snap.syncToken, snap.syncToken);
+        fastos::TimeStamp endTime = fastos::ClockSystem::now();
+        int64_t elapsedTimeMs = (endTime - startTime).ms();
+        EventLogger::loadAttributeComplete(_documentSubDbName, attr->getName(), elapsedTimeMs);
+    }
+    return true;
+}
+
+void
+AttributeInitializer::setupEmptyAttribute(AttributeVector::SP &attr,
+                                          const IndexMetaInfo::Snapshot &snap,
+                                          const AttributeHeader &header) const
+{
+    if (header._createSerialNum > _currentSerialNum) {
+        logAttributeTooNew(attr, header, _currentSerialNum);
+    }
+    if (!header._headerTypeOK) {
+        logAttributeWrongType(attr, header);
+    }
+    LOG(info, "Returning empty attribute vector for '%s'",
+            attr->getBaseFileName().c_str());
+    _factory.setupEmpty(attr, _currentSerialNum);
+    attr->commit(snap.syncToken, snap.syncToken);
+}
+
+AttributeVector::SP
+AttributeInitializer::createAndSetupEmptyAttribute(IndexMetaInfo &info) const
+{
+    vespalib::mkdir(info.getPath(), false);
+    vespalib::string attrFileName =
+            AttributeDiskLayout::getAttributeFileName(_baseDir, _attrName, 0);
+    AttributeVector::SP attr = _factory.create(attrFileName, _cfg);
+    _factory.setupEmpty(attr, _currentSerialNum);
+    info.save();
+    return attr;
+}
+
+AttributeInitializer::AttributeInitializer(const vespalib::string &baseDir,
+                                           const vespalib::string &documentSubDbName,
+                                           const vespalib::string &attrName,
+                                           const search::attribute::Config &cfg,
+                                           uint64_t currentSerialNum,
+                                           const IAttributeFactory &factory)
+    : _baseDir(baseDir),
+      _documentSubDbName(documentSubDbName),
+      _attrName(attrName),
+      _cfg(cfg),
+      _currentSerialNum(currentSerialNum),
+      _factory(factory)
+{
+}
+
+search::AttributeVector::SP
+AttributeInitializer::init() const
+{
+    IndexMetaInfo info(AttributeDiskLayout::getAttributeBaseDir(_baseDir, _attrName));
+    if (info.load()) {
+        return tryLoadAttribute(info);
+    } else {
+        return createAndSetupEmptyAttribute(info);
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h
new file mode 100644
index 00000000000..bfcb9ad5225
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_attribute_factory.h"
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/common/indexmetainfo.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+/**
+ * Class used by an attribute manager to initialize and load attribute vectors from disk.
+ */
+class AttributeInitializer
+{
+public:
+    typedef std::unique_ptr<AttributeInitializer> UP;
+
+    struct AttributeHeader
+    {
+        uint64_t _createSerialNum;
+        bool _headerTypeOK;
+        vespalib::string _btString;
+        vespalib::string _ctString;
+        AttributeHeader();
+    };
+
+private:
+    const vespalib::string          _baseDir;
+    const vespalib::string          _documentSubDbName;
+    const vespalib::string          _attrName;
+    const search::attribute::Config _cfg;
+    const uint64_t                  _currentSerialNum;
+    const IAttributeFactory        &_factory;
+
+    search::AttributeVector::SP tryLoadAttribute(const search::IndexMetaInfo &info) const;
+
+    bool loadAttribute(const search::AttributeVector::SP &attr,
+                       const search::IndexMetaInfo::Snapshot &snap) const;
+
+    void setupEmptyAttribute(search::AttributeVector::SP &attr,
+                             const search::IndexMetaInfo::Snapshot &snap,
+                             const AttributeHeader &header) const;
+
+    search::AttributeVector::SP createAndSetupEmptyAttribute(search::IndexMetaInfo &info) const;
+
+public:
+    AttributeInitializer(const vespalib::string &baseDir,
+                         const vespalib::string &documentSubDbName,
+                         const vespalib::string &attrName,
+                         const search::attribute::Config &cfg,
+                         uint64_t currentSerialNum,
+                         const IAttributeFactory &factory);
+
+    search::AttributeVector::SP init() const;
+    uint64_t getCurrentSerialNum() const { return _currentSerialNum; }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp
new file mode 100644
index 00000000000..37d4c0fa5b2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.attribute_manager_explorer");
+
+#include "attribute_manager_explorer.h"
+#include "attribute_vector_explorer.h"
+#include "exclusive_attribute_read_accessor.h"
+#include <vespa/searchlib/attribute/attributeguard.h>
+
+using vespalib::slime::Inserter;
+
+namespace proton {
+
+AttributeManagerExplorer::AttributeManagerExplorer(const proton::IAttributeManager::SP &mgr)
+    : _mgr(mgr)
+{
+}
+
+void
+AttributeManagerExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    (void) full;
+    inserter.insertObject();
+}
+
+std::vector<vespalib::string>
+AttributeManagerExplorer::get_children_names() const
+{
+    std::vector<search::AttributeGuard> attributes;
+    _mgr->getAttributeListAll(attributes);
+    std::vector<vespalib::string> names;
+    for (const auto &attr : attributes) {
+        names.push_back(attr.get().getName());
+    }
+    return names;
+}
+
+std::unique_ptr<vespalib::StateExplorer>
+AttributeManagerExplorer::get_child(vespalib::stringref name) const
+{
+    ExclusiveAttributeReadAccessor::UP attr = _mgr->getExclusiveReadAccessor(name);
+    if (attr.get() != nullptr) {
+        return std::unique_ptr<vespalib::StateExplorer>(new AttributeVectorExplorer(std::move(attr)));
+    }
+    return std::unique_ptr<vespalib::StateExplorer>();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.h
new file mode 100644
index 00000000000..cf3e764911c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_attribute_manager.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of an attribute manager and its attribute vectors.
+ */
+class AttributeManagerExplorer : public vespalib::StateExplorer
+{
+private:
+    proton::IAttributeManager::SP _mgr;
+
+public:
+    AttributeManagerExplorer(const proton::IAttributeManager::SP &mgr);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+    virtual std::vector<vespalib::string> get_children_names() const override;
+    virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp
new file mode 100644
index 00000000000..98509cb5315
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "attribute_manager_initializer.h"
+#include "attribute_collection_spec_factory.h"
+#include "sequential_attributes_initializer.h"
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+
+using search::AttributeVector;
+using search::GrowStrategy;
+using search::SerialNum;
+using vespa::config::search::AttributesConfig;
+
+namespace proton {
+
+using initializer::InitializerTask;
+
+namespace {
+
+class AttributeInitializerTask : public InitializerTask
+{
+private:
+    AttributeInitializer::UP _initializer;
+    DocumentMetaStore::SP _documentMetaStore;
+    InitializedAttributesResult &_result;
+
+public:
+    AttributeInitializerTask(AttributeInitializer::UP initializer,
+                             DocumentMetaStore::SP documentMetaStore,
+                             InitializedAttributesResult &result)
+        : _initializer(std::move(initializer)),
+          _documentMetaStore(documentMetaStore),
+          _result(result)
+    {}
+
+    virtual void run() override {
+        AttributeVector::SP attribute = _initializer->init();
+        if (attribute) {
+            AttributesInitializerBase::considerPadAttribute(*attribute,
+                                                            _initializer->getCurrentSerialNum(),
+                                                            _documentMetaStore->getCommittedDocIdLimit());
+            _result.add(attribute);
+        }
+    }
+};
+
+class AttributeInitializerTasksBuilder : public IAttributeInitializerRegistry
+{
+private:
+    InitializerTask &_attrMgrInitTask;
+    InitializerTask::SP _documentMetaStoreInitTask;
+    DocumentMetaStore::SP _documentMetaStore;
+    InitializedAttributesResult &_attributesResult;
+
+public:
+    AttributeInitializerTasksBuilder(InitializerTask &attrMgrInitTask,
+                                     InitializerTask::SP documentMetaStoreInitTask,
+                                     DocumentMetaStore::SP documentMetaStore,
+                                     InitializedAttributesResult &attributesResult)
+        : _attrMgrInitTask(attrMgrInitTask),
+          _documentMetaStoreInitTask(documentMetaStoreInitTask),
+          _documentMetaStore(documentMetaStore),
+          _attributesResult(attributesResult)
+    {}
+    virtual void add(AttributeInitializer::UP initializer) override {
+        InitializerTask::SP attributeInitTask =
+                std::make_shared<AttributeInitializerTask>(std::move(initializer),
+                                                           _documentMetaStore,
+                                                           _attributesResult);
+        attributeInitTask->addDependency(_documentMetaStoreInitTask);
+        _attrMgrInitTask.addDependency(attributeInitTask);
+    }
+};
+
+}
+
+AttributeCollectionSpec::UP
+AttributeManagerInitializer::createAttributeSpec() const
+{
+    uint32_t docIdLimit = 1; // The real docIdLimit is used after attributes are loaded to pad them
+    AttributeCollectionSpecFactory factory(_attributeGrow, _attributeGrowNumDocs, _fastAccessAttributesOnly);
+    return factory.create(_attrCfg, docIdLimit, _configSerialNum);
+}
+
+AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialNum,
+                                                         initializer::InitializerTask::SP documentMetaStoreInitTask,
+                                                         DocumentMetaStore::SP documentMetaStore,
+                                                         AttributeManager::SP baseAttrMgr,
+                                                         const AttributesConfig &attrCfg,
+                                                         const GrowStrategy &attributeGrow,
+                                                         size_t attributeGrowNumDocs,
+                                                         bool fastAccessAttributesOnly,
+                                                         std::shared_ptr<AttributeManager::SP> attrMgrResult)
+    : _configSerialNum(configSerialNum),
+      _documentMetaStore(documentMetaStore),
+      _attrMgr(),
+      _attrCfg(attrCfg),
+      _attributeGrow(attributeGrow),
+      _attributeGrowNumDocs(attributeGrowNumDocs),
+      _fastAccessAttributesOnly(fastAccessAttributesOnly),
+      _attributesResult(),
+      _attrMgrResult(attrMgrResult)
+{
+    addDependency(documentMetaStoreInitTask);
+    AttributeInitializerTasksBuilder tasksBuilder(*this, documentMetaStoreInitTask, documentMetaStore, _attributesResult);
+    AttributeCollectionSpec::UP attrSpec = createAttributeSpec();
+    _attrMgr = std::make_shared<AttributeManager>(*baseAttrMgr, *attrSpec, tasksBuilder);
+}
+
+void
+AttributeManagerInitializer::run()
+{
+    _attrMgr->addExtraAttribute(_documentMetaStore);
+    _attrMgr->addInitializedAttributes(_attributesResult.get());
+    *_attrMgrResult = _attrMgr;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h
new file mode 100644
index 00000000000..9e1857d5f00
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attributemanager.h"
+#include "initialized_attributes_result.h"
+#include <vespa/searchcommon/common/growstrategy.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/config-attributes.h>
+
+namespace proton {
+
+/**
+ * Class used to initialize an attribute manager.
+ */
+class AttributeManagerInitializer : public initializer::InitializerTask
+{
+private:
+    search::SerialNum _configSerialNum;
+    DocumentMetaStore::SP _documentMetaStore;
+    AttributeManager::SP _attrMgr;
+    vespa::config::search::AttributesConfig _attrCfg;
+    search::GrowStrategy _attributeGrow;
+    size_t _attributeGrowNumDocs;
+    bool _fastAccessAttributesOnly;
+    InitializedAttributesResult _attributesResult;
+    std::shared_ptr<AttributeManager::SP> _attrMgrResult;
+
+    AttributeCollectionSpec::UP createAttributeSpec() const;
+
+public:
+    AttributeManagerInitializer(search::SerialNum configSerialNum,
+                                initializer::InitializerTask::SP documentMetaStoreInitTask,
+                                DocumentMetaStore::SP documentMetaStore,
+                                AttributeManager::SP baseAttrMgr,
+                                const vespa::config::search::AttributesConfig &attrCfg,
+                                const search::GrowStrategy &attributeGrow,
+                                size_t attributeGrowNumDocs,
+                                bool fastAccessAttributesOnly,
+                                std::shared_ptr<AttributeManager::SP> attrMgrResult);
+
+    virtual void run() override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp
new file mode 100644
index 00000000000..4d72da90d35
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.attribute_populator");
+#include "attribute_populator.h"
+
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/searchlib/common/idestructorcallback.h>
+
+using search::IDestructorCallback;
+
+namespace proton {
+
+search::SerialNum
+AttributePopulator::nextSerialNum()
+{
+    return _currSerialNum++;
+}
+
+std::vector<vespalib::string>
+AttributePopulator::getNames() const
+{
+    std::vector<search::AttributeGuard> attrs;
+    _writer.getAttributeManager()->getAttributeList(attrs);
+    std::vector<vespalib::string> names;
+    for (const search::AttributeGuard &attr : attrs) {
+        names.push_back(_subDbName + ".attribute." + attr.get().getName());
+    }
+    return names;
+}
+
+AttributePopulator::AttributePopulator(const proton::IAttributeManager::SP &mgr,
+                                       search::SerialNum initSerialNum,
+                                       const vespalib::string &subDbName)
+    : _writer(mgr),
+      _initSerialNum(initSerialNum),
+      _currSerialNum(initSerialNum),
+      _subDbName(subDbName)
+{
+    if (LOG_WOULD_LOG(event)) {
+        EventLogger::populateAttributeStart(getNames());
+    }
+}
+
+AttributePopulator::~AttributePopulator()
+{
+    if (LOG_WOULD_LOG(event)) {
+        EventLogger::populateAttributeComplete(getNames(),
+                _currSerialNum - _initSerialNum);
+    }
+}
+
+void
+AttributePopulator::handleExisting(uint32_t lid, const document::Document &doc)
+{
+    search::SerialNum serialNum(nextSerialNum());
+    _writer.put(serialNum, doc, lid, true, std::shared_ptr<IDestructorCallback>());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.h
new file mode 100644
index 00000000000..b6c53af6bac
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attribute_writer.h"
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_reader.h>
+
+namespace proton {
+
+/**
+ * Class used to populate attribute vectors based on visiting the content of a document store.
+ */
+class AttributePopulator : public IReprocessingReader
+{
+private:
+    AttributeWriter  _writer;
+    search::SerialNum _initSerialNum;
+    search::SerialNum _currSerialNum;
+    vespalib::string  _subDbName;
+
+    search::SerialNum nextSerialNum();
+
+    std::vector<vespalib::string> getNames() const;
+
+public:
+    typedef std::shared_ptr<AttributePopulator> SP;
+
+    AttributePopulator(const proton::IAttributeManager::SP &mgr,
+                       search::SerialNum initSerialNum,
+                       const vespalib::string &subDbName);
+    ~AttributePopulator();
+
+    const IAttributeWriter &getWriter() const { return _writer; }
+
+    // Implements IReprocessingReader
+    virtual void handleExisting(uint32_t lid, const document::Document &doc);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp
new file mode 100644
index 00000000000..afb478d4ca1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp
@@ -0,0 +1,148 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "attribute_usage_filter.h"
+#include <sstream>
+
+namespace proton {
+
+namespace
+{
+
+void makeAddressSpaceMessage(std::ostream &os,
+                              const AddressSpaceUsageStats &usage)
+{
+    os << "{ used: " <<
+        usage.getUsage().used() << ", limit: " <<
+        usage.getUsage().limit() << "}, attributeName: \"" <<
+        usage.getAttributeName() << "\", subdb: \"" <<
+        usage.getSubDbName() << "\"}";
+}
+
+void makeEnumStoreMessage(std::ostream &os,
+                          double used, double limit,
+                          const AddressSpaceUsageStats &usage)
+{
+    os << "enumStoreLimitReached: { "
+        "action: \""
+        "add more content nodes"
+        "\", "
+        "reason: \""
+        "enum store address space used (" << used << ") > "
+        "limit (" << limit << ")"
+        "\", enumStore: ";
+    makeAddressSpaceMessage(os, usage);
+}
+
+void makeMultiValueMessage(std::ostream &os,
+                           double used, double limit,
+                           const AddressSpaceUsageStats &usage)
+{
+    os << "multiValueLimitReached: { "
+        "action: \""
+        "use 'huge' setting on attribute field or add more content nodes"
+        "\", "
+        "reason: \""
+        "multiValue address space used (" << used << ") > "
+        "limit (" << limit << ")"
+        "\", multiValue: ";
+    makeAddressSpaceMessage(os, usage);
+}
+
+}
+
+void
+AttributeUsageFilter::recalcState(const Guard &guard)
+{
+    (void) guard;
+    bool hasMessage = false;
+    std::ostringstream message;
+    const AddressSpaceUsageStats &enumStoreUsage =
+        _attributeStats.enumStoreUsage();
+    double enumStoreUsed = enumStoreUsage.getUsage().usage();
+    if (enumStoreUsed > _config._enumStoreLimit) {
+        hasMessage = true;
+        makeEnumStoreMessage(message, enumStoreUsed, _config._enumStoreLimit,
+                             enumStoreUsage);
+    }
+    const AddressSpaceUsageStats &multiValueUsage =
+        _attributeStats.multiValueUsage();
+    double multiValueUsed = multiValueUsage.getUsage().usage();
+    if (multiValueUsed > _config._multiValueLimit) {
+        if (hasMessage) {
+            message << ", ";
+        }
+        hasMessage = true;
+        makeMultiValueMessage(message, multiValueUsed, _config._multiValueLimit,
+                              multiValueUsage);
+    }
+    if (hasMessage) {
+        _state = State(false, message.str());
+        _acceptWrite = false;
+    } else {
+        _state = State();
+        _acceptWrite = true;
+    }
+}
+
+AttributeUsageFilter::AttributeUsageFilter()
+    : _lock(),
+      _attributeStats(),
+      _config(),
+      _state(),
+      _acceptWrite(true)
+{
+}
+
+
+void
+AttributeUsageFilter::setAttributeStats(AttributeUsageStats attributeStats_in)
+{
+    Guard guard(_lock);
+    _attributeStats = attributeStats_in;
+    recalcState(guard);
+}
+
+AttributeUsageStats
+AttributeUsageFilter::getAttributeUsageStats() const
+{
+    Guard guard(_lock);
+    return _attributeStats;
+}
+
+void
+AttributeUsageFilter::setConfig(Config config_in)
+{
+    Guard guard(_lock);
+    _config = config_in;
+    recalcState(guard);
+}
+
+double
+AttributeUsageFilter::getEnumStoreUsedRatio() const
+{
+    Guard guard(_lock);
+    return _attributeStats.enumStoreUsage().getUsage().usage();
+}
+
+double
+AttributeUsageFilter::getMultiValueUsedRatio() const
+{
+    Guard guard(_lock);
+    return _attributeStats.multiValueUsage().getUsage().usage();
+}
+
+bool
+AttributeUsageFilter::acceptWriteOperation() const
+{
+    return _acceptWrite;
+}
+
+AttributeUsageFilter::State
+AttributeUsageFilter::getAcceptState() const
+{
+    Guard guard(_lock);
+    return _state;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h
new file mode 100644
index 00000000000..2863fff2805
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
+#include "attribute_usage_stats.h"
+#include "attribute_usage_filter_config.h"
+#include <mutex>
+#include <atomic>
+
+namespace proton {
+
+/*
+ * Class to filter write operations based on sampled information about
+ * attribute resource usage (e.g. enum store and multivalue mapping).
+ * If resource limit is reached then further writes are denied in
+ * order to prevent entering an unrecoverable state.
+ */
+class AttributeUsageFilter : public IResourceWriteFilter {
+public:
+    using Mutex = std::mutex;
+    using Guard = std::lock_guard<Mutex>;
+
+    using Config = AttributeUsageFilterConfig;
+
+private:
+    mutable Mutex _lock; // protect _attributeStats, _config, _state
+    AttributeUsageStats _attributeStats;
+    Config _config;
+    State _state;
+    std::atomic<bool> _acceptWrite;
+
+    void recalcState(const Guard &guard); // called with _lock held
+public:
+    AttributeUsageFilter();
+    void setAttributeStats(AttributeUsageStats attributeStats_in);
+    AttributeUsageStats getAttributeUsageStats() const;
+    void setConfig(Config config);
+    double getEnumStoreUsedRatio() const;
+    double getMultiValueUsedRatio() const;
+    virtual bool acceptWriteOperation() const override;
+    virtual State getAcceptState() const override;
+};
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter_config.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter_config.h
new file mode 100644
index 00000000000..7612543554c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter_config.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton {
+
+/*
+ * Struct representing config for when to filter write operations
+ * due to attribute resource usage (e.g. enum store and multivalue mapping).
+ * If resource limit is reached then further writes are denied in
+ * order to prevent entering an unrecoverable state.
+ *
+ * The config is used by AttributeUsageFilter.
+ */
+struct AttributeUsageFilterConfig
+{
+    double _enumStoreLimit;
+    double _multiValueLimit;
+
+    AttributeUsageFilterConfig()
+        : _enumStoreLimit(1.0),
+          _multiValueLimit(1.0)
+    {
+    }
+
+    AttributeUsageFilterConfig(double enumStoreLimit_in,
+                               double multiValueLimit_in)
+        : _enumStoreLimit(enumStoreLimit_in),
+          _multiValueLimit(multiValueLimit_in)
+    {
+    }
+
+    bool operator==(const AttributeUsageFilterConfig &rhs) const {
+        return ((_enumStoreLimit == rhs._enumStoreLimit) &&
+                (_multiValueLimit == rhs._multiValueLimit));
+    }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp
new file mode 100644
index 00000000000..c4914580e34
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "attribute_usage_sampler_context.h"
+#include "attribute_usage_filter.h"
+
+namespace proton {
+
+AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter &filter)
+    : _usage(),
+      _lock(),
+      _filter(filter)
+{
+}
+
+AttributeUsageSamplerContext::~AttributeUsageSamplerContext()
+{
+    _filter.setAttributeStats(_usage);
+}
+
+void
+AttributeUsageSamplerContext::merge(const search::AddressSpaceUsage &usage,
+                                    const vespalib::string &attributeName,
+                                    const vespalib::string &subDbName)
+{
+    Guard guard(_lock);
+    _usage.merge(usage, attributeName, subDbName);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h
new file mode 100644
index 00000000000..f6310746be7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attribute_usage_stats.h"
+#include <mutex>
+
+namespace proton {
+
+class AttributeUsageFilter;
+
+/*
+ * Context for sampling attribute usage stats.  When instance is
+ * destroyed, the aggregated stats is passed on to attribute usage filter.
+ */
+class AttributeUsageSamplerContext
+{
+    using Mutex = std::mutex;
+    using Guard = std::lock_guard<Mutex>;
+
+    AttributeUsageStats _usage;
+    Mutex _lock;
+    AttributeUsageFilter &_filter;
+public:
+    AttributeUsageSamplerContext(AttributeUsageFilter &filter);
+    ~AttributeUsageSamplerContext();
+    void merge(const search::AddressSpaceUsage &usage,
+               const vespalib::string &attributeName,
+               const vespalib::string &subDbName);
+    const AttributeUsageStats &
+    getUsage() const { return _usage; }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp
new file mode 100644
index 00000000000..1854e243ae3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "attribute_usage_sampler_functor.h"
+#include "attribute_usage_sampler_context.h"
+#include <vespa/searchlib/attribute/attributevector.h>
+
+namespace proton {
+
+AttributeUsageSamplerFunctor::AttributeUsageSamplerFunctor(
+        std::shared_ptr<AttributeUsageSamplerContext> samplerContext,
+        const std::string &subDbName)
+    : _samplerContext(samplerContext),
+      _subDbName(subDbName)
+{
+}
+
+AttributeUsageSamplerFunctor::~AttributeUsageSamplerFunctor()
+{
+}
+
+void
+AttributeUsageSamplerFunctor::operator()(const search::AttributeVector &
+                                         attributeVector)
+{
+    // Executed by attribute writer thread
+    search::AddressSpaceUsage usage = attributeVector.getAddressSpaceUsage();
+    vespalib::string attributeName = attributeVector.getName();
+    _samplerContext->merge(usage, attributeName, _subDbName);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h
new file mode 100644
index 00000000000..61565562232
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_attribute_functor.h"
+
+namespace proton {
+
+class AttributeUsageSamplerContext;
+
+/*
+ * Functor for sampling attribute usage and passing it on to sampler
+ * context.
+ */
+class AttributeUsageSamplerFunctor : public IAttributeFunctor
+{
+    std::shared_ptr<AttributeUsageSamplerContext> _samplerContext;
+    std::string _subDbName;
+public:
+    AttributeUsageSamplerFunctor(std::shared_ptr<AttributeUsageSamplerContext>
+                                 samplerContext,
+                                 const std::string &subDbname);
+    ~AttributeUsageSamplerFunctor();
+    virtual void
+    operator()(const search::AttributeVector &attributeVector) override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp
new file mode 100644
index 00000000000..aa26b6d81df
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "attribute_usage_stats.h"
+
+namespace proton {
+
+AttributeUsageStats::AttributeUsageStats()
+    : _enumStoreUsage(search::AddressSpaceUsage::defaultEnumStoreUsage()),
+      _multiValueUsage(search::AddressSpaceUsage::defaultMultiValueUsage())
+{
+}
+
+void
+AttributeUsageStats::merge(const search::AddressSpaceUsage &usage,
+                           const vespalib::string &attributeName,
+                           const vespalib::string &subDbName)
+{
+    _enumStoreUsage.merge(usage.enumStoreUsage(), attributeName, subDbName);
+    _multiValueUsage.merge(usage.multiValueUsage(), attributeName, subDbName);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h
new file mode 100644
index 00000000000..a85d9ab3cdd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "address_space_usage_stats.h"
+#include <vespa/searchlib/attribute/address_space_usage.h>
+
+namespace proton {
+
+/*
+ * Class representing aggregated attribute usage, with info about
+ * the most bloated attributes with regards to enum store and
+ * multivalue mapping.
+ */
+class AttributeUsageStats
+{
+    AddressSpaceUsageStats _enumStoreUsage;
+    AddressSpaceUsageStats _multiValueUsage;
+
+public:
+    AttributeUsageStats();
+    void merge(const search::AddressSpaceUsage &usage,
+               const vespalib::string &attributeName,
+               const vespalib::string &subDbName);
+
+    const AddressSpaceUsageStats &
+    enumStoreUsage() const { return _enumStoreUsage; }
+    const AddressSpaceUsageStats &
+    multiValueUsage() const { return _multiValueUsage; }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp
new file mode 100644
index 00000000000..d824f381ae7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.attribute_vector_explorer");
+
+#include "attribute_vector_explorer.h"
+#include <vespa/searchlib/attribute/enumstorebase.h>
+#include <vespa/searchlib/attribute/multivaluemapping.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+
+using search::attribute::Status;
+using search::AddressSpace;
+using search::AddressSpaceUsage;
+using search::AttributeVector;
+using search::EnumStoreBase;
+using search::MemoryUsage;
+using search::MultiValueMappingBaseBase;
+using namespace vespalib::slime;
+
+namespace proton {
+
+namespace {
+
+void
+convertStatusToSlime(const Status &status, Cursor &object)
+{
+    object.setLong("numDocs", status.getNumDocs());
+    object.setLong("numValues", status.getNumValues());
+    object.setLong("numUniqueValues", status.getNumUniqueValues());
+    object.setLong("lastSerialNum", status.getLastSyncToken());
+    object.setLong("updateCount", status.getUpdateCount());
+    object.setLong("nonIdempotentUpdateCount", status.getNonIdempotentUpdateCount());
+    object.setLong("bitVectors", status.getBitVectors());
+    {
+        Cursor &memory = object.setObject("memoryUsage");
+        memory.setLong("allocated", status.getAllocated());
+        memory.setLong("used", status.getUsed());
+        memory.setLong("dead", status.getDead());
+        memory.setLong("onHold", status.getOnHold());
+        memory.setLong("onHoldMax", status.getOnHoldMax());
+    }
+}
+
+void
+convertGenerationToSlime(const AttributeVector &attr, Cursor &object)
+{
+    object.setLong("firstUsed", attr.getFirstUsedGeneration());
+    object.setLong("current", attr.getCurrentGeneration());
+}
+
+void
+convertAddressSpaceToSlime(const AddressSpace &addressSpace, Cursor &object)
+{
+    object.setDouble("usage", addressSpace.usage());
+    object.setLong("used", addressSpace.used());
+    object.setLong("limit", addressSpace.limit());
+}
+
+void
+convertAddressSpaceUsageToSlime(const AddressSpaceUsage &usage, Cursor &object)
+{
+    convertAddressSpaceToSlime(usage.enumStoreUsage(), object.setObject("enumStore"));
+    convertAddressSpaceToSlime(usage.multiValueUsage(), object.setObject("multiValue"));
+}
+
+void
+convertMemoryUsageToSlime(const MemoryUsage &usage, Cursor &object)
+{
+    object.setLong("allocated", usage.allocatedBytes());
+    object.setLong("used", usage.usedBytes());
+    object.setLong("dead", usage.deadBytes());
+    object.setLong("onHold", usage.allocatedBytesOnHold());
+}
+
+void
+convertEnumStoreToSlime(const EnumStoreBase &enumStore, Cursor &object)
+{
+    object.setLong("lastEnum", enumStore.getLastEnum());
+    object.setLong("numUniques", enumStore.getNumUniques());
+    convertMemoryUsageToSlime(enumStore.getMemoryUsage(), object.setObject("memoryUsage"));
+    convertMemoryUsageToSlime(enumStore.getTreeMemoryUsage(), object.setObject("treeMemoryUsage"));
+}
+
+void
+convertMultiValueToSlime(const MultiValueMappingBaseBase &multiValue, Cursor &object)
+{
+    object.setLong("totalValueCnt", multiValue.getTotalValueCnt());
+    convertMemoryUsageToSlime(multiValue.getMemoryUsage(), object.setObject("memoryUsage"));
+}
+
+}
+
+AttributeVectorExplorer::AttributeVectorExplorer(ExclusiveAttributeReadAccessor::UP attribute)
+    : _attribute(std::move(attribute))
+{
+}
+
+void
+AttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bool full) const
+{
+    ExclusiveAttributeReadAccessor::Guard::UP readGuard = _attribute->takeGuard();
+    const AttributeVector &attr = readGuard->get();
+    const Status &status = attr.getStatus();
+    Cursor &object = inserter.insertObject();
+    if (full) {
+        convertStatusToSlime(status, object.setObject("status"));
+        convertGenerationToSlime(attr, object.setObject("generation"));
+        convertAddressSpaceUsageToSlime(attr.getAddressSpaceUsage(), object.setObject("addressSpaceUsage"));
+        const EnumStoreBase *enumStore = attr.getEnumStoreBase();
+        if (enumStore) {
+            convertEnumStoreToSlime(*enumStore, object.setObject("enumStore"));
+        }
+        const MultiValueMappingBaseBase *multiValue = attr.getMultiValueBase();
+        if (multiValue) {
+            convertMultiValueToSlime(*multiValue, object.setObject("multiValue"));
+        }
+        object.setLong("committedDocIdLimit", attr.getCommittedDocIdLimit());
+        object.setLong("createSerialNum", attr.getCreateSerialNum());
+    } else {
+        object.setLong("numDocs", status.getNumDocs());
+        object.setLong("lastSerialNum", status.getLastSyncToken());
+        object.setLong("allocatedMemory", status.getAllocated());
+        object.setLong("committedDocIdLimit", attr.getCommittedDocIdLimit());
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h
new file mode 100644
index 00000000000..17b349109f5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "exclusive_attribute_read_accessor.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of an attribute vector.
+ */
+class AttributeVectorExplorer : public vespalib::StateExplorer
+{
+private:
+    ExclusiveAttributeReadAccessor::UP _attribute;
+
+public:
+    AttributeVectorExplorer(ExclusiveAttributeReadAccessor::UP attribute);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
new file mode 100644
index 00000000000..3ab81dadc96
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -0,0 +1,332 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.attributeadapter");
+
+#include "attribute_writer.h"
+#include "attributemanager.h"
+#include <vespa/searchcore/proton/common/attrupdate.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/floatbase.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/stringbase.h>
+#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+
+using namespace document;
+using namespace search;
+
+namespace proton {
+
+namespace {
+
+void
+ensureLidSpace(SerialNum serialNum, DocumentIdT lid, AttributeVector &attr)
+{
+    size_t docIdLimit = lid + 1;
+    if (attr.getStatus().getLastSyncToken() < serialNum) {
+        AttributeManager::padAttribute(attr, docIdLimit);
+    }
+}
+
+void
+applyPutToAttribute(SerialNum serialNum, const FieldValue::UP &fieldValue, DocumentIdT lid,
+                    bool immediateCommit, AttributeVector &attr,
+                    AttributeWriter::OnWriteDoneType)
+{
+    ensureLidSpace(serialNum, lid, attr);
+    if (fieldValue.get()) {
+        AttrUpdate::handleValue(attr, lid, *fieldValue);
+    } else {
+        attr.clearDoc(lid);
+    }
+    if (immediateCommit) {
+        attr.commit(serialNum, serialNum);
+    }
+}
+
+void
+applyRemoveToAttribute(SerialNum serialNum, DocumentIdT lid, bool immediateCommit,
+                       AttributeVector &attr, AttributeWriter::OnWriteDoneType)
+{
+    ensureLidSpace(serialNum, lid, attr);
+    attr.clearDoc(lid);
+    if (immediateCommit) {
+        attr.commit(serialNum, serialNum);
+    }
+}
+
+void
+applyUpdateToAttribute(SerialNum serialNum, const FieldUpdate &fieldUpd,
+                       DocumentIdT lid, bool immediateCommit, AttributeVector &attr,
+                       AttributeWriter::OnWriteDoneType)
+{
+    ensureLidSpace(serialNum, lid, attr);
+    AttrUpdate::handleUpdate(attr, lid, fieldUpd);
+    if (immediateCommit) {
+        attr.commit(serialNum, serialNum);
+    }
+}
+
+
+void
+applyReplayDone(uint32_t docIdLimit, AttributeVector &attr)
+{
+    AttributeManager::padAttribute(attr, docIdLimit);
+    attr.compactLidSpace(docIdLimit);
+}
+
+
+void
+applyHeartBeat(SerialNum serialNum, AttributeVector &attr)
+{
+    attr.removeAllOldGenerations();
+    if (attr.getStatus().getLastSyncToken() <= serialNum) {
+        attr.commit(serialNum, serialNum);
+    }
+}
+
+void
+applyCommit(SerialNum serialNum, AttributeWriter::OnWriteDoneType onWriteDone,
+            AttributeVector &attr)
+{
+    (void) onWriteDone;
+    if (attr.getStatus().getLastSyncToken() <= serialNum) {
+        attr.commit(serialNum, serialNum);
+    }
+}
+
+
+void
+applyCompactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum,
+                     AttributeVector &attr)
+{
+    if (attr.getStatus().getLastSyncToken() < serialNum) {
+        attr.compactLidSpace(wantedLidLimit);
+        attr.commit(serialNum, serialNum);
+    }
+}
+
+}
+
+void
+AttributeWriter::buildFieldPath(const DocumentType & docType, const DataType *dataType)
+{
+    _fieldPaths.clear();
+    _fieldPaths.resize(_writableAttributes.size());
+    size_t i = 0;
+    for (auto attrp : _writableAttributes) {
+        const search::attribute::IAttributeVector & attribute(*attrp);
+        FieldPath::UP fp = docType.buildFieldPath(attribute.getName());
+        if (fp.get() == NULL) {
+            /// Should be exception but due to incomplete unit test we can not be strict enough, must fix unit test proton/docsummary
+            // The above comment is actually incorrect. This is expected during reconfig as long as do not stop accepting feed while doing reconfig.
+            // throw std::runtime_error(vespalib::make_string("Mismatch between documentdefinition and schema. No field named '%s' from schema in document type '%s'", attribute.getName().c_str(), docType.getName().c_str()));
+            LOG(warning,
+                "Mismatch between documentdefinition and schema. "
+                "No field named '%s' from schema in document type '%s'. "
+                "This might happen if an attribute field has been added and you are feeding while reconfiguring",
+                attribute.getName().c_str(),
+                docType.getName().c_str());
+        }
+        _fieldPaths[i] = std::move(fp);
+        ++i;
+    }
+    _dataType = dataType;
+}
+
+void
+AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid,
+                             bool immediateCommit, OnWriteDoneType onWriteDone)
+{
+    size_t fieldId = 0;
+    for (auto attrp : _writableAttributes) {
+        AttributeVector &attr = *attrp;
+        if (attr.getStatus().getLastSyncToken() >= serialNum) {
+            LOG(debug, "internalPut(): change already applied: serial=%" PRIu64 ""
+                    ", docId='%s', lid=%u, attribute='%s', lastSyncToken=%" PRIu64 "",
+                    serialNum, doc.getId().toString().c_str(), lid, attr.getName().c_str(),
+                    attr.getStatus().getLastSyncToken());
+            ++fieldId;
+            continue;	// Change already applied
+        }
+        const FieldPath *const fieldPath(_fieldPaths[fieldId].get());
+        FieldValue::UP fv;
+        if (fieldPath != NULL) {
+            fv = doc.getNestedFieldValue(fieldPath->begin(),
+                                         fieldPath->end());
+        }
+
+        _attributeFieldWriter.execute(attr.getName(),
+                [serialNum, fv(std::move(fv)), lid, immediateCommit, &attr, onWriteDone]()
+                { applyPutToAttribute(serialNum, fv, lid, immediateCommit, attr, onWriteDone); });
+        ++fieldId;
+    }
+}
+
+void
+AttributeWriter::internalRemove(SerialNum serialNum, DocumentIdT lid,
+                                bool immediateCommit,
+                                OnWriteDoneType onWriteDone)
+{
+    for (auto &attrp : _writableAttributes) {
+        AttributeVector &attr = *attrp;
+        // XXX: Want to use >=, but must use > due to batch remove
+        // Might be OK due to clearDoc() being idempotent.
+        if (attr.getStatus().getLastSyncToken() > serialNum)
+            continue;	// Change already applied
+        LOG(debug, "About to remove docId %u from attribute vector '%s'.",
+            lid, attr.getName().c_str());
+
+        _attributeFieldWriter.execute(attr.getName(),
+                [serialNum, lid, immediateCommit, &attr, onWriteDone]()
+                { applyRemoveToAttribute(serialNum, lid, immediateCommit, attr, onWriteDone); });
+    }
+}
+
+AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr)
+    : _mgr(mgr),
+      _fieldPaths(),
+      _dataType(),
+      _fieldPathsDocTypeName(),
+      _attributeFieldWriter(mgr->getAttributeFieldWriter()),
+      _writableAttributes(mgr->getWritableAttributes())
+{
+}
+
+std::vector<search::AttributeVector *>
+AttributeWriter::getWritableAttributes() const
+{
+    return _mgr->getWritableAttributes();
+}
+
+
+search::AttributeVector *
+AttributeWriter::getWritableAttribute(const vespalib::string &name) const
+{
+    return _mgr->getWritableAttribute(name);
+}
+
+void
+AttributeWriter::put(SerialNum serialNum, const Document &doc, DocumentIdT lid,
+                     bool immediateCommit, OnWriteDoneType onWriteDone)
+{
+    FieldValue::UP attrVal;
+    LOG(spam,
+        "Handle put: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)",
+        serialNum,
+        doc.getId().toString().c_str(),
+        lid,
+        doc.toString(true).c_str());
+    const DataType *dataType(doc.getDataType());
+    if (_fieldPaths.empty() ||
+        _dataType != dataType ||
+        doc.getType().getName() != _fieldPathsDocTypeName) {
+        buildFieldPath(doc.getType(), dataType);
+        _fieldPathsDocTypeName = doc.getType().getName();
+    }
+    internalPut(serialNum, doc, lid, immediateCommit, onWriteDone);
+}
+
+void
+AttributeWriter::remove(SerialNum serialNum, DocumentIdT lid,
+                        bool immediateCommit, OnWriteDoneType onWriteDone)
+{
+    internalRemove(serialNum, lid, immediateCommit, onWriteDone);
+}
+
+void
+AttributeWriter::remove(const LidVector &lidsToRemove, SerialNum serialNum,
+                        bool immediateCommit, OnWriteDoneType onWriteDone)
+{
+    for (const auto &lid : lidsToRemove) {
+        internalRemove(serialNum, lid, immediateCommit, onWriteDone);
+    }
+}
+
+void
+AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid,
+                        bool immediateCommit, OnWriteDoneType onWriteDone)
+{
+    LOG(debug, "Inspecting update for document %d.", lid);
+    for (const auto &fupd : upd.getUpdates()) {
+        LOG(debug, "Retrieving guard for attribute vector '%s'.",
+            fupd.getField().getName().c_str());
+        AttributeVector *attrp =
+            _mgr->getWritableAttribute(fupd.getField().getName());
+        if (attrp == nullptr) {
+            LOG(spam, "Failed to find attribute vector %s",
+                fupd.getField().getName().c_str());
+            continue;
+        }
+        AttributeVector &attr = *attrp;
+        // TODO: Check if we must use > due to multiple entries for same
+        // document and attribute.
+        if (attr.getStatus().getLastSyncToken() >= serialNum)
+            continue;
+
+        LOG(debug, "About to apply update for docId %u in attribute vector '%s'.",
+                lid, attr.getName().c_str());
+
+        // NOTE: The lifetime of the field update will be ensured by keeping the document update alive
+        // in a operation done context object.
+        _attributeFieldWriter.execute(attr.getName(),
+                [serialNum, &fupd, lid, immediateCommit, &attr, onWriteDone]()
+                { applyUpdateToAttribute(serialNum, fupd, lid, immediateCommit, attr, onWriteDone); });
+    }
+}
+
+void
+AttributeWriter::heartBeat(SerialNum serialNum)
+{
+    for (auto attrp : _writableAttributes) {
+        auto &attr = *attrp;
+        _attributeFieldWriter.execute(attr.getName(),
+                                      [serialNum, &attr]()
+                                      { applyHeartBeat(serialNum, attr); });
+    }
+}
+
+
+void
+AttributeWriter::commit(SerialNum serialNum, OnWriteDoneType onWriteDone)
+{
+    for (auto attrp : _writableAttributes) {
+        auto &attr = *attrp;
+        _attributeFieldWriter.execute(attr.getName(),
+                                      [serialNum, onWriteDone, &attr]()
+                                      { applyCommit(serialNum, onWriteDone,
+                                                    attr); });
+    }
+}
+
+
+void
+AttributeWriter::onReplayDone(uint32_t docIdLimit)
+{
+    for (auto attrp : _writableAttributes) {
+        auto &attr = *attrp;
+        _attributeFieldWriter.execute(attr.getName(),
+                                      [docIdLimit, &attr]()
+                                      { applyReplayDone(docIdLimit, attr); });
+    }
+    _attributeFieldWriter.sync();
+}
+
+
+void
+AttributeWriter::compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum)
+{
+    for (auto attrp : _writableAttributes) {
+        auto &attr = *attrp;
+        _attributeFieldWriter.
+            execute(attr.getName(),
+                    [wantedLidLimit, serialNum, &attr]()
+                    { applyCompactLidSpace(wantedLidLimit, serialNum, attr); });
+    }
+    _attributeFieldWriter.sync();
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
new file mode 100644
index 00000000000..e0db78bcbcd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_attribute_manager.h"
+#include "i_attribute_writer.h"
+#include <vespa/searchcore/proton/common/commit_time_tracker.h>
+
+namespace proton {
+
+/**
+ * Concrete attribute writer that handles writes in form of put, update and remove
+ * to the attribute vectors managed by the underlying attribute manager.
+ */
+class AttributeWriter : public IAttributeWriter
+{
+private:
+    typedef search::AttributeVector AttributeVector;
+    typedef document::FieldPath FieldPath;
+    typedef document::DataType DataType;
+    typedef document::DocumentType DocumentType;
+    typedef document::FieldValue FieldValue;
+    typedef std::vector<std::unique_ptr<FieldPath> > AttributeFieldPaths;
+    const IAttributeManager::SP _mgr;
+    AttributeFieldPaths         _fieldPaths;
+    const DataType             *_dataType;
+    vespalib::string            _fieldPathsDocTypeName;
+    search::ISequencedTaskExecutor &_attributeFieldWriter;
+    const std::vector<search::AttributeVector *> &_writableAttributes;
+
+    void buildFieldPath(const DocumentType &docType, const DataType *dataType);
+    void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid,
+                     bool immediateCommit, OnWriteDoneType onWriteDone);
+    void internalRemove(SerialNum serialNum, DocumentIdT lid,
+                        bool immediateCommit, OnWriteDoneType onWriteDone);
+
+public:
+    AttributeWriter(const proton::IAttributeManager::SP &mgr);
+
+    /**
+     * Implements IAttributeWriter.
+     */
+    std::vector<search::AttributeVector *>
+    getWritableAttributes() const override;
+    search::AttributeVector *
+    getWritableAttribute(const vespalib::string &name) const override;
+    void put(SerialNum serialNum, const Document &doc, DocumentIdT lid,
+             bool immediateCommit, OnWriteDoneType onWriteDone) override;
+    void remove(SerialNum serialNum, DocumentIdT lid,
+                bool immediateCommit, OnWriteDoneType onWriteDone) override;
+    void remove(const LidVector &lidVector, SerialNum serialNum,
+                bool immediateCommit, OnWriteDoneType onWriteDone) override;
+    void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid,
+                bool immediateCommit, OnWriteDoneType onWriteDone) override;
+    void heartBeat(SerialNum serialNum) override;
+    void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override;
+    const proton::IAttributeManager::SP &getAttributeManager() const override {
+        return _mgr;
+    }
+    void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) override;
+
+    virtual void onReplayDone(uint32_t docIdLimit) override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
new file mode 100644
index 00000000000..b5fa7db0983
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
@@ -0,0 +1,134 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.attributedisklayout");
+#include "attributedisklayout.h"
+#include <vespa/searchcommon/common/schemaconfigurer.h>
+
+using search::IndexMetaInfo;
+using search::index::SchemaBuilder;
+using search::index::Schema;
+using search::AttributeVector;
+
+namespace proton
+{
+
+
+bool
+AttributeDiskLayout::removeOldSnapshots(IndexMetaInfo &snapInfo,
+                                        vespalib::Lock &snapInfoLock)
+{
+    IndexMetaInfo::Snapshot best = snapInfo.getBestSnapshot();
+    if (!best.valid) {
+        return true;
+    }
+    std::vector<IndexMetaInfo::Snapshot> toRemove;
+    const IndexMetaInfo::SnapshotList & list = snapInfo.snapshots();
+    for (const auto &snap : list) {
+        if (!(snap == best)) {
+            toRemove.push_back(snap);
+        }
+    }
+    LOG(debug,
+        "About to remove %zu old snapshots. "
+        "Will keep best snapshot with sync token %" PRIu64,
+        toRemove.size(),
+        best.syncToken);
+    for (const auto &snap : toRemove) {
+        if (snap.valid) {
+            {
+                vespalib::LockGuard guard(snapInfoLock);
+                snapInfo.invalidateSnapshot(snap.syncToken);
+            }
+            if (!snapInfo.save()) {
+                LOG(warning,
+                    "Could not save meta info file in directory '%s' after "
+                    "invalidating snapshot with sync token %" PRIu64,
+                    snapInfo.getPath().c_str(),
+                    snap.syncToken);
+                return false;
+            }
+        }
+        vespalib::string rmDir =
+            getSnapshotRemoveDir(snapInfo.getPath(), snap.dirName);
+        FastOS_StatInfo statInfo;
+        if (!FastOS_File::Stat(rmDir.c_str(), &statInfo) &&
+            statInfo._error == FastOS_StatInfo::FileNotFound)
+        {
+            // Directory already removed
+        } else {
+            FastOS_FileInterface:: EmptyAndRemoveDirectory(rmDir.c_str());
+#if 0
+            LOG(warning,
+                "Could not remove snapshot directory '%s'",
+                rmDir.c_str());
+            return false;
+#endif
+        }
+        {
+            vespalib::LockGuard guard(snapInfoLock);
+            snapInfo.removeSnapshot(snap.syncToken);
+        }
+        if (!snapInfo.save()) {
+            LOG(warning,
+                "Could not save meta info file in directory '%s' after "
+                "removing snapshot with sync token %" PRIu64,
+                snapInfo.getPath().c_str(), snap.syncToken);
+            return false;
+        }
+        LOG(debug, "Removed snapshot directory '%s'", rmDir.c_str());
+    }
+    return true;
+}
+
+bool
+AttributeDiskLayout::removeAttribute(const vespalib::string &baseDir,
+                                     const vespalib::string &attrName)
+{
+    const vespalib::string currDir = getAttributeBaseDir(baseDir, attrName);
+    const vespalib::string rmDir =
+        getAttributeBaseDir(baseDir,
+                            vespalib::make_string("remove.%s",
+                                    attrName.c_str()));
+
+    FastOS_StatInfo statInfo;
+    if (FastOS_File::Stat(rmDir.c_str(), &statInfo) && statInfo._isDirectory) {
+        FastOS_FileInterface::EmptyAndRemoveDirectory(rmDir.c_str());
+#if 0
+        if (!FastOS_FileInterface::EmptyAndRemoveDirectory(rmDir.c_str())) {
+            LOG(warning,
+                "Could not remove attribute directory '%s'",
+                rmDir.c_str());
+            return false;
+        }
+#endif
+    }
+    if (!FastOS_File::Stat(currDir.c_str(), &statInfo) &&
+        statInfo._error == FastOS_StatInfo::FileNotFound)
+    {
+        // Directory already removed
+        return true;
+    }
+    if (!FastOS_FileInterface::MoveFile(currDir.c_str(), rmDir.c_str())) {
+        LOG(warning,
+            "Could not move attribute directory '%s' to '%s'",
+            currDir.c_str(),
+            rmDir.c_str());
+        return false;
+    }
+    FastOS_FileInterface::EmptyAndRemoveDirectory(rmDir.c_str());
+#if 0
+    if (!FastOS_FileInterface::EmptyAndRemoveDirectory(rmDir.c_str())) {
+        LOG(warning,
+            "Could not remove attribute directory '%s'",
+            rmDir.c_str());
+        return false;
+    }
+#endif
+    return true;
+}
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h
new file mode 100644
index 00000000000..1fbcf7ab8b3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/common/indexmetainfo.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/searchcommon/common/schema.h>
+
+namespace proton {
+
+/**
+ * Class with utility functions for handling the disk directory layout for attribute vectors.
+ */
+class AttributeDiskLayout
+{
+private:
+    static vespalib::string
+    getSnapshotDir(uint64_t syncToken)
+    {
+        return vespalib::make_string("snapshot-%" PRIu64, syncToken);
+    }
+
+    static vespalib::string
+    getSnapshotRemoveDir(const vespalib::string &baseDir,
+                         const vespalib::string &snapDir)
+    {
+        if (baseDir.empty()) {
+            return snapDir;
+        }
+        return vespalib::make_string("%s/%s",
+                                     baseDir.c_str(),
+                                     snapDir.c_str());
+    }
+public:
+    static vespalib::string
+    getAttributeBaseDir(const vespalib::string &baseDir,
+                        const vespalib::string &attrName)
+    {
+        if (baseDir.empty()) {
+            return attrName;
+        }
+        return vespalib::make_string("%s/%s",
+                                     baseDir.c_str(),
+                                     attrName.c_str());
+    }
+
+    static search::AttributeVector::BaseName
+    getAttributeFileName(const vespalib::string &baseDir,
+                         const vespalib::string &attrName,
+                         uint64_t syncToken)
+    {
+        return search::AttributeVector::BaseName(getAttributeBaseDir(baseDir,
+                                                         attrName),
+                getSnapshotDir(syncToken),
+                attrName);
+    }
+
+    static bool
+    removeOldSnapshots(search::IndexMetaInfo &snapInfo,
+                       vespalib::Lock &snapInfoLock);
+
+    static bool
+    removeAttribute(const vespalib::string &baseDir,
+                    const vespalib::string &attrName);
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
new file mode 100644
index 00000000000..35541176a57
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
@@ -0,0 +1,466 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.attributemanager");
+#include "attribute_factory.h"
+#include "attribute_initializer.h"
+#include "attributedisklayout.h"
+#include "attributemanager.h"
+#include "flushableattribute.h"
+#include "sequential_attributes_initializer.h"
+#include <vespa/searchlib/attribute/attributecontext.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/searchlib/attribute/interlock.h>
+#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <memory>
+#include "i_attribute_functor.h"
+
+using search::AttributeContext;
+using search::AttributeEnumGuard;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::IndexMetaInfo;
+using search::TuneFileAttributes;
+using search::attribute::IAttributeContext;
+using search::index::Schema;
+using search::common::FileHeaderContext;
+
+namespace proton {
+
+AttributeVector::SP
+AttributeManager::internalAddAttribute(const vespalib::string &name,
+                                       const Config &cfg,
+                                       uint64_t serialNum,
+                                       const IAttributeFactory &factory)
+{
+    AttributeInitializer initializer(_baseDir, _documentSubDbName, name, cfg, serialNum, factory);
+    AttributeVector::SP attr = initializer.init();
+    if (attr.get() != NULL) {
+        attr->setInterlock(_interlock);
+        addAttribute(attr);
+    }
+    return attr;
+}
+
+void
+AttributeManager::addAttribute(const AttributeWrap &attribute)
+{
+    LOG(debug,
+        "Adding attribute vector '%s'",
+        attribute->getBaseFileName().c_str());
+    _attributes[attribute->getName()] = attribute;
+    assert(attribute->getInterlock() == _interlock);
+    if ( ! attribute.isExtra() ) {
+        // Flushing of extra attributes is handled elsewhere
+        _flushables[attribute->getName()] = FlushableAttribute::SP
+                (new FlushableAttribute(attribute, _baseDir,
+                                        _tuneFileAttributes,
+                                        _fileHeaderContext,
+                                        _attributeFieldWriter));
+        _writableAttributes.push_back(attribute.get());
+    }
+}
+
+AttributeVector::SP
+AttributeManager::findAttribute(const vespalib::string &name) const
+{
+    AttributeMap::const_iterator itr = _attributes.find(name);
+    return (itr != _attributes.end())
+        ? static_cast<const AttributeVector::SP &>(itr->second)
+        : AttributeVector::SP();
+}
+
+FlushableAttribute::SP
+AttributeManager::findFlushable(const vespalib::string &name) const
+{
+    FlushableMap::const_iterator itr = _flushables.find(name);
+    return (itr != _flushables.end()) ? itr->second : FlushableAttribute::SP();
+}
+
+void
+AttributeManager::createBaseDir()
+{
+    vespalib::mkdir(_baseDir, false);
+}
+
+void
+AttributeManager::transferExistingAttributes(const AttributeManager &currMgr,
+                                             const Spec &newSpec,
+                                             Spec::AttributeList &toBeAdded)
+{
+    for (const auto &aspec : newSpec.getAttributes()) {
+        AttributeVector::SP av = currMgr.findAttribute(aspec.getName());
+        if (av.get() != NULL) { // transfer attribute
+            LOG(debug,
+                "Transferring attribute vector '%s' with %u docs and "
+                "serial number %" PRIu64 " from current manager",
+                av->getName().c_str(),
+                av->getNumDocs(),
+                av->getStatus().getLastSyncToken());
+            addAttribute(av);
+        } else {
+            toBeAdded.push_back(aspec);
+        }
+    }
+}
+
+void
+AttributeManager::flushRemovedAttributes(const AttributeManager &currMgr,
+                                         const Spec &newSpec)
+{
+    for (const auto &kv : currMgr._attributes) {
+        if (!newSpec.hasAttribute(kv.first) &&
+            !kv.second.isExtra() &&
+            kv.second->getStatus().getLastSyncToken() <
+            newSpec.getCurrentSerialNum()) {
+            FlushableAttribute::SP flushable =
+                currMgr.findFlushable(kv.first);
+            vespalib::Executor::Task::UP flushTask =
+                flushable->initFlush(newSpec.getCurrentSerialNum());
+            if (flushTask.get() != NULL) {
+                LOG(debug,
+                    "Flushing removed attribute vector '%s' with %u docs "
+                    "and serial number %" PRIu64,
+                    kv.first.c_str(),
+                    kv.second->getNumDocs(),
+                    kv.second->getStatus().getLastSyncToken());
+                flushTask->run();
+            }
+        }
+    }
+}
+
+void
+AttributeManager::addNewAttributes(const Spec &newSpec,
+                                   const Spec::AttributeList &toBeAdded,
+                                   IAttributeInitializerRegistry &initializerRegistry)
+{
+    for (const auto &aspec : toBeAdded) {
+        LOG(debug, "Creating initializer for attribute vector '%s': docIdLimit=%u, serialNumber=%" PRIu64,
+            aspec.getName().c_str(),
+            newSpec.getDocIdLimit(),
+            newSpec.getCurrentSerialNum());
+
+        AttributeInitializer::UP initializer =
+                std::make_unique<AttributeInitializer>(_baseDir, _documentSubDbName, aspec.getName(),
+                        aspec.getConfig(), newSpec.getCurrentSerialNum(), *_factory);
+        initializerRegistry.add(std::move(initializer));
+
+        // TODO: Might want to use hardlinks to make attribute vector
+        // appear to have been flushed at resurrect time, eliminating
+        // flushDone serials going backwards in document db, and allowing
+        // for pruning of transaction log up to the resurrect serial
+        // without having to reflush the resurrected attribute vector.
+
+        // XXX: Need to wash attribute at resurrection time to get rid of
+        // ghost values (lid freed and not reused), foreign values
+        // (lid freed and reused by another document) and stale values
+        // (lid still used by newer versions of the same document).
+    }
+
+}
+
+void
+AttributeManager::transferExtraAttributes(const AttributeManager &currMgr)
+{
+    for (const auto &kv : currMgr._attributes) {
+        if (kv.second.isExtra()) {
+            addAttribute(kv.second);
+        }
+    }
+}
+
+AttributeManager::AttributeManager(const vespalib::string &baseDir,
+                                   const vespalib::string &documentSubDbName,
+                                   const TuneFileAttributes &tuneFileAttributes,
+                                   const FileHeaderContext &fileHeaderContext,
+                                   search::ISequencedTaskExecutor &
+                                   attributeFieldWriter)
+    : boost::noncopyable(),
+      proton::IAttributeManager(),
+      _attributes(),
+      _flushables(),
+      _writableAttributes(),
+      _baseDir(baseDir),
+      _documentSubDbName(documentSubDbName),
+      _tuneFileAttributes(tuneFileAttributes),
+      _fileHeaderContext(fileHeaderContext),
+      _factory(new AttributeFactory()),
+      _interlock(std::make_shared<search::attribute::Interlock>()),
+      _attributeFieldWriter(attributeFieldWriter)
+{
+    createBaseDir();
+}
+
+
+AttributeManager::AttributeManager(const vespalib::string &baseDir,
+                                   const vespalib::string &documentSubDbName,
+                                   const search::TuneFileAttributes &tuneFileAttributes,
+                                   const search::common::FileHeaderContext &fileHeaderContext,
+                                   search::ISequencedTaskExecutor &
+                                   attributeFieldWriter,
+                                   const IAttributeFactory::SP &factory)
+    : boost::noncopyable(),
+      proton::IAttributeManager(),
+      _attributes(),
+      _flushables(),
+      _writableAttributes(),
+      _baseDir(baseDir),
+      _documentSubDbName(documentSubDbName),
+      _tuneFileAttributes(tuneFileAttributes),
+      _fileHeaderContext(fileHeaderContext),
+      _factory(factory),
+      _interlock(std::make_shared<search::attribute::Interlock>()),
+      _attributeFieldWriter(attributeFieldWriter)
+{
+    createBaseDir();
+}
+
+AttributeManager::AttributeManager(const AttributeManager &currMgr,
+                                   const Spec &newSpec,
+                                   IAttributeInitializerRegistry &initializerRegistry)
+    : boost::noncopyable(),
+      proton::IAttributeManager(),
+      _attributes(),
+      _flushables(),
+      _writableAttributes(),
+      _baseDir(currMgr._baseDir),
+      _documentSubDbName(currMgr._documentSubDbName),
+      _tuneFileAttributes(currMgr._tuneFileAttributes),
+      _fileHeaderContext(currMgr._fileHeaderContext),
+      _factory(currMgr._factory),
+      _interlock(currMgr._interlock),
+      _attributeFieldWriter(currMgr._attributeFieldWriter)
+{
+    Spec::AttributeList toBeAdded;
+    transferExistingAttributes(currMgr, newSpec, toBeAdded);
+    flushRemovedAttributes(currMgr, newSpec);
+    addNewAttributes(newSpec, toBeAdded, initializerRegistry);
+    transferExtraAttributes(currMgr);
+}
+
+AttributeVector::SP
+AttributeManager::addAttribute(const vespalib::string &name,
+                               const Config &cfg,
+                               uint64_t serialNum)
+{
+    return internalAddAttribute(name, cfg, serialNum, *_factory);
+}
+
+void
+AttributeManager::addInitializedAttributes(const std::vector<search::AttributeVector::SP> &attributes)
+{
+    for (const auto &attribute : attributes) {
+        attribute->setInterlock(_interlock);
+        addAttribute(attribute);
+    }
+}
+
+void
+AttributeManager::addExtraAttribute(const AttributeVector::SP &attribute)
+{
+    attribute->setInterlock(_interlock);
+    addAttribute(AttributeWrap(attribute, true));
+}
+
+void
+AttributeManager::flushAll(SerialNum currentSerial)
+{
+    for (const auto &kv : _flushables) {
+        vespalib::Executor::Task::UP task;
+        task = kv.second->initFlush(currentSerial);
+        if (task.get() != NULL) {
+            task->run();
+        }
+    }
+}
+
+FlushableAttribute::SP
+AttributeManager::getFlushable(const vespalib::string &name)
+{
+    return findFlushable(name);
+}
+
+size_t
+AttributeManager::getNumDocs() const
+{
+    return _attributes.empty()
+        ? 0
+        : _attributes.begin()->second->getNumDocs();
+}
+
+void
+AttributeManager::padAttribute(AttributeVector &v, uint32_t docIdLimit)
+{
+    uint32_t needCommit = 0;
+    uint32_t docId(v.getNumDocs());
+    while (v.getNumDocs() < docIdLimit) {
+        if (!v.addDoc(docId)) {
+            throw vespalib::IllegalStateException
+                (vespalib::make_string("Failed to pad doc %u/%u to "
+                                       "attribute vector '%s'",
+                                       docId,
+                                       docIdLimit,
+                                       v.getName().c_str()));
+        }
+        v.clearDoc(docId);
+        if (++needCommit >= 1024) {
+            needCommit = 0;
+            v.commit();
+        }
+    }
+    if (needCommit > 1)
+        v.commit();
+    assert(v.getNumDocs() >= docIdLimit);
+}
+
+AttributeGuard::UP
+AttributeManager::getAttribute(const vespalib::string &name) const
+{
+    return AttributeGuard::UP(new AttributeGuard(findAttribute(name)));
+}
+
+AttributeGuard::UP
+AttributeManager::getAttributeStableEnum(const vespalib::string &name) const
+{
+    return AttributeGuard::UP(new AttributeEnumGuard(findAttribute(name)));
+}
+
+void
+AttributeManager::getAttributeList(std::vector<AttributeGuard> &list) const
+{
+    list.reserve(_attributes.size());
+    for (const auto &kv : _attributes) {
+        if (!kv.second.isExtra()) {
+            list.push_back(AttributeGuard(kv.second));
+        }
+    }
+}
+
+IAttributeContext::UP
+AttributeManager::createContext() const
+{
+    return IAttributeContext::UP(new AttributeContext(*this));
+}
+
+proton::IAttributeManager::SP
+AttributeManager::create(const Spec &spec) const
+{
+    SequentialAttributesInitializer initializer(spec.getDocIdLimit());
+    proton::AttributeManager::SP result = std::make_shared<AttributeManager>(*this, spec, initializer);
+    result->addInitializedAttributes(initializer.getInitializedAttributes());
+    return result;
+}
+
+std::vector<IFlushTarget::SP>
+AttributeManager::getFlushTargets() const
+{
+    std::vector<IFlushTarget::SP> list;
+    list.reserve(_flushables.size());
+    for (const auto &kv : _flushables) {
+        list.push_back(kv.second);
+    }
+    return list;
+}
+
+search::SerialNum
+AttributeManager::getFlushedSerialNum(const vespalib::string &name) const
+{
+    FlushableAttribute::SP flushable = findFlushable(name);
+    if (flushable.get() != nullptr) {
+        return flushable->getFlushedSerialNum();
+    }
+    return 0;
+}
+
+search::SerialNum
+AttributeManager::getOldestFlushedSerialNumber() const
+{
+    SerialNum num = -1;
+    for (const auto &kv : _flushables) {
+        num = std::min(num, kv.second->getFlushedSerialNum());
+    }
+    return num;
+}
+
+search::SerialNum
+AttributeManager::getNewestFlushedSerialNumber() const
+{
+    SerialNum num = 0;
+    for (const auto &kv : _flushables) {
+        num = std::max(num, kv.second->getFlushedSerialNum());
+    }
+    return num;
+}
+
+void
+AttributeManager::getAttributeListAll(std::vector<AttributeGuard> &list) const
+{
+    list.reserve(_attributes.size());
+    for (const auto &kv : _attributes) {
+        list.push_back(AttributeGuard(kv.second));
+    }
+}
+
+void
+AttributeManager::wipeHistory(const Schema &historySchema)
+{
+    for (uint32_t i = 0; i < historySchema.getNumAttributeFields(); ++i) {
+        const Schema::AttributeField & field =
+            historySchema.getAttributeField(i);
+        AttributeDiskLayout::removeAttribute(_baseDir, field.getName());
+    }
+}
+
+search::ISequencedTaskExecutor &
+AttributeManager::getAttributeFieldWriter() const
+{
+    return _attributeFieldWriter;
+}
+
+
+AttributeVector *
+AttributeManager::getWritableAttribute(const vespalib::string &name) const
+{
+    AttributeMap::const_iterator itr = _attributes.find(name);
+    if (itr == _attributes.end() || itr->second.isExtra()) {
+        return nullptr;
+    }
+    return itr->second.get();
+}
+
+
+const std::vector<AttributeVector *> &
+AttributeManager::getWritableAttributes() const
+{
+    return _writableAttributes;
+}
+
+
+void
+AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>
+                                        func) const
+{
+    for (const auto &attr : _attributes) {
+        if (attr.second.isExtra()) {
+            continue;
+        }
+        AttributeVector::SP attrsp = attr.second;
+        _attributeFieldWriter.
+            execute(attr.first, [attrsp, func]() { (*func)(*attrsp); });
+    }
+}
+
+ExclusiveAttributeReadAccessor::UP
+AttributeManager::getExclusiveReadAccessor(const vespalib::string &name) const
+{
+    AttributeVector::SP attribute = findAttribute(name);
+    if (attribute) {
+        return std::make_unique<ExclusiveAttributeReadAccessor>(attribute, _attributeFieldWriter);
+    }
+    return ExclusiveAttributeReadAccessor::UP();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
new file mode 100644
index 00000000000..1a31ee3e7f1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
@@ -0,0 +1,185 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attribute_collection_spec.h"
+#include "i_attribute_factory.h"
+#include "i_attribute_manager.h"
+#include "i_attribute_initializer_registry.h"
+#include <set>
+#include <boost/utility.hpp>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchcore/proton/attribute/flushableattribute.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchcommon/common/schema.h>
+
+namespace search
+{
+
+namespace common
+{
+
+class FileHeaderContext;
+
+}
+
+}
+
+namespace proton
+{
+
+
+/**
+ * Specialized attribute manager for proton.
+ */
+class AttributeManager : public boost::noncopyable,
+                         public proton::IAttributeManager
+{
+private:
+    typedef search::attribute::Config Config;
+    typedef search::SerialNum SerialNum;
+    typedef AttributeCollectionSpec Spec;
+
+    class AttributeWrap : public search::AttributeVector::SP
+    {
+    private:
+        bool _isExtra;
+    public:
+        AttributeWrap() : search::AttributeVector::SP(), _isExtra(false) { }
+        AttributeWrap(const search::AttributeVector::SP & a, bool isExtra_ = false) : 
+            search::AttributeVector::SP(a),
+            _isExtra(isExtra_)
+        { }
+        bool isExtra() const { return _isExtra; }
+    };
+
+    typedef vespalib::hash_map<vespalib::string, AttributeWrap> AttributeMap;
+    typedef vespalib::hash_map<vespalib::string, FlushableAttribute::SP> FlushableMap;
+
+    AttributeMap _attributes;
+    FlushableMap _flushables;
+    std::vector<search::AttributeVector *> _writableAttributes;
+    vespalib::string _baseDir;
+    vespalib::string _documentSubDbName;
+    const search::TuneFileAttributes _tuneFileAttributes;
+    const search::common::FileHeaderContext &_fileHeaderContext;
+    IAttributeFactory::SP _factory;
+    std::shared_ptr<search::attribute::Interlock> _interlock;
+    search::ISequencedTaskExecutor &_attributeFieldWriter;
+
+    search::AttributeVector::SP internalAddAttribute(const vespalib::string &name,
+                                                     const Config &cfg,
+                                                     uint64_t serialNum,
+                                                     const IAttributeFactory &factory);
+
+    void addAttribute(const AttributeWrap &attribute);
+
+    search::AttributeVector::SP findAttribute(const vespalib::string &name) const;
+
+    FlushableAttribute::SP findFlushable(const vespalib::string &name) const;
+
+    void createBaseDir();
+
+
+    void transferExistingAttributes(const AttributeManager &currMgr,
+                                    const Spec &newSpec,
+                                    Spec::AttributeList &toBeAdded);
+
+    void flushRemovedAttributes(const AttributeManager &currMgr,
+                                const Spec &newSpec);
+
+    void addNewAttributes(const Spec &newSpec,
+                          const Spec::AttributeList &toBeAdded,
+                          IAttributeInitializerRegistry &initializerRegistry);
+
+    void transferExtraAttributes(const AttributeManager &currMgr);
+
+public:
+    typedef std::shared_ptr<AttributeManager> SP;
+
+    AttributeManager(const vespalib::string &baseDir,
+                     const vespalib::string &documentSubDbName,
+                     const search::TuneFileAttributes &tuneFileAttributes,
+                     const search::common::FileHeaderContext &
+                     fileHeaderContext,
+                     search::ISequencedTaskExecutor &attributeFieldWriter);
+
+    AttributeManager(const vespalib::string &baseDir,
+                     const vespalib::string &documentSubDbName,
+                     const search::TuneFileAttributes &tuneFileAttributes,
+                     const search::common::FileHeaderContext &
+                     fileHeaderContext,
+                     search::ISequencedTaskExecutor &attributeFieldWriter,
+                     const IAttributeFactory::SP &factory);
+
+    AttributeManager(const AttributeManager &currMgr,
+                     const Spec &newSpec,
+                     IAttributeInitializerRegistry &initializerRegistry);
+
+    search::AttributeVector::SP addAttribute(const vespalib::string &name,
+                                             const Config &cfg,
+                                             uint64_t serialNum);
+
+    void addInitializedAttributes(const std::vector<search::AttributeVector::SP> &attributes);
+
+    void addExtraAttribute(const search::AttributeVector::SP &attribute);
+
+    void flushAll(SerialNum currentSerial);
+
+    FlushableAttribute::SP getFlushable(const vespalib::string &name);
+
+    size_t getNumDocs() const;
+
+    static void padAttribute(search::AttributeVector &v, uint32_t docIdLimit);
+
+    // Implements search::IAttributeManager
+    virtual search::AttributeGuard::UP getAttribute(const vespalib::string &name) const;
+
+    virtual search::AttributeGuard::UP getAttributeStableEnum(const vespalib::string &name) const;
+
+    /**
+     * Fills all regular registered attributes (not extra attributes)
+     * into the given list.
+     */
+    virtual void getAttributeList(std::vector<search::AttributeGuard> &list) const;
+
+    virtual search::attribute::IAttributeContext::UP createContext() const;
+
+
+    // Implements proton::IAttributeManager
+
+    virtual proton::IAttributeManager::SP create(const Spec &spec) const;
+
+    virtual std::vector<IFlushTarget::SP> getFlushTargets() const;
+
+    virtual search::SerialNum getFlushedSerialNum(const vespalib::string &name) const;
+
+    virtual SerialNum getOldestFlushedSerialNumber() const;
+
+    virtual search::SerialNum
+    getNewestFlushedSerialNumber() const;
+
+    virtual void getAttributeListAll(std::vector<search::AttributeGuard> &list) const;
+
+    virtual void wipeHistory(const search::index::Schema &historySchema);
+
+    virtual const IAttributeFactory::SP &getFactory() const { return _factory; }
+
+    virtual search::ISequencedTaskExecutor &
+    getAttributeFieldWriter() const override;
+
+    virtual search::AttributeVector *
+    getWritableAttribute(const vespalib::string &name) const override;
+
+    virtual const std::vector<search::AttributeVector *> &
+    getWritableAttributes() const override;
+
+    virtual void
+    asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override;
+
+    virtual ExclusiveAttributeReadAccessor::UP
+    getExclusiveReadAccessor(const vespalib::string &name) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.cpp
new file mode 100644
index 00000000000..234c46147dd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.cpp
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "attributes_initializer_base.h"
+#include "attributemanager.h"
+
+namespace proton {
+
+void
+AttributesInitializerBase::considerPadAttribute(search::AttributeVector &attribute,
+                                                search::SerialNum currentSerialNum,
+                                                uint32_t newDocIdLimit)
+{
+    /*
+     * Sizing requirements for other components to work with the
+     * new attributes vectors:
+     *
+     * Document meta store doesn't need to be resized here ever.
+     * It is always present and is the authorative source for
+     * allocation of new lids after replay of transaction log has
+     * completed. The transaction log should never be pruned
+     * beyond the last saved version of the document meta store,
+     * and the document meta store will grow as needed during
+     * replay unless the transaction log is corrupted.
+     *
+     * If a newly loaded attribute vector is shorter than the
+     * document meta store then it needs to be padded upwards to
+     * the same size to ensure that further operations will work.
+     * This is not needed if the system has never performed any
+     * reconfiguration introducing/removing attribute vectors,
+     * i.e.  if the newest saved config is still at serial number
+     * 1, since a replay of a non-corrupted transaction log should
+     * grow the attribute as needed.
+     */
+    if (attribute.getStatus().getLastSyncToken() < currentSerialNum) {
+        AttributeManager::padAttribute(attribute, newDocIdLimit);
+        attribute.commit();
+        assert(newDocIdLimit <= attribute.getNumDocs());
+    }
+}
+
+AttributesInitializerBase::AttributesInitializerBase()
+    : IAttributeInitializerRegistry(),
+      _initializedAttributes()
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.h b/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.h
new file mode 100644
index 00000000000..0f2d879718a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.h
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_attribute_initializer_registry.h"
+#include <vespa/searchlib/attribute/attributevector.h>
+
+namespace proton {
+
+/**
+ * Base class for initialization and loading of a set of attribute vectors.
+ */
+class AttributesInitializerBase : public IAttributeInitializerRegistry
+{
+public:
+    typedef std::vector<search::AttributeVector::SP> AttributesVector;
+
+protected:
+    AttributesVector _initializedAttributes;
+
+public:
+    static void considerPadAttribute(search::AttributeVector &attribute,
+                                     search::SerialNum currentSerialNum,
+                                     uint32_t newDocIdLimit);
+
+    AttributesInitializerBase();
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp
new file mode 100644
index 00000000000..5d7a1526bf0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "attributesconfigscout.h"
+#include <vespa/searchlib/attribute/configconverter.h>
+
+using search::attribute::ConfigConverter;
+
+namespace proton
+{
+
+AttributesConfigScout::AttributesConfigScout(const AttributesConfig &live)
+    : _live(live),
+      _map()
+{
+    uint32_t i = 0;
+    for (const auto &attr : live.attribute) {
+        _map[attr.name] = i;
+        ++i;
+    }
+}
+
+
+void
+AttributesConfigScout::adjust(AttributesConfig::Attribute &attr,
+                              const AttributesConfig::Attribute &liveAttr)
+{
+    attr.enablebitvectors = liveAttr.enablebitvectors;
+    attr.enableonlybitvector = liveAttr.enableonlybitvector;
+    attr.fastsearch = liveAttr.fastsearch;
+    attr.huge = liveAttr.huge;
+    // Note: Predicate attributes only handle changes for the dense-posting-list-threshold config.
+    attr.densepostinglistthreshold = liveAttr.densepostinglistthreshold;
+}
+
+
+void
+AttributesConfigScout::adjust(AttributesConfig::Attribute &attr)
+{
+    search::attribute::Config cfg = ConfigConverter::convert(attr);
+    const auto it = _map.find(attr.name);
+    if (it != _map.end()) {
+        const auto &liveAttr = _live.attribute[it->second];
+        search::attribute::Config liveCfg =
+            ConfigConverter::convert(liveAttr);
+        if (cfg.basicType() == liveCfg.basicType() &&
+            cfg.collectionType() == liveCfg.collectionType()) {
+            adjust(attr, liveAttr);
+        }
+    }
+}
+
+
+std::shared_ptr<AttributesConfigScout::AttributesConfig>
+AttributesConfigScout::adjust(const AttributesConfig &config)
+{
+    std::shared_ptr<AttributesConfigBuilder> result =
+        std::make_shared<AttributesConfigBuilder>(config);
+    for (auto &attr : result->attribute) {
+        adjust(attr);
+    }
+    return result;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.h b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.h
new file mode 100644
index 00000000000..845a1ed2830
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/config-attributes.h>
+
+namespace proton
+{
+
+/**
+ * Class to create adjusted attributes config that minimizes the number of
+ * proton restarts needed due to config changes.  Grab the portions from
+ * live (supposedly future) config that is safe to apply early during
+ * initialization and replay.
+ */
+class AttributesConfigScout
+{
+public:
+    using AttributesConfig = vespa::config::search::AttributesConfig;
+    using AttributesConfigBuilder =
+        vespa::config::search::AttributesConfigBuilder;
+
+private:
+    const AttributesConfig &_live;
+    std::map<vespalib::string, uint32_t> _map;
+    
+    static void
+    adjust(AttributesConfig::Attribute &attr,
+           const AttributesConfig::Attribute &liveAttr);
+
+    void
+    adjust(AttributesConfig::Attribute &attr);
+
+public:
+    AttributesConfigScout(const AttributesConfig &live);
+
+    std::shared_ptr<AttributesConfig>
+    adjust(const AttributesConfig &config);
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.cpp
new file mode 100644
index 00000000000..702a77bbebe
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.cpp
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.document_field_populator");
+#include "document_field_populator.h"
+#include "document_field_retriever.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+
+using document::Document;
+using search::index::Schema;
+using search::AttributeGuard;
+
+namespace proton {
+
+namespace {
+
+vespalib::string
+getFieldName(const vespalib::string &subDbName,
+             const vespalib::string &fieldName)
+{
+    return subDbName + ".documentfield." + fieldName;
+}
+
+}
+
+DocumentFieldPopulator::DocumentFieldPopulator(const Schema::AttributeField &field,
+                                               search::AttributeVector::SP attr,
+                                               const vespalib::string &subDbName)
+    : _field(field),
+      _attr(attr),
+      _subDbName(subDbName),
+      _documentsPopulated(0)
+{
+    if (LOG_WOULD_LOG(event)) {
+        EventLogger::populateDocumentFieldStart(getFieldName(subDbName, field.getName()));
+    }
+}
+
+DocumentFieldPopulator::~DocumentFieldPopulator()
+{
+    if (LOG_WOULD_LOG(event)) {
+        EventLogger::populateDocumentFieldComplete(getFieldName(_subDbName, _field.getName()),
+                _documentsPopulated);
+    }
+}
+
+void
+DocumentFieldPopulator::handleExisting(uint32_t lid, Document &doc)
+{
+    DocumentFieldRetriever::populate(lid, doc, _field, *_attr, false);
+    ++_documentsPopulated;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.h b/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.h
new file mode 100644
index 00000000000..a16401b775e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_rewriter.h>
+#include <vespa/searchlib/attribute/attributeguard.h>
+
+namespace proton {
+
+/**
+ * Class used to populate a document field based on the content from an attribute vector.
+ */
+class DocumentFieldPopulator : public IReprocessingRewriter
+{
+private:
+    search::index::Schema::AttributeField _field;
+    search::AttributeVector::SP _attr;
+    vespalib::string _subDbName;
+    int64_t _documentsPopulated;
+
+public:
+    DocumentFieldPopulator(const search::index::Schema::AttributeField &field,
+                           search::AttributeVector::SP attr,
+                           const vespalib::string &subDbName);
+
+    ~DocumentFieldPopulator();
+
+    const search::AttributeVector &getAttribute() const {
+        return *_attr;
+    }
+
+    // Implements IReprocessingRewriter
+    virtual void handleExisting(uint32_t lid, document::Document &doc);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp
new file mode 100644
index 00000000000..07c7381ca4a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp
@@ -0,0 +1,146 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.document_field_retriever");
+#include "document_field_retriever.h"
+
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/searchlib/attribute/tensorattribute.h>
+#include <vespa/vespalib/tensor/tensor.h>
+
+using search::DocumentIdT;
+using document::ArrayFieldValue;
+using document::Document;
+using document::Field;
+using document::FieldValue;
+using document::TensorFieldValue;
+using document::WeightedSetFieldValue;
+using search::index::Schema;
+using search::attribute::AttributeContent;
+using search::attribute::IAttributeVector;
+using search::attribute::WeightedType;
+using search::attribute::TensorAttribute;
+using vespalib::IllegalStateException;
+
+namespace proton {
+
+namespace {
+
+template <typename T>
+void
+setValue(DocumentIdT lid,
+         Document &doc,
+         const Schema::AttributeField &field,
+         const IAttributeVector &attr)
+{
+    switch (field.getCollectionType()) {
+    case Schema::SINGLE:
+    {
+        if ( ! attr.isUndefined(lid) ) {
+            AttributeContent<T> content;
+            content.fill(attr, lid);
+            doc.set(field.getName(), content[0]);
+        } else {
+            doc.remove(field.getName());
+        }
+        break;
+    }
+    case Schema::ARRAY:
+    {
+        AttributeContent<T> content;
+        content.fill(attr, lid);
+        Field f = doc.getField(field.getName());
+        if (!doc.getValue(f) && content.size() == 0) {
+            break;
+        }
+        FieldValue::UP fv = f.getDataType().createFieldValue();
+        if (fv.get() && fv->getClass().id() != ArrayFieldValue::classId) {
+            throw IllegalStateException("Field " + field.getName() + " does not contain an array.", VESPA_STRLOC);
+        }
+        ArrayFieldValue &array = static_cast<ArrayFieldValue &>(*fv.get());
+        array.resize(content.size());
+        for (uint32_t j(0); j < content.size(); ++j) {
+            array[j] = content[j];
+        }
+        doc.setValue(f, *fv);
+        break;
+    }
+    case Schema::WEIGHTEDSET:
+    {
+        AttributeContent<WeightedType<T> > content;
+        content.fill(attr, lid);
+        Field f = doc.getField(field.getName());
+        if (!doc.getValue(f) && content.size() == 0) {
+            break;
+        }
+        FieldValue::UP fv = f.getDataType().createFieldValue();
+        if (fv.get() && fv->getClass().id() != WeightedSetFieldValue::classId) {
+            throw IllegalStateException("Field " + field.getName() + " does not contain a wset.", VESPA_STRLOC);
+        }
+        WeightedSetFieldValue & wset(static_cast<WeightedSetFieldValue &>(*fv.get()));
+        wset.resize(content.size());
+        auto it(wset.begin());
+        for (uint32_t j(0); j < content.size(); ++j, ++it) {
+            *it->first = content[j].getValue();
+            *it->second = content[j].getWeight();
+        }
+        doc.setValue(f, *fv);
+        break;
+    }
+    default:
+        LOG(warning, "Unknown attribute collection type in Schema.");
+        break;
+    }
+}
+
+}
+
+void
+DocumentFieldRetriever::populate(DocumentIdT lid,
+                                 Document &doc,
+                                 const Schema::AttributeField &field,
+                                 const IAttributeVector &attr,
+                                 bool isIndexField)
+{
+    switch (field.getDataType()) {
+    case Schema::UINT1:
+    case Schema::UINT2:
+    case Schema::UINT4:
+    case Schema::INT8:
+    case Schema::INT16:
+    case Schema::INT32:
+    case Schema::INT64:
+        setValue<IAttributeVector::largeint_t>(
+                lid, doc, field, attr);
+        break;
+    case Schema::FLOAT:
+    case Schema::DOUBLE:
+        setValue<double>(lid, doc, field, attr);
+        break;
+    case Schema::STRING:
+        // If it is a stringfield we also need to check if
+        // it is an index field. In that case we shall
+        // keep the original in order to preserve annotations.
+        if (isIndexField) {
+            break;
+        }
+    case Schema::RAW:
+        setValue<const char *>(lid, doc, field, attr);
+        break;
+    case Schema::BOOLEANTREE:
+        // Predicate attribute doesn't store documents, it only indexes them.
+        break;
+    case Schema::TENSOR:
+        // Tensor attribute is not authorative.  Partial updates must update
+        // document store.
+        break;
+    default:
+        LOG(warning, "Unknown attribute data type in Schema.");
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.h b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.h
new file mode 100644
index 00000000000..d91acb7c00b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/searchcommon/attribute/iattributevector.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchlib/query/base.h>
+
+namespace proton {
+
+/**
+ * Class used to retrieve a document field and populate it with the content from an attribute vector.
+ */
+struct DocumentFieldRetriever
+{
+    static void populate(search::DocumentIdT lid,
+                         document::Document &doc,
+                         const search::index::Schema::AttributeField &field,
+                         const search::attribute::IAttributeVector &attr,
+                         bool isIndexField);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp
new file mode 100644
index 00000000000..3420d3a8dd0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.exclusive_attribute_read_accessor");
+
+#include "exclusive_attribute_read_accessor.h"
+#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/executor.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton {
+
+using search::AttributeVector;
+using search::ISequencedTaskExecutor;
+using vespalib::Executor;
+using vespalib::Gate;
+
+using GateSP = std::shared_ptr<Gate>;
+
+ExclusiveAttributeReadAccessor::Guard::Guard(const AttributeVector &attribute,
+                                             const GateSP &exitGate)
+    : _attribute(attribute),
+      _exitGate(exitGate)
+{
+}
+
+ExclusiveAttributeReadAccessor::Guard::~Guard()
+{
+    _exitGate->countDown();
+}
+
+ExclusiveAttributeReadAccessor::
+ExclusiveAttributeReadAccessor(const AttributeVector::SP &attribute,
+                               ISequencedTaskExecutor &attributeFieldWriter)
+    : _attribute(attribute),
+      _attributeFieldWriter(attributeFieldWriter)
+{
+}
+
+namespace {
+
+void
+attributeWriteBlockingTask(GateSP entranceGate,
+                           GateSP exitGate)
+{
+    entranceGate->countDown();
+    exitGate->await();
+}
+
+}
+
+ExclusiveAttributeReadAccessor::Guard::UP
+ExclusiveAttributeReadAccessor::takeGuard()
+{
+    GateSP entranceGate = std::make_shared<Gate>();
+    GateSP exitGate = std::make_shared<Gate>();
+    _attributeFieldWriter.execute(_attribute->getName(),
+            [entranceGate, exitGate]()
+            { attributeWriteBlockingTask(entranceGate, exitGate); });
+    entranceGate->await();
+    return std::make_unique<Guard>(*_attribute, exitGate);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h
new file mode 100644
index 00000000000..e3a8920f91d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/attribute/attributevector.h>
+
+namespace search { class ISequencedTaskExecutor; }
+namespace vespalib { class Gate; }
+
+namespace proton {
+
+/**
+ * Class that provides exclusive read access to an attribute vector
+ * while the write thread for that attribute is blocked.
+ *
+ * The attribute write thread is blocked while a guard is held.
+ */
+class ExclusiveAttributeReadAccessor
+{
+public:
+    class Guard
+    {
+    private:
+        const search::AttributeVector &_attribute;
+        std::shared_ptr<vespalib::Gate> _exitGate;
+
+    public:
+        using UP = std::unique_ptr<Guard>;
+        Guard(const search::AttributeVector &attribute,
+              const std::shared_ptr<vespalib::Gate> &exitGate);
+        ~Guard();
+        const search::AttributeVector &get() const { return _attribute; }
+    };
+
+private:
+    search::AttributeVector::SP _attribute;
+    search::ISequencedTaskExecutor &_attributeFieldWriter;
+
+public:
+    using UP = std::unique_ptr<ExclusiveAttributeReadAccessor>;
+
+    ExclusiveAttributeReadAccessor(const search::AttributeVector::SP &attribute,
+                                   search::ISequencedTaskExecutor &attributeFieldWriter);
+    Guard::UP takeGuard();
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
new file mode 100644
index 00000000000..3f7d4059447
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
@@ -0,0 +1,117 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.filter_attribute_manager");
+
+#include "filter_attribute_manager.h"
+#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include "i_attribute_functor.h"
+
+using search::AttributeGuard;
+
+namespace proton {
+
+bool
+FilterAttributeManager::acceptAttribute(const vespalib::string &name) const
+{
+    return _acceptedAttributes.count(name) > 0;
+}
+
+FilterAttributeManager::FilterAttributeManager(const AttributeSet &acceptedAttributes,
+                                               const IAttributeManager::SP &mgr)
+    : _acceptedAttributes(acceptedAttributes),
+      _mgr(mgr)
+{
+    // Assume that list of attributes in mgr doesn't change
+    for (const auto attr : _mgr->getWritableAttributes()) {
+        if (acceptAttribute(attr->getName())) {
+            _acceptedWritableAttributes.push_back(attr);
+        }
+    }
+}
+
+AttributeGuard::UP
+FilterAttributeManager::getAttribute(const vespalib::string &name) const
+{
+    if (acceptAttribute(name)) {
+        return _mgr->getAttribute(name);
+    }
+    return AttributeGuard::UP();
+}
+
+void
+FilterAttributeManager::getAttributeList(std::vector<AttributeGuard> &list) const
+{
+    std::vector<AttributeGuard> completeList;
+    _mgr->getAttributeList(completeList);
+    for (const auto &attr : completeList) {
+        if (acceptAttribute(attr->getName())) {
+            list.push_back(attr);
+        }
+    }
+}
+
+search::SerialNum
+FilterAttributeManager::getFlushedSerialNum(const vespalib::string &name) const
+{
+    if (acceptAttribute(name)) {
+        return _mgr->getFlushedSerialNum(name);
+    }
+    return 0;
+}
+
+
+search::ISequencedTaskExecutor &
+FilterAttributeManager::getAttributeFieldWriter() const
+{
+    return _mgr->getAttributeFieldWriter();
+}
+
+
+search::AttributeVector *
+FilterAttributeManager::getWritableAttribute(const vespalib::string &name) const
+{
+    if (acceptAttribute(name)) {
+        return _mgr->getWritableAttribute(name);
+    } else {
+        return nullptr;
+    }
+}
+
+const std::vector<search::AttributeVector *> &
+FilterAttributeManager::getWritableAttributes() const
+{
+    return _acceptedWritableAttributes;
+}
+
+void
+FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>
+                                        func) const
+{
+    // Run by document db master thread
+    std::vector<AttributeGuard> completeList;
+    _mgr->getAttributeList(completeList);
+    search::ISequencedTaskExecutor &attributeFieldWriter =
+        getAttributeFieldWriter();
+    for (auto &guard : completeList) {
+        search::AttributeVector::SP attrsp = guard.getSP();
+        // Name must be extracted in document db master thread or attribute
+        // writer thread
+        vespalib::string attributeName = attrsp->getName();
+        attributeFieldWriter.
+            execute(attributeName, [attrsp, func]() { (*func)(*attrsp); });
+    }
+}
+
+ExclusiveAttributeReadAccessor::UP
+FilterAttributeManager::getExclusiveReadAccessor(const vespalib::string &name) const
+{
+    if (acceptAttribute(name)) {
+        return _mgr->getExclusiveReadAccessor(name);
+    } else {
+        return ExclusiveAttributeReadAccessor::UP();
+    }
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
new file mode 100644
index 00000000000..0add0bb9714
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_attribute_manager.h"
+#include <vespa/vespalib/util/exceptions.h>
+#include <set>
+
+namespace proton {
+
+/**
+ * An attribute manager that wraps another attribute manager and only gives access to a
+ * subset of the attribute vectors in the wrapped manager.
+ *
+ * This manager only implements the subset of functions needed when used by
+ * and attribute adapter in the context of an attribute populator.
+ */
+class FilterAttributeManager : public IAttributeManager
+{
+public:
+    typedef std::set<vespalib::string> AttributeSet;
+
+private:
+    AttributeSet          _acceptedAttributes;
+    IAttributeManager::SP _mgr;
+    std::vector<search::AttributeVector *> _acceptedWritableAttributes;
+
+    bool acceptAttribute(const vespalib::string &name) const;
+
+public:
+    FilterAttributeManager(const AttributeSet &acceptedAttributes,
+                           const IAttributeManager::SP &mgr);
+
+    // Implements search::IAttributeManager
+    virtual search::AttributeGuard::UP getAttribute(const vespalib::string &name) const;
+    virtual search::AttributeGuard::UP getAttributeStableEnum(const vespalib::string &) const {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+    virtual void getAttributeList(std::vector<search::AttributeGuard> &list) const;
+    virtual search::attribute::IAttributeContext::UP createContext() const {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+
+    // Implements proton::IAttributeManager
+    virtual IAttributeManager::SP create(const AttributeCollectionSpec &) const {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+    virtual std::vector<searchcorespi::IFlushTarget::SP> getFlushTargets() const {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+    virtual search::SerialNum getFlushedSerialNum(const vespalib::string &name) const;
+    virtual search::SerialNum getOldestFlushedSerialNumber() const {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+    virtual search::SerialNum getNewestFlushedSerialNumber() const {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+    virtual void getAttributeListAll(std::vector<search::AttributeGuard> &) const {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+    virtual void wipeHistory(const search::index::Schema &) {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+    virtual const IAttributeFactory::SP &getFactory() const {
+        throw vespalib::IllegalArgumentException("Not implemented");
+    }
+
+    virtual search::ISequencedTaskExecutor &
+    getAttributeFieldWriter() const override;
+
+    virtual search::AttributeVector *
+    getWritableAttribute(const vespalib::string &name) const override;
+
+    virtual const std::vector<search::AttributeVector *> &
+    getWritableAttributes() const override;
+
+    virtual void
+    asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override;
+
+    virtual ExclusiveAttributeReadAccessor::UP
+    getExclusiveReadAccessor(const vespalib::string &name) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
new file mode 100644
index 00000000000..f834ce64c67
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
@@ -0,0 +1,292 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.attribute.flushableattribute");
+
+#include "attributedisklayout.h"
+#include "flushableattribute.h"
+#include <vespa/searchlib/attribute/attributefilesavetarget.h>
+#include <vespa/searchlib/attribute/attributesaver.h>
+#include <vespa/searchlib/util/dirtraverse.h>
+#include <vespa/searchlib/util/filekit.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <fstream>
+#include <vespa/searchlib/common/serialnumfileheadercontext.h>
+#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <future>
+
+using namespace search;
+using namespace vespalib;
+using search::common::FileHeaderContext;
+using search::common::SerialNumFileHeaderContext;
+using vespalib::makeTask;
+using vespalib::makeClosure;
+
+namespace proton {
+
+FlushableAttribute::Flusher::Flusher(FlushableAttribute & fattr, SerialNum syncToken)
+    : _fattr(fattr),
+      _saveTarget(),
+      _saver(),
+      _syncToken(syncToken),
+      _flushFile("")
+{
+    fattr._attr->commit(syncToken, syncToken);
+    AttributeVector &attr = *_fattr._attr;
+    // Called by attribute field writer executor
+    if (attr.canShrinkLidSpace()) {
+        attr.shrinkLidSpace();
+    }
+    _flushFile = AttributeDiskLayout::getAttributeFileName(_fattr._baseDir,
+                                                           attr.getName(),
+                                                           _syncToken);
+    attr.setBaseFileName(_flushFile);
+    _saver = attr.initSave();
+    if (!_saver) {
+        // New style background save not available, use old style save.
+        attr.save(_saveTarget);
+    }
+}
+
+FlushableAttribute::Flusher::~Flusher()
+{
+    // empty
+}
+
+bool
+FlushableAttribute::Flusher::saveSnapInfo()
+{
+    if (!_fattr._snapInfo.save()) {
+        LOG(warning,
+            "Could not save meta-info file for attribute vector '%s' to disk",
+            _fattr._attr->getBaseFileName().c_str());
+        return false;
+    }
+    return true;
+}
+
+bool
+FlushableAttribute::Flusher::saveAttribute()
+{
+    vespalib::mkdir(_flushFile.getDirName(), false);
+    SerialNumFileHeaderContext fileHeaderContext(_fattr._fileHeaderContext,
+                                                 _syncToken);
+    if (_saver) {
+        search::AttributeFileSaveTarget saveTarget(_fattr._tuneFileAttributes,
+                                                   fileHeaderContext);
+        bool saveSuccess = _saver->save(saveTarget);
+        _saver.reset();
+        return saveSuccess;
+    } else {
+        return _saveTarget.writeToFile(_fattr._tuneFileAttributes,
+                                       fileHeaderContext);
+    }
+}
+
+bool
+FlushableAttribute::Flusher::flush()
+{
+    IndexMetaInfo::Snapshot newSnap(false, _syncToken,
+                                    _flushFile.getSnapshotName());
+    {
+        vespalib::LockGuard guard(_fattr._snapInfoLock);
+        _fattr._snapInfo.addSnapshot(newSnap);
+    }
+    if (!saveSnapInfo()) {
+        return false;
+    }
+    if (!saveAttribute()) {
+        LOG(warning, "Could not write attribute vector '%s' to disk",
+            _flushFile.c_str());
+        return false;
+    }
+    {
+        vespalib::LockGuard guard(_fattr._snapInfoLock);
+        _fattr._snapInfo.validateSnapshot(_syncToken);
+    }
+    if (!saveSnapInfo()) {
+        return false;
+    }
+    _fattr._lastFlushTime =
+        search::FileKit::getModificationTime(_flushFile.getDirName());
+    return true;
+}
+
+void
+FlushableAttribute::Flusher::updateStats()
+{
+    _fattr._lastStats.setPath(_flushFile.getDirName());
+}
+
+bool
+FlushableAttribute::Flusher::cleanUp()
+{
+    if (_fattr._cleanUpAfterFlush) {
+        if (!AttributeDiskLayout::removeOldSnapshots(_fattr._snapInfo,
+                    _fattr._snapInfoLock)) {
+            LOG(warning,
+                "Encountered problems when removing old snapshot directories"
+                "after flushing attribute vector '%s' to disk",
+                _fattr._attr->getBaseFileName().c_str());
+            return false;
+        }
+    }
+    return true;
+}
+
+void
+FlushableAttribute::Flusher::run()
+{
+    vespalib::LockGuard guard(_fattr._flusherLock);
+    if (_syncToken <= _fattr.getFlushedSerialNum()) {
+        // another flusher has created an equal or better snapshot
+        // after this flusher was created
+        return;
+    }
+    if (!flush()) {
+        // TODO (geirst): throw exception ?
+    }
+    updateStats();
+    if (!cleanUp()) {
+        // TODO (geirst): throw exception ?
+    }
+}
+
+FlushableAttribute::FlushableAttribute(const AttributeVector::SP attr,
+                                       const vespalib::string & baseDir,
+                                       const TuneFileAttributes &
+                                       tuneFileAttributes,
+                                       const FileHeaderContext &
+                                       fileHeaderContext,
+                                       search::ISequencedTaskExecutor &
+                                       attributeFieldWriter)
+    : IFlushTarget(vespalib::make_string(
+                           "attribute.%s",
+                           attr->getName().c_str()),
+            Type::SYNC, Component::ATTRIBUTE),
+      _attr(attr),
+      _baseDir(baseDir),
+      _snapInfo(AttributeDiskLayout::getAttributeBaseDir(baseDir,
+                        attr->getName())),
+      _snapInfoLock(),
+      _flusherLock(),
+      _cleanUpAfterFlush(true),
+      _lastStats(),
+      _tuneFileAttributes(tuneFileAttributes),
+      _fileHeaderContext(fileHeaderContext),
+      _lastFlushTime(),
+      _attributeFieldWriter(attributeFieldWriter)
+{
+    if (!_snapInfo.load()) {
+        _snapInfo.save();
+    } else {
+        vespalib::string dirName =
+            AttributeDiskLayout::getAttributeFileName(_baseDir,
+                    _attr->getName(),
+                    getFlushedSerialNum()).getDirName();
+        _lastFlushTime = search::FileKit::getModificationTime(dirName);
+    }
+    _lastStats.setPathElementsToLog(8);
+}
+
+
+FlushableAttribute::~FlushableAttribute()
+{
+}
+
+
+IFlushTarget::SerialNum
+FlushableAttribute::getFlushedSerialNum() const
+{
+    vespalib::LockGuard guard(_snapInfoLock);
+    IndexMetaInfo::Snapshot bestSnap = _snapInfo.getBestSnapshot();
+    return bestSnap.valid ? bestSnap.syncToken : 0;
+}
+
+IFlushTarget::MemoryGain
+FlushableAttribute::getApproxMemoryGain() const
+{
+    int64_t used(_attr->getStatus().getUsed());
+    int64_t canFree = 0;
+    if (_attr->canShrinkLidSpace()) {
+        uint32_t committedDocIdLimit = _attr->getCommittedDocIdLimit();
+        uint32_t numDocs = _attr->getNumDocs();
+        const attribute::Config &cfg = _attr->getConfig();
+        if (committedDocIdLimit < numDocs) {
+            uint32_t elemSize = 4;
+            if (cfg.collectionType().isMultiValue()) {
+                if (cfg.huge()) {
+                    elemSize = 8;
+                }
+            } else if (cfg.fastSearch()) {
+                // keep elemSize at 4
+            } else {
+                elemSize = cfg.basicType().fixedSize();
+            }
+            canFree = static_cast<int64_t>(elemSize) * 
+                      (numDocs - committedDocIdLimit);
+            if (canFree > used)
+                canFree = used;
+        }
+    }
+    return MemoryGain(used,
+                      used - canFree);
+}
+
+IFlushTarget::DiskGain
+FlushableAttribute::getApproxDiskGain() const
+{
+    return DiskGain(0, 0);
+}
+
+IFlushTarget::Time
+FlushableAttribute::getLastFlushTime() const
+{
+    return _lastFlushTime;
+}
+
+IFlushTarget::Task::UP
+FlushableAttribute::internalInitFlush(SerialNum currentSerial)
+{
+    // Called by document db executor
+    (void)currentSerial;
+    _attr->removeAllOldGenerations();
+    SerialNum syncToken = currentSerial;
+    syncToken = std::max(currentSerial,
+                         _attr->getStatus().getLastSyncToken());
+    if (syncToken <= getFlushedSerialNum()) {
+        vespalib::LockGuard guard(_flusherLock);
+        _lastFlushTime = fastos::ClockSystem::now();
+        LOG(debug,
+            "No attribute vector to flush."
+            " Update flush time to current: lastFlushTime(%f)",
+            _lastFlushTime.sec());
+        return Task::UP();
+    }
+    return Task::UP(new Flusher(*this, syncToken));
+}
+
+
+IFlushTarget::Task::UP
+FlushableAttribute::initFlush(SerialNum currentSerial)
+{
+    std::promise<IFlushTarget::Task::UP> promise;
+    std::future<IFlushTarget::Task::UP> future = promise.get_future();
+    _attributeFieldWriter.execute(_attr->getName(),
+                                  [&]() { promise.set_value(
+                                              internalInitFlush(currentSerial));
+                                         });
+    return future.get();
+}
+
+
+uint64_t
+FlushableAttribute::getApproxBytesToWriteToDisk() const
+{
+    return _attr->getEstimatedSaveByteSize();
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h
new file mode 100644
index 00000000000..1ae64e745ca
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <vespa/searchlib/attribute/attributememorysavetarget.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/common/indexmetainfo.h>
+
+
+namespace search
+{
+
+class ISequencedTaskExecutor;
+
+namespace common
+{
+
+class FileHeaderContext;
+
+}
+
+}
+
+namespace proton
+{
+
+using searchcorespi::FlushStats;
+using searchcorespi::IFlushTarget;
+
+/**
+ * Implementation of IFlushTarget interface for attribute vectors.
+ */
+class FlushableAttribute : public IFlushTarget
+{
+private:
+    /**
+     * Task performing the actual flushing to disk.
+     **/
+    class Flusher : public Task {
+    private:
+        FlushableAttribute              & _fattr;
+        search::AttributeMemorySaveTarget      _saveTarget;
+        std::unique_ptr<search::AttributeSaver> _saver;
+        uint64_t                          _syncToken;
+        search::AttributeVector::BaseName _flushFile;
+
+        bool saveAttribute(); // not updating snap info.
+    public:
+        Flusher(FlushableAttribute & fattr, uint64_t syncToken);
+        ~Flusher();
+        uint64_t getSyncToken() const { return _syncToken; }
+        bool saveSnapInfo();
+        bool flush();
+        void updateStats();
+        bool cleanUp();
+        // Implements vespalib::Executor::Task
+        virtual void run();
+
+        virtual SerialNum
+        getFlushSerial(void) const
+        {
+            return _syncToken;
+        } 
+    };
+
+    search::AttributeVector::SP _attr;
+    vespalib::string            _baseDir;
+    search::IndexMetaInfo       _snapInfo;
+    vespalib::Lock		_snapInfoLock;
+    vespalib::Lock              _flusherLock;
+    bool                        _cleanUpAfterFlush;
+    FlushStats                  _lastStats;
+    const search::TuneFileAttributes _tuneFileAttributes;
+    const search::common::FileHeaderContext &_fileHeaderContext;
+    fastos::TimeStamp           _lastFlushTime;
+    search::ISequencedTaskExecutor &_attributeFieldWriter;
+
+    Task::UP internalInitFlush(SerialNum currentSerial);
+
+public:
+    typedef std::shared_ptr<FlushableAttribute> SP;
+
+    /**
+     * Creates a new instance using the given attribute vector and the
+     * given base dir where all attribute vectors are located.
+     *
+     * fileHeaderContext must be kept alive by caller.
+     **/
+    FlushableAttribute(const search::AttributeVector::SP attr,
+                       const vespalib::string & baseDir,
+                       const search::TuneFileAttributes &tuneFileAttributes,
+                       const search::common::FileHeaderContext &
+                       fileHeaderContext,
+                       search::ISequencedTaskExecutor &attributeFieldWriter);
+
+    virtual
+    ~FlushableAttribute();
+
+    void setCleanUpAfterFlush(bool cleanUp) { _cleanUpAfterFlush = cleanUp; }
+
+    // Implements IFlushTarget
+    virtual MemoryGain getApproxMemoryGain() const;
+    virtual DiskGain getApproxDiskGain() const;
+    virtual Time getLastFlushTime() const;
+    virtual SerialNum getFlushedSerialNum() const;
+    virtual Task::UP initFlush(SerialNum currentSerial);
+    virtual FlushStats getLastFlushStats() const { return _lastStats; }
+    virtual uint64_t getApproxBytesToWriteToDisk() const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_factory.h
new file mode 100644
index 00000000000..35f48c021c1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_factory.h
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+/**
+ * Interface for a factory for creating and setting up attribute vectors used by
+ * an attribute manager.
+ */
+struct IAttributeFactory
+{
+    typedef std::shared_ptr<IAttributeFactory> SP;
+    virtual ~IAttributeFactory() {}
+    virtual search::AttributeVector::SP create(const vespalib::string &name,
+                                               const search::attribute::Config &cfg) const = 0;
+    virtual void setupEmpty(const search::AttributeVector::SP &vec,
+                            search::SerialNum serialNum) const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_functor.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_functor.h
new file mode 100644
index 00000000000..0bf295a8bff
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_functor.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search { class AttributeVector; }
+
+namespace proton {
+
+/*
+ * Interface class for access attribute in correct attribute write
+ * thread as async callback from asyncForEachAttribute() call on
+ * attribute manager.
+ */
+class IAttributeFunctor
+{
+public:
+    virtual void operator()(const search::AttributeVector &attributeVector) = 0;
+    virtual ~IAttributeFunctor() { }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_initializer_registry.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_initializer_registry.h
new file mode 100644
index 00000000000..db0fec113f9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_initializer_registry.h
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attribute_initializer.h"
+
+namespace proton {
+
+/**
+ * Interface for registering a set of attribute initializers,
+ * later to be used to initialize and load the set of attributes.
+ */
+struct IAttributeInitializerRegistry
+{
+    virtual ~IAttributeInitializerRegistry() {}
+    virtual void add(AttributeInitializer::UP initializer) = 0;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
new file mode 100644
index 00000000000..4d1fe4134b1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attribute_collection_spec.h"
+#include "exclusive_attribute_read_accessor.h"
+#include "i_attribute_factory.h"
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+#include <vespa/searchlib/common/serialnum.h>
+
+namespace search
+{
+
+class IDestructorCallback;
+class ISequencedTaskExecutor;
+
+}
+
+namespace proton {
+
+class IAttributeFunctor;
+
+/**
+ * Proton specific interface for an attribute manager that handles a set of attribute vectors.
+ *
+ * The attribute manager should handle initialization and loading of attribute vectors,
+ * and then provide access to the attributes for feeding, searching and flushing.
+ */
+struct IAttributeManager : public search::IAttributeManager
+{
+    typedef std::shared_ptr<IAttributeManager> SP;
+
+    using OnWriteDoneType = const std::shared_ptr<search::IDestructorCallback> &;
+
+    virtual ~IAttributeManager() {}
+
+    /**
+     * Create a new attribute manager based on the content of the current one and
+     * the given attribute collection spec.
+     */
+    virtual IAttributeManager::SP create(const AttributeCollectionSpec &spec) const = 0;
+
+    /**
+     * Return the list of flush targets for this attribute manager.
+     */
+    virtual std::vector<searchcorespi::IFlushTarget::SP> getFlushTargets() const = 0;
+
+    /**
+     * Returns the flushed serial num for the given attribute.
+     * Return 0 if attribute is not found.
+     */
+    virtual search::SerialNum getFlushedSerialNum(const vespalib::string &name) const = 0;
+
+    /**
+     * Return the oldest flushed serial number among the underlying attribute vectors.
+     */
+    virtual search::SerialNum getOldestFlushedSerialNumber() const = 0;
+
+    virtual search::SerialNum
+    getNewestFlushedSerialNumber() const = 0;
+
+    /**
+     * Fills all underlying attribute vectors (including extra attributes) into the given list.
+     */
+    virtual void getAttributeListAll(std::vector<search::AttributeGuard> &list) const = 0;
+
+    /**
+     * Wipe history using the given schema.
+     */
+    virtual void wipeHistory(const search::index::Schema &historySchema) = 0;
+
+    /**
+     * Returns the attribute factory used by this manager.
+     */
+    virtual const IAttributeFactory::SP &getFactory() const = 0;
+
+    virtual search::ISequencedTaskExecutor &
+    getAttributeFieldWriter() const = 0;
+
+    /*
+     * Get pointer to named writable attribute.  If attribute isn't
+     * found or is an extra attribute then nullptr is returned.
+     *
+     * The attribute writer doesn't need attribute guards to access
+     * attributes.  Lifetime should be guaranteed by syncing threads
+     * at config changes.
+     */
+    virtual search::AttributeVector *
+    getWritableAttribute(const vespalib::string &name) const = 0;
+
+    /*
+     * Get pointers to all writable attributes.
+     *
+     * The attribute writer doesn't need attribute guards to access
+     * attributes.  Lifetime should be guaranteed by syncing threads
+     * at config changes.
+     */
+    virtual const std::vector<search::AttributeVector *> &
+    getWritableAttributes() const = 0;
+
+    virtual void
+    asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const = 0;
+
+    virtual ExclusiveAttributeReadAccessor::UP
+    getExclusiveReadAccessor(const vespalib::string &name) const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
new file mode 100644
index 00000000000..27fcbe03d38
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
+
+namespace search {
+
+class IDestructorCallback;
+
+}
+
+namespace proton {
+
+/**
+ * Interface for an attribute writer that handles writes in form of put, update and remove
+ * to an underlying set of attribute vectors.
+ */
+class IAttributeWriter {
+public:
+    typedef std::unique_ptr<IAttributeWriter> UP;
+    typedef std::shared_ptr<IAttributeWriter> SP;
+    typedef std::vector<search::AttributeGuard> AttributeGuardList;
+    typedef LidVectorContext::LidVector LidVector;
+    typedef search::SerialNum SerialNum;
+    typedef search::DocumentIdT DocumentIdT;
+    typedef document::DocumentUpdate DocumentUpdate;
+    typedef document::Document Document;
+    using OnWriteDoneType = const std::shared_ptr<search::IDestructorCallback> &;
+
+    virtual ~IAttributeWriter() {}
+
+    virtual std::vector<search::AttributeVector *>
+    getWritableAttributes() const = 0;
+    virtual search::AttributeVector *
+    getWritableAttribute(const vespalib::string &attrName) const = 0;
+    virtual void put(SerialNum serialNum, const Document &doc, DocumentIdT lid,
+                     bool immediateCommit, OnWriteDoneType onWriteDone) = 0;
+    virtual void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit,
+                        OnWriteDoneType onWriteDone) = 0;
+    virtual void remove(const LidVector &lidVector, SerialNum serialNum,
+                        bool immediateCommit, OnWriteDoneType onWriteDone) = 0;
+    /**
+     * Update the underlying attributes based on the content of the given DocumentUpdate.
+     * The OnWriteDoneType instance should ensure the lifetime of the given DocumentUpdate instance.
+     */
+    virtual void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid,
+                        bool immediateCommit, OnWriteDoneType onWriteDone) = 0;
+    virtual void heartBeat(SerialNum serialNum) = 0;
+    /**
+     * Compact the lid space of the underlying attribute vectors.
+     */
+    virtual void compactLidSpace(uint32_t wantedLidLimi, SerialNum serialNum) = 0;
+    virtual const proton::IAttributeManager::SP &getAttributeManager() const = 0;
+
+    /**
+     * Commit all underlying attribute vectors with the given serial number.
+     */
+    virtual void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0;
+
+    virtual void onReplayDone(uint32_t docIdLimit) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/initialized_attributes_result.cpp b/searchcore/src/vespa/searchcore/proton/attribute/initialized_attributes_result.cpp
new file mode 100644
index 00000000000..105ef1abbf4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/initialized_attributes_result.cpp
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "initialized_attributes_result.h"
+
+using search::AttributeVector;
+
+namespace proton {
+
+InitializedAttributesResult::InitializedAttributesResult()
+    : _attributes(),
+      _lock()
+{}
+
+void
+InitializedAttributesResult::add(AttributeVector::SP attribute)
+{
+    std::lock_guard<std::mutex> lockGuard(_lock);
+    _attributes.push_back(attribute);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/initialized_attributes_result.h b/searchcore/src/vespa/searchcore/proton/attribute/initialized_attributes_result.h
new file mode 100644
index 00000000000..587c5c23f3f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/initialized_attributes_result.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <mutex>
+
+namespace proton {
+
+/**
+ * Class used to track a set of initialized attribute vectors.
+ */
+class InitializedAttributesResult
+{
+private:
+    std::vector<search::AttributeVector::SP> _attributes;
+    std::mutex _lock;
+
+public:
+    InitializedAttributesResult();
+    void add(search::AttributeVector::SP attribute);
+    const std::vector<search::AttributeVector::SP> &get() const { return _attributes; }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.cpp
new file mode 100644
index 00000000000..1c898bfd1b9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.cpp
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "sequential_attributes_initializer.h"
+#include "attributemanager.h"
+
+using search::AttributeVector;
+using search::SerialNum;
+
+namespace proton {
+
+SequentialAttributesInitializer::SequentialAttributesInitializer(uint32_t docIdLimit)
+    : AttributesInitializerBase(),
+      _docIdLimit(docIdLimit)
+{
+}
+
+void
+SequentialAttributesInitializer::add(AttributeInitializer::UP initializer)
+{
+    AttributeVector::SP attribute = initializer->init();
+    if (attribute) {
+        considerPadAttribute(*attribute, initializer->getCurrentSerialNum(), _docIdLimit);
+        _initializedAttributes.push_back(attribute);
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.h
new file mode 100644
index 00000000000..758c4484ddd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.h
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attributes_initializer_base.h"
+#include <vespa/searchlib/attribute/attributevector.h>
+
+namespace proton {
+
+/**
+ * Class that initializes and loads a set of attribute vectors in sequence.
+ */
+class SequentialAttributesInitializer : public AttributesInitializerBase
+{
+private:
+    uint32_t _docIdLimit;
+
+public:
+    SequentialAttributesInitializer(uint32_t docIdLimit);
+    AttributesVector getInitializedAttributes() const { return _initializedAttributes; }
+    virtual void add(AttributeInitializer::UP initializer) override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt
new file mode 100644
index 00000000000..358e8d4c32f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_bucketdb STATIC
+    SOURCES
+    bucket_db_explorer.cpp
+    bucket_db_owner.cpp
+    bucketdb.cpp
+    bucketdbhandler.cpp
+    bucketsessionbase.cpp
+    bucketstate.cpp
+    joinbucketssession.cpp
+    splitbucketsession.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.cpp
new file mode 100644
index 00000000000..8777f96a7c9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.cpp
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "bucket_db_explorer.h"
+
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+using document::BucketId;
+using vespalib::slime::Cursor;
+using vespalib::slime::Inserter;
+
+namespace proton {
+
+namespace {
+
+vespalib::string
+bucketIdToString(const BucketId &bucketId)
+{
+    vespalib::asciistream stream;
+    stream << "0x" << vespalib::hex << vespalib::setw(sizeof(BucketId::Type)*2)
+           << vespalib::setfill('0') << bucketId.getId();
+    return stream.str();
+}
+
+vespalib::string
+checksumToString(storage::spi::BucketChecksum checksum)
+{
+    vespalib::asciistream stream;
+    stream << "0x" << vespalib::hex << checksum;
+    return stream.str();
+}
+
+void
+convertBucketsToSlime(const BucketDB &bucketDb, Cursor &array)
+{
+    for (auto itr = bucketDb.begin(); itr != bucketDb.end(); ++itr) {
+        Cursor &object = array.addObject();
+        object.setString("id", bucketIdToString(itr->first));
+        const bucketdb::BucketState &state = itr->second;
+        object.setString("checksum", checksumToString(state.getChecksum()));
+        object.setLong("readyCount", state.getReadyCount());
+        object.setLong("notReadyCount", state.getNotReadyCount());
+        object.setLong("removedCount", state.getRemovedCount());
+        object.setBool("active", state.isActive());
+    }
+}
+
+}
+
+BucketDBExplorer::BucketDBExplorer(BucketDBOwner::Guard bucketDb)
+    : _bucketDb(std::move(bucketDb))
+{
+}
+
+void
+BucketDBExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    Cursor &object = inserter.insertObject();
+    if (full) {
+        object.setLong("numBuckets", _bucketDb->size());
+        convertBucketsToSlime(*_bucketDb, object.setArray("buckets"));
+    } else {
+        object.setLong("numBuckets", _bucketDb->size());
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.h
new file mode 100644
index 00000000000..81dd26b7434
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "bucket_db_owner.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of a bucket db and its buckets.
+ */
+class BucketDBExplorer : public vespalib::StateExplorer
+{
+private:
+    BucketDBOwner::Guard _bucketDb;
+
+public:
+    BucketDBExplorer(BucketDBOwner::Guard bucketDb);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.cpp
new file mode 100644
index 00000000000..4e417d91eac
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.cpp
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "bucket_db_owner.h"
+
+namespace proton {
+
+BucketDBOwner::Guard::Guard(BucketDB *bucketDB, Mutex &mutex)
+    : _bucketDB(bucketDB),
+      _guard(mutex)
+{
+}
+
+
+BucketDBOwner::Guard::Guard(Guard &&rhs)
+    : _bucketDB(rhs._bucketDB),
+      _guard(std::move(rhs._guard))
+{
+}
+
+
+BucketDBOwner::BucketDBOwner()
+    : _bucketDB(),
+      _mutex()
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h
new file mode 100644
index 00000000000..4b26b1eaba1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <mutex>
+#include "bucketdb.h"
+
+namespace proton {
+
+/**
+ * Class that owns and provides guarded access to a bucket database.
+ */
+class BucketDBOwner
+{
+    using Mutex = std::mutex;
+
+public:
+    class Guard
+    {
+    private:
+        BucketDB *_bucketDB;
+        std::unique_lock<Mutex> _guard;
+
+    public:
+        Guard(BucketDB *bucketDB, Mutex &mutex);
+        Guard(const Guard &) = delete;
+        Guard(Guard &&rhs);
+        Guard &operator=(const Guard &) = delete;
+        Guard &operator=(Guard &&rhs) = delete;
+        BucketDB *operator->() { return _bucketDB; }
+        BucketDB &operator*() { return *_bucketDB; }
+        const BucketDB *operator->() const { return _bucketDB; }
+        const BucketDB &operator*() const { return *_bucketDB; }
+    };
+
+private:
+    BucketDB _bucketDB;
+    Mutex _mutex;
+
+public:
+    typedef std::shared_ptr<BucketDBOwner> SP;
+
+    BucketDBOwner();
+    Guard takeGuard() {
+        return Guard(&_bucketDB, _mutex);
+    }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp
new file mode 100644
index 00000000000..9f96d4ae02a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp
@@ -0,0 +1,257 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.bucketdb");
+#include "bucketdb.h"
+
+using document::GlobalId;
+using storage::spi::BucketChecksum;
+
+namespace proton
+{
+
+
+BucketDB::BucketDB(void)
+    : _map(),
+      _cachedBucketId(),
+      _cachedBucketState()
+{
+}
+
+
+BucketDB::~BucketDB(void)
+{
+    checkEmpty();
+    clear();
+}
+
+
+bucketdb::BucketState *
+BucketDB::getBucketStatePtr(const BucketId &bucket)
+{
+    MapIterator it(_map.find(bucket));
+    if (it != _map.end()) {
+        return &it->second;
+    }
+    return nullptr;
+}
+
+
+void
+BucketDB::unloadBucket(const BucketId &bucket, const BucketState &delta)
+{
+    BucketState *state = getBucketStatePtr(bucket);
+    assert(state);
+    *state -= delta;
+}
+
+
+const bucketdb::BucketState &
+BucketDB::add(const GlobalId &gid,
+              const BucketId &bucketId,
+              const Timestamp &timestamp,
+              SubDbType subDbType)
+{
+    BucketState &state = _map[bucketId];
+    state.add(gid, timestamp, subDbType);
+    return state;
+}
+
+void
+BucketDB::remove(const GlobalId &gid,
+                 const BucketId &bucketId,
+                 const Timestamp &timestamp,
+                 SubDbType subDbType)
+{
+    BucketState &state = _map[bucketId];
+    state.remove(gid, timestamp, subDbType);
+}
+
+
+void
+BucketDB::modify(const GlobalId &gid,
+                 const BucketId &oldBucketId,
+                 const Timestamp &oldTimestamp,
+                 const BucketId &newBucketId,
+                 const Timestamp &newTimestamp,
+                 SubDbType subDbType)
+{
+    if (oldBucketId == newBucketId) {
+        BucketState &state = _map[oldBucketId];
+        state.modify(oldTimestamp, newTimestamp, subDbType);
+    } else {
+        remove(gid, oldBucketId, oldTimestamp, subDbType);
+        add(gid, newBucketId, newTimestamp, subDbType);
+    }
+}
+
+
+bucketdb::BucketState
+BucketDB::get(const BucketId &bucketId) const
+{
+    Map::const_iterator itr = _map.find(bucketId);
+    if (itr != _map.end()) {
+        return itr->second;
+    }
+    return BucketState();
+}
+
+void
+BucketDB::cacheBucket(const BucketId &bucketId)
+{
+    _cachedBucketId = bucketId;
+    _cachedBucketState = get(bucketId);
+}
+
+void
+BucketDB::uncacheBucket()
+{
+    _cachedBucketId = BucketId();
+    _cachedBucketState = BucketState();
+}
+
+bool
+BucketDB::isCachedBucket(const BucketId &bucketId) const
+{
+    return _cachedBucketId == bucketId;
+}
+
+bucketdb::BucketState
+BucketDB::cachedGet(const BucketId &bucketId) const
+{
+    if (isCachedBucket(bucketId)) {
+        return _cachedBucketState;
+    }
+    return get(bucketId);
+}
+
+bool
+BucketDB::hasBucket(const BucketId &bucketId) const
+{
+    Map::const_iterator itr = _map.find(bucketId);
+    if (itr != _map.end()) {
+        return true;
+    }
+    return false;
+}
+
+
+bool
+BucketDB::isActiveBucket(const BucketId &bucketId) const
+{
+    Map::const_iterator itr = _map.find(bucketId);
+    if (itr != _map.end()) {
+        return itr->second.isActive();
+    }
+    return false;
+}
+
+void
+BucketDB::getBuckets(BucketId::List &buckets) const
+{
+    buckets.reserve(_map.size());
+    for (const auto & entry : _map) {
+        buckets.push_back(entry.first);
+    }
+}
+
+bool
+BucketDB::empty(void) const
+{
+    return _map.empty();
+}
+
+void
+BucketDB::clear(void)
+{
+    _map.clear();
+}
+
+
+void
+BucketDB::checkEmpty(void) const
+{
+    for (auto &entry : _map) {
+        const BucketState &state = entry.second;
+        assert(state.empty());
+    }
+}
+
+
+void
+BucketDB::setBucketState(const BucketId &bucketId, bool active)
+{
+    BucketState &state = _map[bucketId];
+    state.setActive(active);
+}
+
+
+void
+BucketDB::createBucket(const BucketId &bucketId)
+{
+    BucketState &state = _map[bucketId];
+    (void) state;
+}
+
+
+void
+BucketDB::deleteEmptyBucket(const BucketId &bucketId)
+{
+    Map::iterator itr = _map.find(bucketId);
+    if (itr == _map.end()) {
+        return;
+    }
+    const BucketState &state = itr->second;
+    if (state.empty()) {
+        _map.erase(itr);
+    }
+}
+
+
+void
+BucketDB::getActiveBuckets(BucketId::List &buckets) const
+{
+    for (const auto & entry : _map) {
+        if (entry.second.isActive()) {
+            buckets.push_back(entry.first);
+        }
+    }
+}
+
+
+void
+BucketDB::populateActiveBuckets(const BucketId::List &buckets,
+                                BucketId::List &fixupBuckets)
+{
+    typedef BucketId::List BIV;
+    BIV sorted(buckets);
+    BIV toAdd;
+    std::sort(sorted.begin(), sorted.end());
+    BIV::const_iterator si(sorted.begin());
+    BIV::const_iterator se(sorted.end());
+    for (const auto & entry : _map) {
+        for (; si != se && !(entry.first < *si); ++si) {
+            if (*si < entry.first) {
+                toAdd.push_back(*si);
+            } else if (!entry.second.isActive()) {
+                fixupBuckets.push_back(*si);
+                setBucketState(*si, true);
+            }
+        }
+    }
+    for (; si != se; ++si) {
+        toAdd.push_back(*si);
+    }
+    BIV::const_iterator ai(toAdd.begin());
+    BIV::const_iterator ae(toAdd.end());
+    BucketState activeState;
+    activeState.setActive(true);
+    for (; ai != ae; ++ai) {
+        InsertResult ins(_map.insert(std::make_pair(*ai, activeState)));
+        assert(ins.second);
+    }
+}
+
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h
new file mode 100644
index 00000000000..aa267d33af0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/base/globalid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/persistence/spi/bucketinfo.h>
+#include <vespa/searchcore/proton/common/subdbtype.h>
+#include <vespa/persistence/spi/result.h>
+#include "bucketstate.h"
+
+namespace proton
+{
+
+class BucketDB
+{
+public:
+    typedef document::GlobalId GlobalId;
+    typedef document::BucketId BucketId;
+    typedef storage::spi::Timestamp Timestamp;
+    typedef storage::spi::BucketChecksum BucketChecksum;
+    typedef bucketdb::BucketState BucketState;
+    typedef std::map<BucketId, BucketState> Map;
+    typedef Map::iterator MapIterator;
+    typedef Map::const_iterator ConstMapIterator;
+    typedef std::pair<MapIterator, bool> InsertResult;
+
+private:
+    Map _map;
+    BucketId _cachedBucketId;
+    BucketState _cachedBucketState;
+
+    void clear(void);
+    void checkEmpty(void) const;
+public:
+    BucketDB(void);
+    virtual ~BucketDB(void);
+
+    const BucketState &
+    add(const GlobalId &gid,
+        const BucketId &bucketId,
+        const Timestamp &timestamp,
+        SubDbType subDbType);
+
+    void add(const BucketId &bucketId, const BucketState & state) {
+        _map[bucketId] += state;
+    }
+
+    void
+    remove(const GlobalId &gid,
+           const BucketId &bucketId,
+           const Timestamp &timestamp,
+           SubDbType subDbType);
+
+    void
+    modify(const GlobalId &gid,
+           const BucketId &oldBucketId,
+           const Timestamp &oldTimestamp,
+           const BucketId &newBucketId,
+           const Timestamp &newTimestamp,
+           SubDbType subDbType);
+
+    BucketState get(const BucketId &bucketId) const;
+    void cacheBucket(const BucketId &bucketId);
+    void uncacheBucket();
+    bool isCachedBucket(const BucketId &bucketId) const;
+    BucketState cachedGet(const BucketId &bucketId) const;
+    bool hasBucket(const BucketId &bucketId) const;
+    void getBuckets(BucketId::List & buckets) const;
+    bool empty(void) const;
+    void setBucketState(const BucketId &bucketId, bool active);
+    void createBucket(const BucketId &bucketId);
+    void deleteEmptyBucket(const BucketId &bucketId);
+    void getActiveBuckets(BucketId::List &buckets) const;
+    void populateActiveBuckets(const BucketId::List &buckets, BucketId::List &fixupBuckets);
+
+    ConstMapIterator begin() const { return _map.begin(); }
+    ConstMapIterator end() const { return _map.end(); }
+    ConstMapIterator lowerBound(const BucketId &bucket) const { return _map.lower_bound(bucket); }
+    ConstMapIterator upperBound(const BucketId &bucket) const { return _map.upper_bound(bucket); }
+    size_t size() const { return _map.size(); }
+    bool isActiveBucket(const BucketId &bucketId) const;
+    BucketState *getBucketStatePtr(const BucketId &bucket);
+    void unloadBucket(const BucketId &bucket, const BucketState &delta);
+};
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.cpp
new file mode 100644
index 00000000000..a36fca85def
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.cpp
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "bucketdb.h"
+#include "bucketsessionbase.h"
+#include "splitbucketsession.h"
+#include "joinbucketssession.h"
+#include <vespa/searchlib/common/serialnum.h>
+#include "bucketdbhandler.h"
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+
+BucketDBHandler::BucketDBHandler(BucketDBOwner &bucketDB)
+    : _bucketDB(bucketDB),
+      _dmsv()
+{
+}
+
+
+void
+BucketDBHandler::addDocumentMetaStore(IDocumentMetaStore *dms,
+                                      search::SerialNum flushedSerialNum)
+{
+    _dmsv.push_back(MetaStoreDesc(dms, flushedSerialNum));
+}
+
+
+void
+BucketDBHandler::handleSplit(search::SerialNum serialNum,
+                             const BucketId &source,
+                             const BucketId &target1,
+                             const BucketId &target2)
+{
+    // Called by writer thread
+    assert(source.valid());
+    assert(target1.valid() || target2.valid());
+    if (target1.valid()) {
+        assert(source.getUsedBits() < target1.getUsedBits());
+        assert(source.contains(target1));
+    }
+    if (target2.valid()) {
+        assert(source.getUsedBits() < target2.getUsedBits());
+        assert(source.contains(target2));
+    }
+    if (target1.valid() && target2.valid()) {
+        assert(target1 != target2);
+        assert(!target1.contains(target2));
+        assert(!target2.contains(target1));
+    }
+    SplitBucketSession session(_bucketDB, source, target1, target2);
+    session.setup();
+    for (auto &desc : _dmsv) {
+        IDocumentMetaStore *dms = desc._dms;
+        if (serialNum > desc._flushedSerialNum) {
+            BucketDeltaPair deltas = dms->handleSplit(session);
+            session.applyDeltas(deltas);
+            dms->commit(serialNum, serialNum);
+        }
+    }
+    session.finish();
+}
+
+
+void
+BucketDBHandler::handleJoin(search::SerialNum serialNum,
+                            const BucketId &source1,
+                            const BucketId &source2,
+                            const BucketId &target)
+{
+    // Called by writer thread
+    JoinBucketsSession session(_bucketDB, source1, source2, target);
+    session.setup();
+    for (auto &desc : _dmsv) {
+        IDocumentMetaStore *dms = desc._dms;
+        if (serialNum > desc._flushedSerialNum) {
+            BucketDeltaPair deltas = dms->handleJoin(session);
+            session.applyDeltas(deltas);
+            dms->commit(serialNum, serialNum);
+        }
+    }
+    session.finish();
+}
+
+
+void
+BucketDBHandler::handleCreateBucket(const BucketId &bucketId)
+{
+    _bucketDB.takeGuard()->createBucket(bucketId);
+}
+
+
+void
+BucketDBHandler::handleDeleteBucket(const BucketId &bucketId)
+{
+    _bucketDB.takeGuard()->deleteEmptyBucket(bucketId);
+}
+
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.h
new file mode 100644
index 00000000000..464cb2e69a1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.h
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "bucket_db_owner.h"
+#include "ibucketdbhandler.h"
+#include "ibucketdbhandlerinitializer.h"
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+/**
+ * The BucketDBHandler class handles operations on a bucket db.
+ */
+class BucketDBHandler : public IBucketDBHandler,
+                        public IBucketDBHandlerInitializer
+{
+private:
+    struct MetaStoreDesc
+    {
+        IDocumentMetaStore *_dms;
+        search::SerialNum   _flushedSerialNum;
+
+        MetaStoreDesc(IDocumentMetaStore *dms,
+                      search::SerialNum flushedSerialNum)
+            : _dms(dms),
+              _flushedSerialNum(flushedSerialNum)
+        {
+        }
+    };
+
+    BucketDBOwner             &_bucketDB;
+    std::vector<MetaStoreDesc> _dmsv;
+
+public:
+    BucketDBHandler(BucketDBOwner &bucketDB);
+
+    void
+    setBucketDB(BucketDBOwner &bucketDB);
+
+    virtual void
+    addDocumentMetaStore(IDocumentMetaStore *dms,
+                         search::SerialNum flushedSerialNum) override;
+
+    virtual void
+    handleSplit(search::SerialNum serialNum,
+                const BucketId &source,
+                const BucketId &target1,
+                const BucketId &target2) override;
+
+    virtual void
+    handleJoin(search::SerialNum serialNum,
+               const BucketId &source1,
+               const BucketId &source2,
+               const BucketId &target) override;
+
+    virtual void
+    handleCreateBucket(const BucketId &bucketId) override;
+
+    virtual void
+    handleDeleteBucket(const BucketId &bucketId) override;
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdeltapair.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdeltapair.h
new file mode 100644
index 00000000000..f0db5e17867
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdeltapair.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+/**
+ * Class BucketDeltaPair represent the deltas to bucket states caused by
+ * a join or split operation.
+ */
+class BucketDeltaPair
+{
+public:
+    BucketState _delta1;
+    BucketState _delta2;
+
+    BucketDeltaPair()
+        : _delta1(),
+          _delta2()
+    {
+    }
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.cpp
new file mode 100644
index 00000000000..4173cc28f5d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.cpp
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "bucketdb.h"
+#include "bucketsessionbase.h"
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+BucketSessionBase::BucketSessionBase(BucketDBOwner &bucketDB)
+    : _bucketDB(bucketDB.takeGuard())
+{
+}
+
+
+bool
+BucketSessionBase::extractInfo(const BucketId &bucket, BucketState *&state)
+{
+    if (bucket.valid()) {
+        state = _bucketDB->getBucketStatePtr(bucket);
+    }
+    return state && state->isActive();
+}
+
+
+bool
+BucketSessionBase::calcFixupNeed(BucketState *state, bool wantActive,
+                                 bool fixup)
+{
+    if (state && state->isActive() != wantActive) {
+        if (fixup) {
+            state->setActive(wantActive);
+        }
+        return state->getReadyCount() != 0;
+    }
+    return false;
+}
+
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.h
new file mode 100644
index 00000000000..64983ae45bf
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "bucket_db_owner.h"
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+/**
+ * Base class for split/join handling utility classes that bundles temporary
+ * variables used during the operation.
+ */
+class BucketSessionBase
+{
+public:
+    typedef document::GlobalId GlobalId;
+    typedef document::BucketId BucketId;
+    typedef storage::spi::Timestamp Timestamp;
+
+protected:
+    BucketDBOwner::Guard _bucketDB;
+
+public:
+    BucketSessionBase(BucketDBOwner &bucketDB);
+
+    bool
+    extractInfo(const BucketId &bucket, BucketState *&info);
+
+    static bool
+    calcFixupNeed(BucketState *state, bool wantActive, bool fixup);
+};
+
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp
new file mode 100644
index 00000000000..2b33639121b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp
@@ -0,0 +1,157 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+
+#include "bucketdb.h"
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+namespace
+{
+
+uint32_t
+gidChecksum(const document::GlobalId &gid)
+{
+    union {
+        const unsigned char *_c;
+        const uint32_t *_i;
+    } u;
+    u._c = gid.get();
+    const uint32_t *i = u._i;
+    return i[0] + i[1] + i[2];
+}
+
+
+uint32_t
+timestampChecksum(const storage::spi::Timestamp &timestamp)
+{
+    return (timestamp >> 32) + timestamp;
+}
+
+inline uint32_t toIdx(SubDbType subDbType)
+{
+    return static_cast<uint32_t>(subDbType);
+}
+
+}
+
+uint32_t
+BucketState::calcChecksum(const GlobalId &gid,
+                          const Timestamp &timestamp)
+{
+    return gidChecksum(gid) + timestampChecksum(timestamp);
+}
+
+
+void
+BucketState::add(const GlobalId &gid, const Timestamp &timestamp,
+                 SubDbType subDbType)
+{
+    assert(subDbType < SubDbType::COUNT);
+    if (subDbType != SubDbType::REMOVED) {
+        _checksum += calcChecksum(gid, timestamp);
+    }
+    ++_docCount[toIdx(subDbType)];
+}
+
+
+void
+BucketState::remove(const GlobalId &gid, const Timestamp &timestamp,
+                    SubDbType subDbType)
+{
+    assert(subDbType < SubDbType::COUNT);
+    uint32_t subDbTypeIdx = toIdx(subDbType);
+    assert(_docCount[subDbTypeIdx] > 0);
+    if (subDbType != SubDbType::REMOVED) {
+        _checksum -= calcChecksum(gid, timestamp);
+    }
+    --_docCount[subDbTypeIdx];
+}
+
+
+void
+BucketState::modify(const Timestamp &oldTimestamp,
+                    const Timestamp &newTimestamp,
+                    SubDbType subDbType)
+{
+    assert(subDbType < SubDbType::COUNT);
+    assert(_docCount[toIdx(subDbType)] > 0);
+    if (subDbType != SubDbType::REMOVED) {
+        _checksum = _checksum - timestampChecksum(oldTimestamp) +
+                    timestampChecksum(newTimestamp);
+    }
+}
+
+
+bool
+BucketState::empty(void) const
+{
+    if (getReadyCount() != 0 || getRemovedCount() != 0 ||
+        getNotReadyCount() != 0)
+        return false;
+    assert(_checksum == 0);
+    return true;
+}
+
+
+BucketState &
+BucketState::operator+=(const BucketState &rhs)
+{
+    for (uint32_t i = 0; i < COUNTS; ++i) {
+        _docCount[i] += rhs._docCount[i];
+    }
+    _checksum += rhs._checksum;
+    return *this;
+}
+
+
+BucketState &
+BucketState::operator-=(const BucketState &rhs)
+{
+    for (uint32_t i = 0; i < COUNTS; ++i) {
+        assert(_docCount[i] >= rhs._docCount[i]);
+    }
+    for (uint32_t i = 0; i < COUNTS; ++i) {
+        _docCount[i] -= rhs._docCount[i];
+    }
+    _checksum -= rhs._checksum;
+    return *this;
+}
+
+
+void
+BucketState::applyDelta(BucketState *src, BucketState *dst) const
+{
+    if (empty())
+        return;
+    assert(src);
+    assert(dst);
+    *src -= *this;
+    *dst += *this;
+}
+
+BucketState::operator storage::spi::BucketInfo() const
+{
+    uint32_t notReady = getNotReadyCount();
+    uint32_t documentCount = getReadyCount() + notReady;
+    uint32_t entryCount = documentCount + getRemovedCount();
+
+    using BucketInfo = storage::spi::BucketInfo;
+
+    return BucketInfo(storage::spi::BucketChecksum(_checksum),
+                      documentCount,
+                      0,
+                      entryCount,
+                      0,
+                      notReady > 0 ? BucketInfo::NOT_READY : BucketInfo::READY,
+                      _active ? BucketInfo::ACTIVE : BucketInfo::NOT_ACTIVE);
+}
+
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h
new file mode 100644
index 00000000000..bd3f64e381c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+/**
+ * Class BucketState represent the known state of a bucket in raw form.
+ */
+class BucketState
+{
+public:
+    typedef document::GlobalId GlobalId;
+    typedef storage::spi::Timestamp Timestamp;
+
+private:
+    static constexpr uint32_t READY = static_cast<uint32_t>(SubDbType::READY);
+    static constexpr uint32_t REMOVED =
+        static_cast<uint32_t>(SubDbType::REMOVED);
+    static constexpr uint32_t NOTREADY =
+        static_cast<uint32_t>(SubDbType::NOTREADY);
+    static constexpr uint32_t COUNTS = static_cast<uint32_t>(SubDbType::COUNT);
+    uint32_t _docCount[COUNTS];
+    uint32_t _checksum;
+    bool _active;
+
+public:
+    BucketState() :
+        _docCount(),
+        _checksum(0),
+        _active(false)
+    {
+        for (uint32_t i = 0; i < COUNTS; ++i) {
+            _docCount[i] = 0;
+        }
+    }
+
+    static uint32_t
+    calcChecksum(const GlobalId &gid,
+                 const Timestamp &timestamp);
+
+    void
+    add(const GlobalId &gid, const Timestamp &timestamp, SubDbType subDbType);
+
+    void
+    remove(const GlobalId &gid, const Timestamp &timestamp,
+           SubDbType subDbType);
+
+    void
+    modify(const Timestamp &oldTimestamp, const Timestamp &newTimestamp,
+           SubDbType subDbType);
+
+    bool
+    isActive() const
+    {
+        return _active;
+    }
+
+    BucketState &
+    setActive(bool active)
+    {
+        _active = active;
+        return *this;
+    }
+
+    uint32_t
+    getReadyCount() const
+    {
+        return _docCount[READY];
+    }
+
+    uint32_t
+    getRemovedCount() const
+    {
+        return _docCount[REMOVED];
+    }
+
+    uint32_t
+    getNotReadyCount() const
+    {
+        return _docCount[NOTREADY];
+    }
+
+    uint32_t
+    getDocumentCount() const
+    {
+        return getReadyCount() + getNotReadyCount();
+    }
+
+    uint32_t
+    getEntryCount() const
+    {
+        return getDocumentCount() + getRemovedCount();
+    }
+
+    storage::spi::BucketChecksum
+    getChecksum() const
+    {
+        return storage::spi::BucketChecksum(_checksum);
+    }
+
+    bool
+    empty() const;
+
+    BucketState &
+    operator+=(const BucketState &rhs);
+
+    BucketState &
+    operator-=(const BucketState &rhs);
+
+    void
+    applyDelta(BucketState *src, BucketState *dst) const;
+
+    operator storage::spi::BucketInfo() const;
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/ibucketdbhandler.h b/searchcore/src/vespa/searchcore/proton/bucketdb/ibucketdbhandler.h
new file mode 100644
index 00000000000..f73dd4788a0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/ibucketdbhandler.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/searchlib/common/serialnum.h>
+
+namespace proton
+{
+
+class IDocumentMetaStore;
+
+namespace bucketdb
+{
+
+/**
+ * The IBucketDBHandler class handles operations on a bucket db.
+ */
+class IBucketDBHandler
+{
+public:
+    typedef document::BucketId BucketId;
+
+    IBucketDBHandler()
+    {
+    }
+
+    virtual ~IBucketDBHandler()
+    {
+    }
+
+    virtual void
+    handleSplit(search::SerialNum serialNum,
+                const BucketId &source, const BucketId &target1,
+                const BucketId &target2) = 0;
+
+    virtual void
+    handleJoin(search::SerialNum serialNum,
+               const BucketId &source1, const BucketId &source2,
+               const BucketId &target) = 0;
+
+    virtual void
+    handleCreateBucket(const BucketId &bucketId) = 0;
+
+    virtual void
+    handleDeleteBucket(const BucketId &bucketId) = 0;
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/ibucketdbhandlerinitializer.h b/searchcore/src/vespa/searchcore/proton/bucketdb/ibucketdbhandlerinitializer.h
new file mode 100644
index 00000000000..85a3b5e88a1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/ibucketdbhandlerinitializer.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+/**
+ * The IBucketDBHandlerInitiaizer class handles initialization of a 
+ * BucketDBHandler.
+ */
+class IBucketDBHandlerInitializer
+{
+public:
+    IBucketDBHandlerInitializer()
+    {
+    }
+
+    virtual ~IBucketDBHandlerInitializer()
+    {
+    }
+
+    virtual void
+    addDocumentMetaStore(IDocumentMetaStore *dms,
+                         search::SerialNum flushedSerialNum) = 0;
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.cpp
new file mode 100644
index 00000000000..ce46df2c450
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.cpp
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "bucketdb.h"
+#include "bucketsessionbase.h"
+#include "joinbucketssession.h"
+#include "bucketdeltapair.h"
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+JoinBucketsSession::JoinBucketsSession(BucketDBOwner &bucketDB,
+                                       const BucketId &source1,
+                                       const BucketId &source2,
+                                       const BucketId &target)
+    : BucketSessionBase(bucketDB),
+      _source1Delta(),
+      _source2Delta(),
+      _wantTargetActive(false),
+      _adjustSource1ActiveLids(false),
+      _adjustSource2ActiveLids(false),
+      _adjustTargetActiveLids(false),
+      _source1(source1),
+      _source2(source2),
+      _target(target)
+{
+}
+
+
+void
+JoinBucketsSession::setup()
+{
+    if (_target.valid()) {
+        _bucketDB->createBucket(_target);
+    }
+    BucketState *source1State = nullptr;
+    BucketState *source2State = nullptr;
+    bool source1Active = extractInfo(_source1, source1State);
+    bool source2Active = extractInfo(_source2, source2State);
+    _wantTargetActive = source1Active || source2Active;
+
+    _adjustSource1ActiveLids = calcFixupNeed(source1State, _wantTargetActive,
+                                             false);
+    _adjustSource2ActiveLids = calcFixupNeed(source2State, _wantTargetActive,
+                                             false);
+    BucketState *targetState = nullptr;
+    (void) extractInfo(_target, targetState);
+    _adjustTargetActiveLids = calcFixupNeed(targetState, _wantTargetActive,
+                                            true);
+}
+
+
+bool
+JoinBucketsSession::mustFixupTargetActiveLids(bool movedSource1Docs,
+                                              bool movedSource2Docs) const
+{
+    return _adjustTargetActiveLids ||
+        (_adjustSource1ActiveLids && movedSource1Docs) ||
+        (_adjustSource2ActiveLids && movedSource2Docs);
+}
+
+
+void
+JoinBucketsSession::applyDeltas(const BucketDeltaPair &deltas)
+{
+    _source1Delta += deltas._delta1;
+    _source2Delta += deltas._delta2;
+}
+
+
+bool
+JoinBucketsSession::applyDelta(const BucketState &delta, BucketId &srcBucket,
+                              BucketState *dst)
+{
+    if (!srcBucket.valid()) {
+        assert(delta.empty());
+        return false;
+    }
+    BucketState *src = _bucketDB->getBucketStatePtr(srcBucket);
+    if (delta.empty()) {
+        return src && src->empty();
+    }
+    delta.applyDelta(src, dst);
+    return src->empty();
+}
+
+
+void
+JoinBucketsSession::finish()
+{
+    if (!_target.valid()) {
+        assert(_source1Delta.empty());
+        assert(_source2Delta.empty());
+        return;
+    }
+    BucketState *targetState = _bucketDB->getBucketStatePtr(_target);
+    bool source1Empty = applyDelta(_source1Delta, _source1, targetState);
+    bool source2Empty = applyDelta(_source2Delta, _source2, targetState);
+    if (source1Empty) {
+        _bucketDB->deleteEmptyBucket(_source1);
+    }
+    if (source2Empty) {
+        _bucketDB->deleteEmptyBucket(_source2);
+    }
+}
+
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.h b/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.h
new file mode 100644
index 00000000000..f05521a06c0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.h
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+class BucketDeltaPair;
+
+/**
+ * The JoinBucketsSession class bundles some temporary variables used
+ * during a join operation, allowing for a cleaner API when calling
+ * methods to perform some of the steps in the join operation.
+ *
+ * It sets up variables in the constructor, contains a few methods that
+ * are forwarded to BucketDB with appropriate argument shuffling, and
+ * also removes empty source buckets after join.
+ *
+ */
+class JoinBucketsSession : public BucketSessionBase
+{
+private:
+    BucketState _source1Delta;
+    BucketState _source2Delta;
+    bool _wantTargetActive;
+    bool _adjustSource1ActiveLids;
+    bool _adjustSource2ActiveLids;
+    bool _adjustTargetActiveLids;
+    BucketId _source1;
+    BucketId _source2;
+    BucketId _target;
+
+    bool
+    applyDelta(const BucketState &delta, BucketId &srcBucket, BucketState *dst);
+
+public:
+    JoinBucketsSession(BucketDBOwner &bucketDB,
+                       const BucketId &source1,
+                       const BucketId &source2,
+                       const BucketId &target);
+
+    void
+    applyDeltas(const BucketDeltaPair &deltas);
+
+    bool
+    getWantTargetActive() const
+    {
+        return _wantTargetActive;
+    }
+
+    bool
+    mustFixupTargetActiveLids(bool movedSource1Docs,
+                              bool movedSource2Docs) const;
+
+
+    void
+    setup();
+
+    void
+    finish();
+
+    const BucketId &
+    getSource1() const
+    {
+        return _source1;
+    }
+
+    const BucketId &
+    getSource2() const
+    {
+        return _source2;
+    }
+
+    const BucketId &
+    getTarget() const
+    {
+        return _target;
+    }
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.cpp
new file mode 100644
index 00000000000..5cc4d74cf25
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.cpp
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "bucketdb.h"
+#include "bucketsessionbase.h"
+#include "splitbucketsession.h"
+#include "bucketdeltapair.h"
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+
+SplitBucketSession::SplitBucketSession(BucketDBOwner &bucketDB,
+                                       const BucketId &source,
+                                       const BucketId &target1,
+                                       const BucketId &target2)
+    : BucketSessionBase(bucketDB),
+      _target1Delta(),
+      _target2Delta(),
+      _sourceActive(false),
+      _adjustTarget1ActiveLids(false),
+      _adjustTarget2ActiveLids(false),
+      _source(source),
+      _target1(target1),
+      _target2(target2)
+{
+}
+
+
+void
+SplitBucketSession::setup()
+{
+    if (_target1.valid()) {
+        _bucketDB->createBucket(_target1);
+    }
+    if (_target2.valid()) {
+        _bucketDB->createBucket(_target2);
+    }
+
+    BucketState *sourceState = nullptr;
+    _sourceActive = extractInfo(_source, sourceState);
+
+    if (_target1.valid()) {
+        BucketState *target1State = _bucketDB->getBucketStatePtr(_target1);
+        _adjustTarget1ActiveLids = calcFixupNeed(target1State, _sourceActive,
+                                                 true);
+    }
+    if (_target2.valid()) {
+        BucketState *target2State = _bucketDB->getBucketStatePtr(_target2);
+        _adjustTarget2ActiveLids = calcFixupNeed(target2State, _sourceActive,
+                                                 true);
+    }
+}
+
+
+void
+SplitBucketSession::applyDeltas(const BucketDeltaPair &deltas)
+{
+    _target1Delta += deltas._delta1;
+    _target2Delta += deltas._delta2;
+}
+
+
+void
+SplitBucketSession::applyDelta(const BucketState &delta, BucketState *src,
+                            BucketId &dstBucket)
+{
+    if (delta.empty())
+        return;
+    assert(dstBucket.valid());
+    BucketState *dst = _bucketDB->getBucketStatePtr(dstBucket);
+    delta.applyDelta(src, dst);
+}
+
+
+void
+SplitBucketSession::finish()
+{
+    BucketState *sourceState = nullptr;
+    (void) extractInfo(_source, sourceState);
+    if (!sourceState) {
+        assert(_target1Delta.empty());
+        assert(_target2Delta.empty());
+        return;
+    }
+    applyDelta(_target1Delta, sourceState, _target1);
+    applyDelta(_target2Delta, sourceState, _target2);
+    if (sourceState && sourceState->empty()) {
+        _bucketDB->deleteEmptyBucket(_source);
+    }
+}
+
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.h b/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.h
new file mode 100644
index 00000000000..260a32d7dbd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.h
@@ -0,0 +1,105 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+namespace bucketdb
+{
+
+class BucketDeltaPair;
+
+/**
+ * The SplitBucketSession class bundles some temporary variables used
+ * during a split operation, allowing for a cleaner API when calling
+ * methods to perform some of the steps in the split operation.
+ *
+ * It sets up variables in the constructor, contains a few methods that
+ * are forwarded to BucketDB with appropriate argument shuffling, and
+ * also removes empty source bucket after split.
+ *
+ */
+class SplitBucketSession : public BucketSessionBase
+{
+private:
+    BucketState _target1Delta;
+    BucketState _target2Delta;
+    bool _sourceActive;
+    bool _adjustTarget1ActiveLids;
+    bool _adjustTarget2ActiveLids;
+    BucketId _source;
+    BucketId _target1;
+    BucketId _target2;
+
+    void
+    applyDelta(const BucketState &delta, BucketState *src, BucketId &dstBucket);
+
+public:
+    SplitBucketSession(BucketDBOwner &bucketDB,
+                       const BucketId &source,
+                       const BucketId &target1,
+                       const BucketId &target2);
+
+    /*
+     * Reflect move of documents to target1 and target2 in bucket states
+     */
+    void
+    applyDeltas(const BucketDeltaPair &deltas);
+
+    bool
+    getSourceActive() const
+    {
+        return _sourceActive;
+    }
+
+    /*
+     * Return true if bitvector for active lids need to be adjusted in
+     * document meta store due to old documents in target1 and active
+     * state change.
+     */
+    bool
+    mustFixupTarget1ActiveLids() const
+    {
+        return _adjustTarget1ActiveLids;
+    }
+
+    /*
+     * Return true if bitvector for active lids need to be adjusted in
+     * document meta store due to old documents in target2 and active
+     * state change.
+     */
+    bool
+    mustFixupTarget2ActiveLids() const
+    {
+        return _adjustTarget2ActiveLids;
+    }
+
+    void
+    setup();
+
+    void
+    finish();
+
+    const BucketId &
+    getSource() const
+    {
+        return _source;
+    }
+
+    const BucketId &
+    getTarget1() const
+    {
+        return _target1;
+    }
+
+    const BucketId &
+    getTarget2() const
+    {
+        return _target2;
+    }
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/common/.gitignore b/searchcore/src/vespa/searchcore/proton/common/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
new file mode 100644
index 00000000000..2af74b40dd0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
@@ -0,0 +1,21 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_pcommon STATIC
+    SOURCES
+    attributefieldvaluenode.cpp
+    attrupdate.cpp
+    bucketfactory.cpp
+    cachedselect.cpp
+    commit_time_tracker.cpp
+    dbdocumentid.cpp
+    doctypename.cpp
+    document_type_inspector.cpp
+    eventlogger.cpp
+    feeddebugger.cpp
+    feedtoken.cpp
+    schemautil.cpp
+    selectpruner.cpp
+    selectcontext.cpp
+    state_reporter_utils.cpp
+    DEPENDS
+    searchcore_proton_metrics
+)
diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
new file mode 100644
index 00000000000..73df7d511c9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.attributefieldvaluenode");
+#include "attributefieldvaluenode.h"
+#include "selectcontext.h"
+#include <vespa/document/select/value.h>
+#include <vespa/searchcommon/attribute/attributecontent.h>
+
+namespace proton
+{
+
+using document::select::Context;
+using document::select::FloatValue;
+using document::select::IntegerValue;
+using document::select::NullValue;
+using document::select::StringValue;
+using document::select::Value;
+using document::select::ValueNode;
+using document::select::Visitor;
+using search::AttributeVector;
+using search::attribute::AttributeContent;
+using search::attribute::BasicType;
+using search::attribute::CollectionType;
+using search::attribute::IAttributeVector;
+
+
+AttributeFieldValueNode::
+AttributeFieldValueNode(const vespalib::string& doctype,
+                        const vespalib::string& field,
+                        const search::AttributeVector::SP &attribute)
+    : FieldValueNode(doctype, field),
+      _attribute(attribute)
+{
+    const AttributeVector &v(*_attribute);
+    // Only handle single value attribute vectors for now
+    assert(v.getCollectionType() == CollectionType::SINGLE);
+    (void) v;
+}
+
+
+std::unique_ptr<document::select::Value>
+AttributeFieldValueNode::
+getValue(const Context &context) const
+{
+    const SelectContext &sc(static_cast<const SelectContext &>(context)); 
+    uint32_t docId(sc._docId); 
+    assert(docId != 0u);
+    const AttributeVector &v(*_attribute);
+    if (v.isUndefined(docId)) {
+        return Value::UP(new NullValue);
+    }
+    switch (v.getBasicType()) {
+    case BasicType::STRING:
+        do {
+            AttributeContent<const char *> content;
+            content.fill(v, docId);
+            assert(content.size() == 1u);
+            return Value::UP(new StringValue(content[0]));
+        } while (0);
+        break;
+    case BasicType::UINT1:
+    case BasicType::UINT2:
+    case BasicType::UINT4:
+    case BasicType::INT8:
+    case BasicType::INT16:
+    case BasicType::INT32:
+    case BasicType::INT64:
+        do {
+            AttributeContent<IAttributeVector::largeint_t> content;
+            content.fill(v, docId);
+            assert(content.size() == 1u);
+            return Value::UP(new IntegerValue(content[0], false));
+        } while (0);
+        break;
+    case BasicType::FLOAT:
+    case BasicType::DOUBLE:
+        do {
+            AttributeContent<double> content;
+            content.fill(v, docId);
+            assert(content.size() == 1u);
+            return Value::UP(new FloatValue(content[0]));
+        } while (0);
+        break;
+    default:
+        abort();
+    }
+    return Value::UP();
+
+}
+
+
+std::unique_ptr<Value>
+AttributeFieldValueNode::traceValue(const Context &context,
+                                    std::ostream& out) const
+{
+    return defaultTrace(getValue(context), out);
+}
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h
new file mode 100644
index 00000000000..0f9716727a8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/select/valuenode.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+
+namespace proton
+{
+
+
+class AttributeFieldValueNode : public document::select::FieldValueNode
+{
+    search::AttributeVector::SP _attribute;
+
+public:
+    AttributeFieldValueNode(const vespalib::string& doctype,
+                            const vespalib::string& field,
+                            const search::AttributeVector::SP &attribute);
+
+    virtual std::unique_ptr<document::select::Value>
+    getValue(const document::select::Context &context) const;
+
+    virtual std::unique_ptr<document::select::Value>
+    traceValue(const document::select::Context &context,
+               std::ostream& out) const;
+
+    document::select::ValueNode::UP
+    clone(void) const
+    {
+        return wrapParens(new AttributeFieldValueNode(getDocType(),
+                                                      getFieldName(),
+                                                      _attribute));
+    }
+};
+
+} // namespace proton
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp
new file mode 100644
index 00000000000..c80b9a6b5cf
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp
@@ -0,0 +1,415 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/literalfieldvalue.h>
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/update/addvalueupdate.h>
+#include <vespa/document/update/removevalueupdate.h>
+#include <vespa/document/update/mapvalueupdate.h>
+#include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/update/clearvalueupdate.h>
+#include <vespa/document/base/forcelink.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/searchlib/common/base.h>
+#include <vespa/searchlib/attribute/attribute.h>
+#include <vespa/searchlib/attribute/tensorattribute.h>
+#include "attrupdate.h"
+
+#include <vespa/searchlib/attribute/attributevector.hpp>
+LOG_SETUP(".attrupdate");
+
+using namespace document;
+using vespalib::make_string;
+using search::attribute::TensorAttribute;
+
+namespace {
+    std::string toString(const FieldUpdate & update) {
+        std::stringstream ss;
+        update.print(ss, true, "");
+        return ss.str();
+    }
+
+    std::string toString(const ValueUpdate & update) {
+        std::stringstream ss;
+        update.print(ss, true, "");
+        return ss.str();
+    }
+
+    std::string toString(const FieldValue & value) {
+        std::stringstream ss;
+        value.print(ss, true, "");
+        return ss.str();
+    }
+}
+
+namespace search {
+    namespace forcelink {
+        void forceLink()
+        {
+            document::ForceLink tmp;
+        }
+    }
+
+struct GetFloat {
+    typedef float T;
+    T operator () (const FieldValue & fv) const { return fv.getAsFloat(); }
+};
+
+struct GetDouble {
+    typedef double T;
+    T operator () (const FieldValue & fv) const { return fv.getAsDouble(); }
+};
+
+struct GetLong {
+    typedef int64_t T;
+    T operator () (const FieldValue & fv) const { return fv.getAsLong(); }
+};
+
+struct GetInt {
+    typedef int32_t T;
+    T operator () (const FieldValue & fv) const { return fv.getAsInt(); }
+};
+
+struct GetString {
+    typedef vespalib::stringref T;
+    T operator () (const FieldValue & fv) const { return static_cast<const LiteralFieldValueB &>(fv).getValueRef(); }
+};
+
+VESPA_IMPLEMENT_EXCEPTION(UpdateException, vespalib::Exception);
+
+template <typename G>
+class ArrayAccessor {
+public:
+    ArrayAccessor(const ArrayFieldValue & array) : _array(array), _current(0), _size(_array.size()) { }
+    size_t size()         const { return _size; }
+    bool isAtEnd()        const { return _current >= _size;}
+    void next()                 {  _current++; }
+    typename G::T value() const { return _accessor(_array[_current]); }
+    int32_t weight()      const { return 1; }
+private:
+    G _accessor;
+    const ArrayFieldValue & _array;
+    size_t _current;
+    size_t _size;
+};
+
+template <typename G>
+class WeightedSetAccessor {
+public:
+    WeightedSetAccessor(const WeightedSetFieldValue & ws) : _size(ws.size()), _current(ws.begin()), _end(ws.end()) { }
+    size_t size()         const { return _size; }
+    bool isAtEnd()        const { return _current == _end;}
+    void next()                 { _current++; }
+    typename G::T value() const { return _accessor(*_current->first); }
+    int32_t weight()      const { return _current->second->getAsInt(); }
+private:
+    G _accessor;
+    size_t _size;
+    MapFieldValue::const_iterator _current;
+    MapFieldValue::const_iterator _end;
+};
+
+template <typename V, typename Accessor>
+void
+AttrUpdate::handleUpdateT(V & vec, Accessor, uint32_t lid, const ValueUpdate & upd)
+{
+    LOG(spam, "handleValueUpdate(%s, %u): %s", vec.getName().c_str(), lid, toString(upd).c_str());
+    ValueUpdate::ValueUpdateType op = upd.getType();
+    if (vec.hasMultiValue()) {
+        if (op == ValueUpdate::Clear) {
+            vec.clearDoc(lid);
+        } else if (op == ValueUpdate::Assign) {
+            vec.clearDoc(lid);
+            const AssignValueUpdate & assign(static_cast<const AssignValueUpdate &>(upd));
+            if (assign.hasValue()) {
+                const FieldValue & fv(assign.getValue());
+                const vespalib::Identifiable::RuntimeClass & rc(fv.getClass());
+                if (rc.inherits(ArrayFieldValue::classId)) {
+                    ArrayAccessor<Accessor> ac(static_cast<const ArrayFieldValue & >(fv));
+                    appendValue(vec, lid, ac);
+                } else if (rc.inherits(WeightedSetFieldValue::classId)) {
+                    WeightedSetAccessor<Accessor> ac(static_cast<const WeightedSetFieldValue & >(fv));
+                    appendValue(vec, lid, ac);
+                } else {
+                    LOG(warning, "Unsupported value %s in assign operation on multivalue vector %s",
+                                 rc.name(), vec.getName().c_str());
+                }
+            }
+        } else if (op == ValueUpdate::Add) {
+            const AddValueUpdate & add(static_cast<const AddValueUpdate &>(upd));
+            appendValue(vec, lid, add.getValue(), add.getWeight());
+        } else if (op == ValueUpdate::Remove) {
+            const RemoveValueUpdate & remove(static_cast<const RemoveValueUpdate &>(upd));
+            removeValue(vec, lid, remove.getKey());
+        } else if (op == ValueUpdate::Map) {
+            const MapValueUpdate & map(static_cast<const MapValueUpdate &>(upd));
+            if (!vec.AttributeVector::apply(lid, map)) {
+                throw UpdateException(make_string("attribute map(%s, %s) failed: %s[%u]",
+                                                  map.getKey().getClass().name(), map.getUpdate().getClass().name(),
+                                                  vec.getName().c_str(), lid));
+            }
+        } else {
+            LOG(warning, "Unsupported value update operation %s on multivalue vector %s",
+                         upd.getClass().name(), vec.getName().c_str());
+        }
+    } else {
+        if (op == ValueUpdate::Assign) {
+            const AssignValueUpdate & assign(static_cast<const AssignValueUpdate &>(upd));
+            if (assign.hasValue()) {
+                updateValue(vec, lid, assign.getValue());
+            }
+        } else if (op == ValueUpdate::Arithmetic) {
+            const ArithmeticValueUpdate & aritmetic(static_cast<const ArithmeticValueUpdate &>(upd));
+            if (!vec.apply(lid, aritmetic)) {
+                throw UpdateException(make_string("attribute arithmetic failed: %s[%u]", vec.getName().c_str(), lid));
+            }
+        } else if (op == ValueUpdate::Clear) {
+            vec.clearDoc(lid);
+        } else {
+            LOG(warning, "Unsupported value update operation %s on singlevalue vector %s", upd.getClass().name(), vec.getName().c_str());
+        }
+    }
+}
+
+template <>
+void AttrUpdate::handleUpdate(PredicateAttribute &vec, uint32_t lid, const ValueUpdate &upd) {
+    LOG(spam, "handleValueUpdate(%s, %u): %s", vec.getName().c_str(), lid, toString(upd).c_str());
+    ValueUpdate::ValueUpdateType op = upd.getType();
+    assert(!vec.hasMultiValue());
+    if (op == ValueUpdate::Assign) {
+        const AssignValueUpdate &assign(static_cast<const AssignValueUpdate &>(upd));
+        if (assign.hasValue()) {
+            vec.clearDoc(lid);
+            updateValue(vec, lid, assign.getValue());
+        }
+    } else if (op == ValueUpdate::Clear) {
+        vec.clearDoc(lid);
+    } else {
+        LOG(warning, "Unsupported value update operation %s on singlevalue vector %s",
+                     upd.getClass().name(), vec.getName().c_str());
+    }
+}
+
+void
+AttrUpdate::handleUpdate(AttributeVector & vec, uint32_t lid, const FieldUpdate & fUpdate)
+{
+    LOG(spam, "handleFieldUpdate(%s, %u): %s", vec.getName().c_str(), lid, toString(fUpdate).c_str());
+    for(const auto & update : fUpdate.getUpdates()) {
+        const ValueUpdate & vUp(*update);
+        ValueUpdate::ValueUpdateType op = vUp.getType();
+
+        if (!vec.hasMultiValue()) {
+            if ((op == ValueUpdate::Add) ||
+                (op == ValueUpdate::Remove) ||
+                (op == ValueUpdate::Map))
+            {
+                LOG(warning, "operation append/remove not supported for "
+                    "single value attribute vectors (%s)", vec.getName().c_str());
+                continue;
+            }
+        }
+
+        const vespalib::Identifiable::RuntimeClass &info = vec.getClass();
+        if (info.inherits(IntegerAttribute::classId)) {
+            handleUpdateT(static_cast<IntegerAttribute &>(vec), GetLong(), lid, vUp);
+        } else if (info.inherits(FloatingPointAttribute::classId)) {
+            handleUpdateT(static_cast<FloatingPointAttribute &>(vec), GetDouble(), lid, vUp);
+        } else if (info.inherits(StringAttribute::classId)) {
+            handleUpdateT(static_cast<StringAttribute &>(vec), GetString(), lid, vUp);
+        } else if (info.inherits(PredicateAttribute::classId)) {
+            handleUpdate(static_cast<PredicateAttribute &>(vec), lid, vUp);
+        } else {
+            LOG(warning, "Unsupported attribute vector '%s' (classname=%s)", vec.getName().c_str(), info.name());
+            return;
+        }
+    }
+}
+
+void
+AttrUpdate::handleValue(AttributeVector & vec, uint32_t lid, const FieldValue & val)
+{
+    LOG(spam, "handleValue(%s, %u): %s", vec.getName().c_str(), lid, toString(val).c_str());
+    const vespalib::Identifiable::RuntimeClass & rc = vec.getClass();
+    if (rc.inherits(IntegerAttribute::classId)) {
+        handleValueT(static_cast<IntegerAttribute &>(vec), GetLong(), lid, val);
+    } else if (rc.inherits(FloatingPointAttribute::classId)) {
+        handleValueT(static_cast<FloatingPointAttribute &>(vec), GetDouble(), lid, val);
+    } else if (rc.inherits(StringAttribute::classId)) {
+        handleValueT(static_cast<StringAttribute &>(vec), GetString(), lid, val);
+    } else if (rc.inherits(PredicateAttribute::classId)) {
+        // PredicateAttribute is never multivalue.
+        updateValue(static_cast<PredicateAttribute &>(vec), lid, val);
+    } else if (rc.inherits(TensorAttribute::classId)) {
+        // TensorAttribute is never multivalue.
+        updateValue(static_cast<TensorAttribute &>(vec), lid, val);
+    } else {
+        LOG(warning, "Unsupported attribute vector '%s' (classname=%s)", vec.getName().c_str(), rc.name());
+        return;
+    }
+}
+
+template <typename V, typename Accessor>
+void
+AttrUpdate::handleValueT(V & vec, Accessor, uint32_t lid, const FieldValue & val)
+{
+    if (vec.hasMultiValue()) {
+        vec.clearDoc(lid);
+        const vespalib::Identifiable::RuntimeClass & rc = val.getClass();
+        if (rc.inherits(ArrayFieldValue::classId)) {
+            ArrayAccessor<Accessor> ac(static_cast<const ArrayFieldValue & >(val));
+            appendValue(vec, lid, ac);
+        } else if (rc.inherits(WeightedSetFieldValue::classId)) {
+            WeightedSetAccessor<Accessor> ac(static_cast<const WeightedSetFieldValue & >(val));
+            appendValue(vec, lid, ac);
+        } else {
+            LOG(warning, "Unsupported value '%s' to assign on multivalue vector '%s'", rc.name(), vec.getName().c_str());
+        }
+    } else {
+        updateValue(vec, lid, val);
+    }
+}
+
+void
+AttrUpdate::appendValue(IntegerAttribute & vec, uint32_t lid, const FieldValue & val, int weight)
+{
+    int64_t v = val.getAsLong();
+    if (!vec.append(lid, v, weight)) {
+        throw UpdateException(make_string("attribute append failed: %s[%u] = %" PRIu64,
+                                          vec.getName().c_str(), lid, v));
+    }
+}
+
+void
+AttrUpdate::removeValue(IntegerAttribute & vec, uint32_t lid, const FieldValue & val)
+{
+    int64_t v = val.getAsLong();
+    if (!vec.remove(lid, v, 1)) {
+        throw UpdateException(make_string("attribute remove failed: %s[%u] = %" PRIu64,
+                                          vec.getName().c_str(), lid, v));
+    }
+}
+
+void
+AttrUpdate::updateValue(IntegerAttribute & vec, uint32_t lid, const FieldValue & val)
+{
+    int64_t v = val.getAsLong();
+    if (!vec.update(lid, v)) {
+        throw UpdateException(make_string("attribute update failed: %s[%u] = %" PRIu64,
+                                          vec.getName().c_str(), lid, v));
+    }
+}
+
+void
+AttrUpdate::appendValue(FloatingPointAttribute & vec, uint32_t lid, const FieldValue & val, int weight)
+{
+    double v = val.getAsDouble();
+    if (!vec.append(lid, v, weight)) {
+        throw UpdateException(make_string("attribute append failed: %s[%u] = %g",
+                                          vec.getName().c_str(), lid, v));
+    }
+}
+
+template <typename V, typename Accessor>
+void
+AttrUpdate::appendValue(V & vec, uint32_t lid, Accessor & ac)
+{
+    if (!vec.append(lid, ac)) {
+        throw UpdateException(make_string("attribute append failed: %s[%u]",
+                                          vec.getName().c_str(), lid));
+    }
+}
+
+void
+AttrUpdate::removeValue(FloatingPointAttribute & vec, uint32_t lid, const FieldValue & val)
+{
+    double v = val.getAsDouble();
+    if (!vec.remove(lid, v, 1)) {
+        throw UpdateException(make_string("attribute remove failed: %s[%u] = %g",
+                                          vec.getName().c_str(), lid, v));
+    }
+}
+
+void
+AttrUpdate::updateValue(FloatingPointAttribute & vec, uint32_t lid, const FieldValue & val)
+{
+    double v = val.getAsDouble();
+    if (!vec.update(lid, v)) {
+        throw UpdateException(make_string("attribute update failed: %s[%u] = %g",
+                                          vec.getName().c_str(), lid, v));
+    }
+}
+
+namespace {
+
+const vespalib::string &
+getString(const search::StringAttribute & attr, uint32_t lid, const FieldValue & val) {
+    if ( ! val.inherits(LiteralFieldValueB::classId) ) {
+        throw UpdateException(make_string("Can not update a string attribute '%s' for lid=%d from a non-literal fieldvalue: %s",
+                                          attr.getName().c_str(), lid, val.toString().c_str()));
+    }
+    return static_cast<const LiteralFieldValueB &>(val).getValue();
+}
+
+}
+
+void
+AttrUpdate::appendValue(StringAttribute & vec, uint32_t lid, const FieldValue & val, int weight)
+{
+    const vespalib::string & s = getString(vec, lid, val);
+    if (!vec.append(lid, s, weight)) {
+        throw UpdateException(make_string("attribute append failed: %s[%u] = %s",
+                                          vec.getName().c_str(), lid, s.c_str()));
+    }
+}
+
+void
+AttrUpdate::removeValue(StringAttribute & vec, uint32_t lid, const FieldValue & val)
+{
+    const vespalib::string & v = getString(vec, lid, val);
+    if (!vec.remove(lid, v, 1)) {
+        throw UpdateException(make_string("attribute remove failed: %s[%u] = %s",
+                                          vec.getName().c_str(), lid, v.c_str()));
+    }
+}
+
+void
+AttrUpdate::updateValue(StringAttribute & vec, uint32_t lid, const FieldValue & val)
+{
+    const vespalib::string & v = getString(vec, lid, val);
+    if (!vec.update(lid, v)) {
+        throw UpdateException(make_string("attribute update failed: %s[%u] = %s",
+                                          vec.getName().c_str(), lid, v.c_str()));
+    }
+}
+
+void AttrUpdate::updateValue(PredicateAttribute &vec, uint32_t lid, const FieldValue &val)
+{
+    if (!val.inherits(PredicateFieldValue::classId)) {
+        throw UpdateException(
+                make_string("PredicateAttribute must be updated with "
+                            "PredicateFieldValues."));
+    }
+    vec.updateValue(lid, static_cast<const PredicateFieldValue &>(val));
+}
+
+void AttrUpdate::updateValue(TensorAttribute &vec, uint32_t lid, const FieldValue &val)
+{
+    if (!val.inherits(TensorFieldValue::classId)) {
+        throw UpdateException(
+                make_string("TensorAttribute must be updated with "
+                            "TensorFieldValues."));
+    }
+    const auto &tensor = static_cast<const TensorFieldValue &>(val).
+                         getAsTensorPtr();
+    if (tensor) {
+        vec.setTensor(lid, *tensor);
+    } else {
+        vec.clearDoc(lid);
+    }
+}
+
+}  // namespace search
diff --git a/searchcore/src/vespa/searchcore/proton/common/attrupdate.h b/searchcore/src/vespa/searchcore/proton/common/attrupdate.h
new file mode 100644
index 00000000000..ed62c02a551
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/attrupdate.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/searchlib/attribute/attribute.h>
+#include <vespa/searchlib/attribute/predicate_attribute.h>
+#include <vespa/vespalib/util/exception.h>
+
+namespace search {
+
+using document::Field;
+using document::FieldValue;
+using document::FieldUpdate;
+using document::ValueUpdate;
+namespace attribute { class TensorAttribute; }
+
+VESPA_DEFINE_EXCEPTION(UpdateException, vespalib::Exception);
+
+class AttrUpdate {
+
+public:
+    static void handleUpdate(AttributeVector & vec, uint32_t lid, const FieldUpdate & upd);
+    static void handleValue(AttributeVector & vec, uint32_t lid, const FieldValue & val);
+
+private:
+    template <typename V>
+    static void handleUpdate(V & vec, uint32_t lid, const ValueUpdate & upd);
+    template <typename V, typename Accessor>
+    static void handleValueT(V & vec, Accessor ac, uint32_t lid, const FieldValue & val);
+    template <typename V, typename Accessor>
+    static void handleUpdateT(V & vec, Accessor ac, uint32_t lid, const ValueUpdate & val);
+    template <typename V, typename Accessor>
+    static void appendValue(V & vec, uint32_t lid, Accessor & ac);
+    static void appendValue(IntegerAttribute & vec, uint32_t lid, const FieldValue & val, int weight=1);
+    static void removeValue(IntegerAttribute & vec, uint32_t lid, const FieldValue & val);
+    static void updateValue(IntegerAttribute & vec, uint32_t lid, const FieldValue & val);
+    static void appendValue(FloatingPointAttribute & vec, uint32_t lid, const FieldValue & val, int weight=1);
+    static void removeValue(FloatingPointAttribute & vec, uint32_t lid, const FieldValue & val);
+    static void updateValue(FloatingPointAttribute & vec, uint32_t lid, const FieldValue & val);
+    static void appendValue(StringAttribute & vec, uint32_t lid, const FieldValue & val, int weight=1);
+    static void removeValue(StringAttribute & vec, uint32_t lid, const FieldValue & val);
+    static void updateValue(StringAttribute & vec, uint32_t lid, const FieldValue & val);
+    static void updateValue(PredicateAttribute & vec, uint32_t lid, const FieldValue & val);
+    static void updateValue(attribute::TensorAttribute & vec, uint32_t lid, const FieldValue & val);
+};
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp b/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp
new file mode 100644
index 00000000000..bdf35a09038
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.bucketfactory");
+
+#include "bucketfactory.h"
+#include <persistence/spi/types.h>
+
+using document::BucketId;
+using document::DocumentId;
+using storage::spi::Bucket;
+using storage::spi::PartitionId;
+
+namespace proton {
+
+BucketId
+BucketFactory::getBucketId(const DocumentId &docId)
+{
+    BucketId bId = docId.getGlobalId().convertToBucketId();
+    bId.setUsedBits(getNumBucketBits());
+    return bId;
+}
+
+
+Bucket
+BucketFactory::getBucket(const DocumentId &docId)
+{
+    return Bucket(getBucketId(docId), PartitionId(0));
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/bucketfactory.h b/searchcore/src/vespa/searchcore/proton/common/bucketfactory.h
new file mode 100644
index 00000000000..2a41319649d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/bucketfactory.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/persistence/spi/bucket.h>
+
+namespace proton {
+
+/**
+ * Helper class for creating bucket ids in order to support
+ * persistence provider spi when getting feed from message bus.
+ */
+class BucketFactory {
+public:
+    static uint32_t getNumBucketBits() { return 8u; }
+    static document::BucketId getBucketId(const document::DocumentId &docId);
+    static storage::spi::Bucket getBucket(const document::DocumentId &docId);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp
new file mode 100644
index 00000000000..52da7aaea2a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp
@@ -0,0 +1,211 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.cachedselect");
+#include "cachedselect.h"
+#include <vespa/document/select/valuenode.h>
+#include <vespa/document/select/cloningvisitor.h>
+#include "attributefieldvaluenode.h"
+#include <vespa/searchlib/attribute/iattributemanager.h>
+#include <vespa/document/select/parser.h>
+#include "selectpruner.h"
+
+namespace proton
+{
+
+using search::index::Schema;
+using search::AttributeVector;
+using search::AttributeGuard;
+using document::select::FieldValueNode;
+using search::attribute::CollectionType;
+
+class AttrVisitor : public document::select::CloningVisitor
+{
+public:
+    typedef std::map<vespalib::string, uint32_t> AttrMap;
+
+    AttrMap _amap;
+    const search::IAttributeManager &_amgr;
+    CachedSelect &_cached;
+    const Schema &_schema;
+    uint32_t _svAttrs;
+    uint32_t _mvAttrs;
+    uint32_t _complexAttrs;
+
+    uint32_t
+    getFieldNodes(void) const
+    {
+        return _fieldNodes;
+    }
+
+    static uint32_t
+    invalidIdx(void)
+    {
+        return std::numeric_limits<uint32_t>::max();
+    }
+    
+    AttrVisitor(const Schema &schema,
+                const search::IAttributeManager &amgr,
+                CachedSelect &cachedSelect);
+
+    /*
+     * Mutate field value nodes representing single value attributes into
+     * attribute field valulue nodes.
+     */
+    virtual void
+    visitFieldValueNode(const FieldValueNode &expr);
+};
+
+
+AttrVisitor::AttrVisitor(const Schema &schema,
+                         const search::IAttributeManager &amgr,
+                         CachedSelect &cachedSelect)
+    : CloningVisitor(),
+      _amap(),
+      _amgr(amgr),
+      _cached(cachedSelect),
+      _schema(schema),
+      _svAttrs(0u),
+      _mvAttrs(0u),
+      _complexAttrs(0u)
+{
+}
+
+
+void
+AttrVisitor::visitFieldValueNode(const FieldValueNode &expr)
+{
+    ++_fieldNodes;
+    // Expression has survived select pruner, thus we know that field is
+    // valid for document type.
+    vespalib::string name(expr.getFieldName()); 
+    bool complex = false;
+    for (uint32_t i = 0; i < name.size(); ++i) {
+        if (name[i] == '.' || name[i] == '{' || name[i] == '[') {
+            // TODO: Check for struct, array, map or weigthed set
+            name = expr.getFieldName().substr(0, i);
+            complex = true;
+            break;
+        }
+    }
+    if (_schema.isAttributeField(name)) {
+        if (complex) {
+            ++_complexAttrs;
+            // Don't try to optimize complex attribute references yet.
+            _valueNode = expr.clone();
+            return;
+        }
+        AttributeGuard::UP ag(_amgr.getAttribute(name));
+        assert(ag.get() != NULL);
+        if (!ag->valid()) {
+            // Fast access document sub where attribute is not marked as
+            // fast-access.  Disable optimization.
+            ++_complexAttrs;
+            _valueNode = expr.clone();
+            return;
+        }
+        AttributeVector::SP av(ag->getSP());
+        if (av->getCollectionType() == CollectionType::SINGLE) {
+            ++_svAttrs;
+            AttrMap::iterator it(_amap.find(name));
+            uint32_t idx(invalidIdx());
+            if (it == _amap.end()) {
+                // Allocate new location for guard
+                idx = _cached._attributes.size();
+                _amap[name] = idx;
+                _cached._attributes.push_back(av);
+            } else {
+                // Already allocated location for guard
+                idx = it->second;
+            }
+            assert(idx != invalidIdx());
+            _valueNode.reset(new AttributeFieldValueNode(expr.getDocType(),
+                                                         name,
+                                                         av));
+        } else {
+            // Don't try to optimize multivalue attribute vectors yet
+            ++_mvAttrs;
+            _valueNode = expr.clone();
+        }
+    } else {
+        _valueNode = expr.clone();
+    }
+}
+
+
+CachedSelect::CachedSelect(void)
+    : _attributes(),
+      _select(),
+      _fieldNodes(0u),
+      _attrFieldNodes(0u),
+      _svAttrFieldNodes(0u),
+      _resultSet(),
+      _allFalse(false),
+      _allTrue(false),
+      _allInvalid(false),
+      _attrSelect()
+{
+}
+
+
+void
+CachedSelect::set(const vespalib::string &selection,
+                  const document::DocumentTypeRepo &repo)
+{
+    try {
+        document::select::Parser parser(repo,
+                                        document::BucketIdFactory());
+        _select = parser.parse(selection);
+    } catch (document::select::ParsingFailedException &) {
+        _select.reset(NULL);
+    }
+    _allFalse = _select.get() == NULL;
+    _allTrue = false;
+    _allInvalid = false;
+}
+
+                  
+void
+CachedSelect::set(const vespalib::string &selection,
+                  const vespalib::string &docTypeName,
+                  const document::Document &emptyDoc,
+                  const document::DocumentTypeRepo &repo,
+                  const search::index::Schema &schema,
+                  const search::IAttributeManager *amgr,
+                  bool hasFields)
+{                  
+    typedef std::unique_ptr<document::select::Node> NodeUP;
+
+    set(selection, repo);
+    NodeUP parsed(std::move(_select));
+    if (parsed.get() == NULL)
+        return;
+    SelectPruner pruner(docTypeName,
+                        schema,
+                        emptyDoc,
+                        repo,
+                        hasFields);
+    pruner.process(*parsed);
+    _resultSet = pruner.getResultSet();
+    _allFalse = pruner.isFalse();
+    _allTrue = pruner.isTrue();
+    _allInvalid = pruner.isInvalid();
+    _select = std::move(pruner.getNode());
+    _fieldNodes = pruner.getFieldNodes();
+    _attrFieldNodes = pruner.getAttrFieldNodes();
+    if (amgr == NULL || _attrFieldNodes == 0u)
+        return;
+    AttrVisitor av(schema, *amgr, *this);
+    _select->visit(av);
+    assert(_fieldNodes == av.getFieldNodes());
+    assert(_attrFieldNodes == av._mvAttrs + av._svAttrs + av._complexAttrs);
+    _svAttrFieldNodes = av._svAttrs;
+    if (_fieldNodes == _svAttrFieldNodes) {
+        _attrSelect = std::move(av.getNode());
+    }
+}
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.h b/searchcore/src/vespa/searchcore/proton/common/cachedselect.h
new file mode 100644
index 00000000000..780d0e31f68
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.h
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/select/node.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+#include <vespa/document/select/resultset.h>
+
+namespace proton
+{
+
+class AttributeManager;
+
+/**
+ * Cached selection expression, to avoid pruning expression for each
+ * new bucket.
+ */
+class CachedSelect
+{
+public:
+    typedef std::shared_ptr<CachedSelect> SP;
+    // Single value attributes referenced from selection expression
+    std::vector<search::AttributeVector::SP> _attributes;
+
+    // Pruned selection expression, specific for a document type
+    std::unique_ptr<document::select::Node> _select;
+    uint32_t _fieldNodes;
+    uint32_t _attrFieldNodes;
+    uint32_t _svAttrFieldNodes;
+    document::select::ResultSet _resultSet;
+    bool _allFalse;
+    bool _allTrue;
+    bool _allInvalid;
+
+    /*
+     * If expression doesn't reference multi value attributes or
+     * non-attribute fields then this selection expression can be used
+     * without retrieving document from document meta store (must use
+     * SelectContext class and populate _docId instead).
+     */
+    std::unique_ptr<document::select::Node> _attrSelect;
+    
+    CachedSelect(void);
+
+    void
+    set(const vespalib::string &selection,
+        const document::DocumentTypeRepo &repo);
+                  
+    void
+    set(const vespalib::string &selection,
+        const vespalib::string &docTypeName,
+        const document::Document &emptyDoc,
+        const document::DocumentTypeRepo &repo,
+        const search::index::Schema &schema,
+        const search::IAttributeManager *amgr,
+        bool hasFields);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.cpp b/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.cpp
new file mode 100644
index 00000000000..6acd99d5285
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.cpp
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "commit_time_tracker.h"
+
+namespace proton {
+
+CommitTimeTracker::CommitTimeTracker(fastos::TimeStamp visibilityDelay)
+    : _visibilityDelay(visibilityDelay),
+      _nextCommit(fastos::ClockSystem::now()),
+      _replayDone(false)
+{
+    _nextCommit += visibilityDelay;
+}
+
+bool
+CommitTimeTracker::needCommit() const
+{
+    if (_visibilityDelay > 0) {
+        if (_replayDone) {
+            return false; // maintenance job will do forced commits now
+        }
+        fastos::TimeStamp now(fastos::ClockSystem::now());
+        if (now >= _nextCommit) {
+            _nextCommit = now + _visibilityDelay;
+            return true;
+        }
+        return false;
+    }
+    return true;
+}
+
+
+
+void
+CommitTimeTracker::setVisibilityDelay(fastos::TimeStamp visibilityDelay)
+{
+    fastos::TimeStamp now(fastos::ClockSystem::now());
+    fastos::TimeStamp nextCommit = now + visibilityDelay;
+    if (nextCommit.val() < _nextCommit.val()) {
+        _nextCommit = nextCommit;
+    }
+    _visibilityDelay = visibilityDelay;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.h b/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.h
new file mode 100644
index 00000000000..ceb8c95cb1e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/fastos/timestamp.h>
+
+namespace proton {
+
+/**
+ * Class used to track when commit is needed based on wanted visibility delay.
+ */
+class CommitTimeTracker
+{
+private:
+    fastos::TimeStamp           _visibilityDelay;
+    mutable fastos::TimeStamp   _nextCommit;
+    bool                        _replayDone;
+
+public:
+    CommitTimeTracker(fastos::TimeStamp visibilityDelay);
+
+    bool needCommit() const;
+
+    void setVisibilityDelay(fastos::TimeStamp visibilityDelay);
+
+    bool hasVisibilityDelay() const { return _visibilityDelay != 0; }
+
+    void setReplayDone() { _replayDone = true; }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.cpp b/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.cpp
new file mode 100644
index 00000000000..ae9911289db
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.cpp
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.dbdocumentid");
+
+#include "dbdocumentid.h"
+
+namespace proton {
+
+DbDocumentId::DbDocumentId()
+    : _subDbId(0),
+      _lid(0)
+{
+}
+
+
+DbDocumentId::DbDocumentId(search::DocumentIdT lid)
+    : _subDbId(0),
+      _lid(lid)
+{
+}
+
+
+DbDocumentId::DbDocumentId(uint32_t subDbId, search::DocumentIdT lid)
+    : _subDbId(subDbId),
+      _lid(lid)
+{
+}
+
+
+vespalib::nbostream &
+operator<<(vespalib::nbostream &os, const DbDocumentId &dbdId)
+{
+    os << dbdId._subDbId;
+    os << dbdId._lid;
+    return os;
+}
+
+
+vespalib::nbostream &
+operator>>(vespalib::nbostream &is, DbDocumentId &dbdId)
+{
+    is >> dbdId._subDbId;
+    is >> dbdId._lid;
+    return is;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.h b/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.h
new file mode 100644
index 00000000000..768c944c9bc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.h
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/query/base.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+namespace proton {
+
+/**
+ * Class used to localize a local document id inside a sub document db.
+ */
+class DbDocumentId {
+private:
+    uint32_t            _subDbId; // sub document db id
+    search::DocumentIdT _lid;     // local document id
+public:
+    DbDocumentId();
+    DbDocumentId(search::DocumentIdT lid);
+    DbDocumentId(uint32_t subDbId, search::DocumentIdT lid);
+    uint32_t getSubDbId() const { return _subDbId; }
+    search::DocumentIdT getLid() const { return _lid; }
+    bool valid() const { return _lid != 0; }
+
+    bool
+    operator!=(const DbDocumentId &rhs) const
+    {
+        return _subDbId != rhs._subDbId ||
+                   _lid != rhs._lid;
+    }
+
+    bool
+    operator==(const DbDocumentId &rhs) const
+    {
+        return _subDbId == rhs._subDbId &&
+                   _lid == rhs._lid;
+    }
+
+    vespalib::string toString() const {
+        return vespalib::make_string("subDbId=%u, lid=%u", _subDbId, _lid);
+    }
+
+    friend vespalib::nbostream &
+    operator<<(vespalib::nbostream &os, const DbDocumentId &dbdId);
+
+    friend vespalib::nbostream &
+    operator>>(vespalib::nbostream &is, DbDocumentId &dbdId);
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/docid_limit.h b/searchcore/src/vespa/searchcore/proton/common/docid_limit.h
new file mode 100644
index 00000000000..8862ad80a4d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/docid_limit.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <atomic>
+
+namespace proton {
+
+/**
+ * Class representing the end of a local document id range.
+ */
+class DocIdLimit
+{
+private:
+    std::atomic<uint32_t> _docIdLimit;
+
+public:
+    explicit DocIdLimit(uint32_t docIdLimit) : _docIdLimit(docIdLimit) {}
+    void set(uint32_t docIdLimit) { _docIdLimit = docIdLimit; }
+    uint32_t get() const { return _docIdLimit; }
+
+    void bumpUpLimit(uint32_t newLimit) {
+        for (;;) {
+            uint32_t oldLimit = _docIdLimit;
+            if (newLimit <= oldLimit)
+                break;
+            if (_docIdLimit.compare_exchange_weak(oldLimit, newLimit,
+                                                  std::memory_order_release,
+                                                  std::memory_order_relaxed))
+                break;
+        }
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp
new file mode 100644
index 00000000000..96a6bc20ecc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.doctypename");
+
+#include "doctypename.h"
+#include <vespa/searchlib/engine/request.h>
+#include <vector>
+
+namespace proton
+{
+
+
+DocTypeName::DocTypeName(const search::engine::Request &request)
+    : _name()
+{
+    _name =
+        request.propertiesMap.matchProperties().lookup("documentdb",
+                "searchdoctype").get("");
+}
+
+
+DocTypeName::DocTypeName(const document::DocumentType &docType)
+    : _name(docType.getName())
+{
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/doctypename.h b/searchcore/src/vespa/searchcore/proton/common/doctypename.h
new file mode 100644
index 00000000000..ee462ac0514
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/doctypename.h
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+namespace search
+{
+
+namespace engine
+{
+
+class Request;
+
+}
+
+}
+
+namespace proton
+{
+
+class DocTypeName
+{
+    vespalib::string _name;
+
+public:
+    explicit
+    DocTypeName(const vespalib::string &name)
+        : _name(name)
+    {
+    }
+
+    explicit
+    DocTypeName(const search::engine::Request &request);
+
+    explicit
+    DocTypeName(const document::DocumentType &docType);
+
+    const vespalib::string &
+    getName(void) const
+    {
+        return _name;
+    }
+
+    bool
+    operator<(const DocTypeName &rhs) const
+    {
+        return _name < rhs._name;
+    }
+
+    vespalib::string
+    toString() const
+    {
+        return _name;
+    }
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp
new file mode 100644
index 00000000000..13b8ec3a56c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.document_type_inspector");
+
+#include "document_type_inspector.h"
+
+namespace proton {
+
+DocumentTypeInspector::DocumentTypeInspector(const document::DocumentType &docType)
+    : _docType(docType)
+{
+}
+
+bool
+DocumentTypeInspector::hasField(const vespalib::string &name) const
+{
+    return _docType.hasField(name);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.h b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.h
new file mode 100644
index 00000000000..9e3455cd6f9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_document_type_inspector.h"
+#include <vespa/document/datatype/documenttype.h>
+
+namespace proton {
+
+/**
+ * Inspector of a concrete document type.
+ */
+class DocumentTypeInspector : public IDocumentTypeInspector
+{
+private:
+    const document::DocumentType &_docType;
+
+public:
+    DocumentTypeInspector(const document::DocumentType &docType);
+
+    virtual bool hasField(const vespalib::string &name) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp
new file mode 100644
index 00000000000..cb3963b4790
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp
@@ -0,0 +1,324 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.eventlogger");
+
+#include "eventlogger.h"
+#include <vespa/searchlib/util/logutil.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using search::util::LogUtil;
+using vespalib::JSONStringer;
+
+namespace {
+
+using search::SerialNum;
+using vespalib::string;
+
+void
+doTransactionLogReplayStart(const string &domainName,
+                            SerialNum first,
+                            SerialNum last,
+                            const string &eventName)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("domain").appendString(domainName);
+    jstr.appendKey("serialnum")
+        .beginObject()
+        .appendKey("first").appendInt64(first)
+        .appendKey("last").appendInt64(last)
+        .endObject();
+    jstr.endObject();
+    EV_STATE(eventName.c_str(), jstr.toString().c_str());
+}
+
+void
+doTransactionLogReplayComplete(const string &domainName,
+                               int64_t elapsedTimeMs,
+                               const string &eventName)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("domain").appendString(domainName);
+    jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
+    jstr.endObject();
+    EV_STATE(eventName.c_str(), jstr.toString().c_str());
+}
+
+}
+
+namespace proton {
+
+void
+EventLogger::transactionLogReplayStart(const string &domainName,
+                                       SerialNum first,
+                                       SerialNum last)
+{
+    doTransactionLogReplayStart(domainName,
+                                first,
+                                last,
+                                "transactionlog.replay.start");
+}
+
+void
+EventLogger::transactionLogReplayProgress(const string &domainName,
+                                          float progress,
+                                          SerialNum first,
+                                          SerialNum last,
+                                          SerialNum current)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("domain").appendString(domainName);
+    jstr.appendKey("progress").appendFloat(progress);
+    jstr.appendKey("serialnum")
+        .beginObject()
+        .appendKey("first").appendInt64(first)
+        .appendKey("last").appendInt64(last)
+        .appendKey("current").appendInt64(current)
+        .endObject();
+    jstr.endObject();
+    EV_STATE("transactionlog.replay.progress", jstr.toString().c_str());
+}
+
+void
+EventLogger::transactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs)
+{
+    doTransactionLogReplayComplete(domainName,
+                                   elapsedTimeMs,
+                                   "transactionlog.replay.complete");
+}
+
+void
+EventLogger::flushInit(const string &name)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("name").appendString(name);
+    jstr.endObject();
+    EV_STATE("flush.init", jstr.toString().c_str());
+}
+
+void
+EventLogger::flushStart(const string &name,
+                        int64_t beforeMemory,
+                        int64_t afterMemory,
+                        int64_t toFreeMemory,
+                        SerialNum unflushed,
+                        SerialNum current)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("name").appendString(name);
+    jstr.appendKey("memory")
+        .beginObject()
+        .appendKey("before").appendInt64(beforeMemory)
+        .appendKey("after").appendInt64(afterMemory)
+        .appendKey("tofree").appendInt64(toFreeMemory)
+        .endObject();
+    jstr.appendKey("serialnum")
+        .beginObject()
+        .appendKey("unflushed").appendInt64(unflushed)
+        .appendKey("current").appendInt64(current)
+        .endObject();
+    jstr.endObject();
+    EV_STATE("flush.start", jstr.toString().c_str());
+}
+
+void
+EventLogger::flushComplete(const string &name,
+                           int64_t elapsedTimeMs,
+                           const string &outputPath,
+                           size_t outputPathElems)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("name").appendString(name);
+    jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
+    jstr.appendKey("output");
+    LogUtil::logDir(jstr, outputPath, outputPathElems);
+    jstr.endObject();
+    EV_STATE("flush.complete", jstr.toString().c_str());
+}
+
+namespace {
+
+void
+addNames(JSONStringer &jstr, const std::vector<string> &names)
+{
+    jstr.appendKey("name");
+    jstr.beginArray();
+    for (auto name : names) {
+        jstr.appendString(name);
+    }
+    jstr.endArray();
+}
+
+}
+
+void
+EventLogger::populateAttributeStart(const std::vector<string> &names)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    addNames(jstr, names);
+    jstr.endObject();
+    EV_STATE("populate.attribute.start", jstr.toString().c_str());
+}
+
+void
+EventLogger::populateAttributeComplete(const std::vector<string> &names, int64_t documentsPopulated)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    addNames(jstr, names);
+    jstr.appendKey("documents.populated").appendInt64(documentsPopulated);
+    jstr.endObject();
+    EV_STATE("populate.attribute.complete", jstr.toString().c_str());
+}
+
+void
+EventLogger::populateDocumentFieldStart(const string &fieldName)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("name").appendString(fieldName);
+    jstr.endObject();
+    EV_STATE("populate.documentfield.start", jstr.toString().c_str());
+}
+
+void
+EventLogger::populateDocumentFieldComplete(const string &fieldName, int64_t documentsPopulated)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("name").appendString(fieldName);
+    jstr.appendKey("documents.populated").appendInt64(documentsPopulated);
+    jstr.endObject();
+    EV_STATE("populate.documentfield.complete", jstr.toString().c_str());
+}
+
+void
+EventLogger::lidSpaceCompactionComplete(const string &subDbName, uint32_t lidLimit)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("documentsubdb").appendString(subDbName);
+    jstr.appendKey("lidlimit").appendInt64(lidLimit);
+    jstr.endObject();
+    EV_STATE("lidspace.compaction.complete", jstr.toString().c_str());
+}
+
+
+void
+EventLogger::reprocessDocumentsStart(const string &subDb, double visitCost)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("documentsubdb").appendString(subDb);
+    jstr.appendKey("visitcost").appendDouble(visitCost);
+    jstr.endObject();
+    EV_STATE("reprocess.documents.start", jstr.toString().c_str());
+}
+    
+
+void
+EventLogger::reprocessDocumentsProgress(const string &subDb, double progress, double visitCost)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("documentsubdb").appendString(subDb);
+    jstr.appendKey("progress").appendDouble(progress);
+    jstr.appendKey("visitcost").appendDouble(visitCost);
+    jstr.endObject();
+    EV_STATE("reprocess.documents.progress", jstr.toString().c_str());
+}
+    
+
+void
+EventLogger::reprocessDocumentsComplete(const string &subDb, double visitCost, int64_t elapsedTimeMs)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("documentsubdb").appendString(subDb);
+    jstr.appendKey("visitcost").appendDouble(visitCost);
+    jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
+    jstr.endObject();
+    EV_STATE("reprocess.documents.complete", jstr.toString().c_str());
+}
+
+void
+EventLogger::loadAttributeStart(const vespalib::string &subDbName, const vespalib::string &attrName)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("documentsubdb").appendString(subDbName);
+    jstr.appendKey("name").appendString(attrName);
+    jstr.endObject();
+    EV_STATE("load.attribute.start", jstr.toString().c_str());
+}
+
+void
+EventLogger::loadAttributeComplete(const vespalib::string &subDbName,
+                                   const vespalib::string &attrName, int64_t elapsedTimeMs)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("documentsubdb").appendString(subDbName);
+    jstr.appendKey("name").appendString(attrName);
+    jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
+    jstr.endObject();
+    EV_STATE("load.attribute.complete", jstr.toString().c_str());
+}
+
+namespace {
+
+void
+loadComponentStart(const vespalib::string &subDbName, const vespalib::string &componentName)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("documentsubdb").appendString(subDbName);
+    jstr.endObject();
+    EV_STATE(vespalib::make_string("load.%s.start", componentName.c_str()).c_str(), jstr.toString().c_str());
+}
+
+void
+loadComponentComplete(const vespalib::string &subDbName, const vespalib::string &componentName, int64_t elapsedTimeMs)
+{
+    JSONStringer jstr;
+    jstr.beginObject();
+    jstr.appendKey("documentsubdb").appendString(subDbName);
+    jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
+    jstr.endObject();
+    EV_STATE(vespalib::make_string("load.%s.complete", componentName.c_str()).c_str(), jstr.toString().c_str());
+}
+
+}
+
+void
+EventLogger::loadDocumentMetaStoreStart(const vespalib::string &subDbName)
+{
+    loadComponentStart(subDbName, "documentmetastore");
+}
+
+void
+EventLogger::loadDocumentMetaStoreComplete(const vespalib::string &subDbName, int64_t elapsedTimeMs)
+{
+    loadComponentComplete(subDbName, "documentmetastore", elapsedTimeMs);
+}
+
+void
+EventLogger::loadDocumentStoreStart(const vespalib::string &subDbName)
+{
+    loadComponentStart(subDbName, "documentstore");
+}
+
+void
+EventLogger::loadDocumentStoreComplete(const vespalib::string &subDbName, int64_t elapsedTimeMs)
+{
+    loadComponentComplete(subDbName, "documentstore", elapsedTimeMs);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/eventlogger.h b/searchcore/src/vespa/searchcore/proton/common/eventlogger.h
new file mode 100644
index 00000000000..e8c828c5a59
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/eventlogger.h
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vector>
+
+namespace proton {
+
+/**
+ * Class used to log various events.
+ **/
+class EventLogger {
+private:
+    typedef search::SerialNum SerialNum;
+    typedef vespalib::string string;
+public:
+    static void transactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs);
+    static void populateAttributeStart(const std::vector<string> &names);
+    static void populateAttributeComplete(const std::vector<string> &names, int64_t documentsVisisted);
+    static void populateDocumentFieldStart(const string &fieldName);
+    static void populateDocumentFieldComplete(const string &fieldName, int64_t documentsVisisted);
+    static void lidSpaceCompactionComplete(const string &subDbName, uint32_t lidLimit);
+    static void reprocessDocumentsStart(const string &subDb, double visitCost);
+    static void reprocessDocumentsProgress(const string &subDb, double progress, double visitCost);
+    static void reprocessDocumentsComplete(const string &subDb, double visitCost, int64_t elapsedTimeMs);
+    static void transactionLogReplayStart(const string &domainName,
+                                          SerialNum first,
+                                          SerialNum last);
+    static void transactionLogReplayProgress(const string &domainName,
+                                             float progress,
+                                             SerialNum first,
+                                             SerialNum last,
+                                             SerialNum current);
+    static void flushInit(const string &name);
+    static void flushStart(const string &name,
+                           int64_t beforeMemory,
+                           int64_t afterMemory,
+                           int64_t toFreeMemory,
+                           SerialNum unflushed,
+                           SerialNum current);
+    static void flushComplete(const string &name,
+                              int64_t elapsedTimeMs,
+                              const string &outputPath,
+                              size_t outputPathElems);
+    static void loadAttributeStart(const vespalib::string &subDbName, const vespalib::string &attrName);
+    static void loadAttributeComplete(const vespalib::string &subDbName,
+                                      const vespalib::string &attrName, int64_t elapsedTimeMs);
+    static void loadDocumentMetaStoreStart(const vespalib::string &subDbName);
+    static void loadDocumentMetaStoreComplete(const vespalib::string &subDbName, int64_t elapsedTimeMs);
+    static void loadDocumentStoreStart(const vespalib::string &subDbName);
+    static void loadDocumentStoreComplete(const vespalib::string &subDbName, int64_t elapsedTimeMs);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp b/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp
new file mode 100644
index 00000000000..c01fdda78db
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "feeddebugger.h"
+#include <vespa/vespalib/text/stringtokenizer.h>
+
+namespace proton
+{
+
+namespace {
+
+void setupDebugging(std::vector<uint32_t> & debugLidList)
+{
+    vespalib::string lidList = getenv("VESPA_PROTON_DEBUG_FEED_LID_LIST");
+    vespalib::StringTokenizer lidTokenizer(lidList);
+    for (size_t i(0); i < lidTokenizer.size(); i++) {
+        vespalib::asciistream is(lidTokenizer[i]);
+        uint32_t lid(0);
+        is >> lid;
+        debugLidList.push_back(lid);
+    }
+}
+
+void setupDebugging(std::vector<document::DocumentId> & debugLidList)
+{
+    vespalib::string lidList = getenv("VESPA_PROTON_DEBUG_FEED_DOCID_LIST");
+    vespalib::StringTokenizer lidTokenizer(lidList);
+    for (size_t i(0); i < lidTokenizer.size(); i++) {
+        debugLidList.push_back(document::DocumentId(lidTokenizer[i]));
+    }
+}
+
+}
+
+FeedDebugger::FeedDebugger() :
+    _enableDebugging(false),
+    _debugLidList(),
+    _debugDocIdList()
+{
+    setupDebugging(_debugLidList);
+    setupDebugging(_debugDocIdList);
+    _enableDebugging = ! (_debugLidList.empty() && _debugDocIdList.empty());
+}
+
+ns_log::Logger::LogLevel
+FeedDebugger::getDebugDebuggerInternal(uint32_t lid, const document::DocumentId * docid) const
+{
+    for(size_t i(0), m(_debugLidList.size()); i < m; i++) {
+        if (lid == _debugLidList[i]) {
+            return ns_log::Logger::info;
+        }
+    }
+    if (docid != NULL) {
+        for(size_t i(0), m(_debugDocIdList.size()); i < m; i++) {
+            if (*docid == _debugDocIdList[i]) {
+                return ns_log::Logger::info;
+            }
+        }
+    }
+    return ns_log::Logger::spam;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/feeddebugger.h b/searchcore/src/vespa/searchcore/proton/common/feeddebugger.h
new file mode 100644
index 00000000000..f0d8062f0b7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/feeddebugger.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/log/log.h>
+#include <vespa/document/base/documentid.h>
+
+namespace proton {
+
+class FeedDebugger
+{
+public:
+    FeedDebugger();
+    bool isDebugging() const { return _enableDebugging; }
+    ns_log::Logger::LogLevel getDebugLevel(uint32_t lid, const document::DocumentId & docid) const {
+        return getDebugLevel(lid, & docid);
+    }
+    ns_log::Logger::LogLevel getDebugLevel(uint32_t lid, const document::DocumentId * docid) const {
+        if (isDebugging()) {
+            return getDebugDebuggerInternal(lid, docid);
+        }
+        return ns_log::Logger::spam;
+    }
+private:
+    ns_log::Logger::LogLevel getDebugDebuggerInternal(uint32_t lid, const document::DocumentId * docid) const;
+    bool                              _enableDebugging;
+    std::vector<uint32_t>             _debugLidList; // List of lids to dump when feeding/replaying log.
+    std::vector<document::DocumentId> _debugDocIdList; // List of docids("doc:bla:blu" to dump when feeding/replaying log.
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp b/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp
new file mode 100644
index 00000000000..f1aefced17a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp
@@ -0,0 +1,105 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.feedtoken");
+
+#include "feedtoken.h"
+#include <vespa/vespalib/util/atomic.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+
+
+namespace proton {
+
+FeedToken::FeedToken(ITransport &transport, mbus::Reply::UP reply) :
+    _state(new State(transport, std::move(reply), 1))
+{
+}
+
+FeedToken::State::State(ITransport & transport, mbus::Reply::UP reply, uint32_t numAcksRequired) :
+    _transport(transport),
+    _reply(std::move(reply)),
+    _result(new storage::spi::Result()),
+    _documentWasFound(false),
+    _unAckedCount(numAcksRequired),
+    _lock(),
+    _startTime()
+{
+    assert(_reply.get() != NULL);
+    _startTime.SetNow();
+}
+
+FeedToken::State::~State()
+{
+    assert(_reply.get() == NULL);
+}
+
+void
+FeedToken::State::ack()
+{
+    assert(_reply.get() != NULL);
+    uint32_t prev(_unAckedCount--);
+    if (prev == 1) {
+        _transport.send(std::move(_reply), std::move(_result), _documentWasFound, _startTime.MilliSecsToNow());
+    }
+    assert(prev >= 1);
+}
+
+
+void
+FeedToken::State::ack(const FeedOperation::Type opType,
+                      PerDocTypeFeedMetrics &metrics)
+{
+    assert(_reply.get() != NULL);
+    uint32_t prev(_unAckedCount--);
+    if (prev == 1) {
+        _transport.send(std::move(_reply), std::move(_result), _documentWasFound, _startTime.MilliSecsToNow());
+        switch (opType) {
+        case FeedOperation::PUT:
+            metrics.RegisterPut(_startTime);
+            break;
+        case FeedOperation::REMOVE:
+        case FeedOperation::REMOVE_BATCH:
+            metrics.RegisterRemove(_startTime);
+            break;
+        case FeedOperation::UPDATE:
+            metrics.RegisterUpdate(_startTime);
+            break;
+        case FeedOperation::MOVE:
+            metrics.RegisterMove(_startTime);
+            break;
+        default:
+            ;
+        }
+    }
+    assert(prev >= 1);
+}
+
+
+void
+FeedToken::State::incNeededAcks(void)
+{
+    assert(_reply.get() != NULL);
+    uint32_t prev(_unAckedCount++);
+    assert(prev >= 1);
+    (void) prev;
+}
+
+
+void
+FeedToken::State::fail(uint32_t errNum, const vespalib::string &errMsg)
+{
+    assert(_reply.get() != NULL);
+    vespalib::LockGuard guard(_lock);
+    _reply->addError(mbus::Error(errNum, errMsg));
+    _transport.send(std::move(_reply), std::move(_result), _documentWasFound, _startTime.MilliSecsToNow());
+}
+
+void
+FeedToken::State::trace(uint32_t traceLevel, const vespalib::string &traceMsg)
+{
+    assert(_reply.get() != NULL);
+    vespalib::LockGuard guard(_lock);
+    _reply->getTrace().trace(traceLevel, traceMsg);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h
new file mode 100644
index 00000000000..eb932c22fe6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h
@@ -0,0 +1,163 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/messagebus/reply.h>
+#include <vespa/persistence/spi/persistenceprovider.h>
+#include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/searchcore/proton/feedoperation/feedoperation.h>
+#include <atomic>
+
+namespace proton
+{
+
+
+class PerDocTypeFeedMetrics;
+typedef std::unique_ptr<storage::spi::Result> ResultUP;
+
+/**
+ * This class is used by the FeedEngine to encapsulate the necessary information
+ * for an IFeedHandler to perform an async reply to an operation. A unique
+ * instance of this class is passed to every invokation of the IFeedHandler.
+ */
+class FeedToken {
+public:
+    class ITransport {
+    public:
+        virtual ~ITransport() { }
+        virtual void send(mbus::Reply::UP reply,
+                          ResultUP result,
+                          bool documentWasFound,
+                          double latency_ms) = 0;
+    };
+
+private:
+    class State : public boost::noncopyable {
+    public:
+        State(ITransport & transport, mbus::Reply::UP reply, uint32_t numAcksRequired);
+        ~State();
+        void setNumAcksRequired(uint32_t numAcksRequired) { _unAckedCount = numAcksRequired; }
+        void ack();
+
+        void
+        ack(const FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics);
+
+        void
+        incNeededAcks(void);
+
+        void fail(uint32_t errNum, const vespalib::string &errMsg);
+        void trace(uint32_t traceLevel, const vespalib::string &traceMsg);
+        bool shouldTrace(uint32_t traceLevel) const { return _reply->getTrace().shouldTrace(traceLevel); }
+        mbus::Reply & getReply() { return *_reply; }
+        void setResult(ResultUP result, bool documentWasFound) {
+            _documentWasFound = documentWasFound;
+            _result = std::move(result);
+        }
+        const storage::spi::Result &getResult() { return *_result; }
+        FastOS_Time getStartTime() const { return _startTime; }
+    private:
+        ITransport       &_transport;
+        mbus::Reply::UP   _reply;
+        ResultUP          _result;
+        bool              _documentWasFound;
+        std::atomic<uint32_t> _unAckedCount;
+        vespalib::Lock    _lock;
+        FastOS_Time       _startTime;
+    };
+    std::shared_ptr<State> _state;
+
+public:
+    typedef std::unique_ptr<FeedToken> UP;
+    typedef std::shared_ptr<FeedToken> SP;
+
+    /**
+     * Constructs a unique FeedToken. This is the constructor used by the
+     * FeedEngine when processing input. If the given message is empty, or it
+     * does not belong to the document protocol, this method throws a
+     * vespalib::IllegalArgumentException.
+     *
+     * @param transport The transport to pass the reply to.
+     * @param reply     The mbus::Reply corresponding to this operation.
+     */
+    FeedToken(ITransport &transport, mbus::Reply::UP reply);
+
+    /**
+     * Passes a receipt back to the originating FeedEngine, declaring that this
+     * operation succeeded. If an error occured while processing the operation,
+     * use fail() instead. Invoking this and/or fail() more than once is void.
+     */
+    void ack() const { _state->ack(); }
+
+    void
+    ack(const FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics) const
+    {
+        _state->ack(opType, metrics);
+    }
+
+    void
+    incNeededAcks(void) const
+    {
+        _state->incNeededAcks();
+    }
+
+    /**
+     * Passes a receipt back to the originating FeedEngine, declaring that this
+     * operation failed for some reason. Invoking this and/or fail() more than
+     * once is void.
+     *
+     * @param errNum A numerical representation of the error.
+     * @param errMsg A readable string detailing the error.
+     */
+    void fail(uint32_t errNum, const vespalib::string &errMsg) const { _state->fail(errNum, errMsg); }
+
+    /**
+     * Writes a trace message to the receipt of this operation that will later
+     * be passed back to the FeedEngine through ack() or fail().
+     *
+     * @param traceLevel The level of the message to write.
+     * @param traceMsg   The message to write.
+     */
+    void trace(uint32_t traceLevel, const vespalib::string &traceMsg) const { _state->trace(traceLevel, traceMsg); }
+
+    /**
+     * Tell you if tracing at this level is enabled
+     *
+     * @param traceLevel The level you want to trace at.
+     * @return if you should trace or not.
+     */
+    bool shouldTrace(uint32_t traceLevel) const { return _state->shouldTrace(traceLevel); }
+
+    /**
+     * Gives you access to the underlying reply message.
+     *
+     * @return The reply
+     */
+    mbus::Reply & getReply() const { return _state->getReply(); }
+
+    /**
+     * Gives you access to the underlying result.
+     *
+     * @return The result
+     */
+    const storage::spi::Result &getResult() const { return _state->getResult(); }
+
+    /**
+     * Sets the underlying result.
+     */
+    void setResult(ResultUP result, bool documentWasFound) {
+        _state->setResult(std::move(result), documentWasFound);
+    }
+
+    /**
+     * This controls how many acks are required before it is acked back to the sender.
+     * Default is 1, and so far only adjusted by multioperation handling.
+     *
+     * @param numAcksRequired How many acks must be received before it is considered acked.
+     */
+    void setNumAcksRequired(uint32_t numAcksRequired) const { _state->setNumAcksRequired(numAcksRequired); }
+
+    FastOS_Time getStartTime() const { return _state->getStartTime(); }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp
new file mode 100644
index 00000000000..0890730eb61
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp
@@ -0,0 +1,219 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <map>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/sequence.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+
+namespace proton {
+
+/**
+ * This template defines a mapping from a document type name to a shared pointer
+ * to the template argument type.
+ */
+template <typename T>
+class HandlerMap {
+private:
+    typedef typename std::shared_ptr<T>    HandlerSP;
+    typedef std::map<DocTypeName, HandlerSP> StdMap;
+    typedef typename StdMap::iterator        MapIterator;
+
+    StdMap _handlers;
+
+public:
+    /**
+     * A snapshot of the currently registered handlers. This
+     * implementation simply takes a copy of all the shared pointers
+     * in the map to keep the handlers alive. However, the current
+     * abstraction allows as to make a snapshot based on bald pointers
+     * using event barriers to resolve visibility constraints in the
+     * future without changing the external API.
+     **/
+    class Snapshot : public vespalib::Sequence<T*>
+    {
+    private:
+        std::vector<HandlerSP> _handlers;
+        size_t                 _offset;
+
+    public:
+        typedef std::unique_ptr<Snapshot> UP;
+
+        Snapshot(StdMap &map) : _handlers(), _offset(0)
+        {
+            _handlers.reserve(map.size());
+            for (MapIterator pos = map.begin(); pos != map.end(); ++pos) {
+                _handlers.push_back(pos->second);
+            }
+        }
+        virtual bool valid() const { return (_offset < _handlers.size()); }
+        virtual T *get() const { return _handlers[_offset].get(); }
+        HandlerSP getSP() const { return _handlers[_offset]; }
+        virtual void next() { ++_offset; }
+    };
+
+    /**
+     * Convenience typedefs.
+     */
+    typedef typename StdMap::iterator       iterator;
+    typedef typename StdMap::const_iterator const_iterator;
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    HandlerMap()
+        : _handlers()
+    {
+        // empty
+    }
+
+    /**
+     * Registers a new handler for the given document type. If another handler
+     * was already registered under the same type, this method will return a
+     * pointer to that handler.
+     *
+     * @param docType The document type to register a handler for.
+     * @param handler The handler to register.
+     * @return The replaced handler, if any.
+     */
+    HandlerSP
+    putHandler(const DocTypeName &docTypeNameVer,
+               const HandlerSP &handler)
+    {
+        if (handler.get() == NULL) {
+            throw vespalib::IllegalArgumentException(vespalib::make_string(
+                            "Handler is null for docType '%s'",
+                            docTypeNameVer.toString().c_str()));
+        }
+        iterator it = _handlers.find(docTypeNameVer);
+        if (it == _handlers.end()) {
+            _handlers[docTypeNameVer] = handler;
+            return HandlerSP();
+        }
+        HandlerSP ret = it->second;
+        it->second = handler;
+        return ret;
+    }
+
+    /**
+     * Returns the handler for the given document type. If no handler was
+     * registered, this method returns an empty shared pointer.
+     *
+     * @param docType The document type whose handler to return.
+     * @return The registered handler, if any.
+     */
+    HandlerSP
+    getHandler(const DocTypeName &docTypeNameVer) const
+    {
+        const_iterator it = _handlers.find(docTypeNameVer);
+        if (it != _handlers.end()) {
+            return it->second;
+        }
+        return HandlerSP();
+    }
+
+    /**
+     * Removes and returns the handler for the given document type. If no
+     * handler was registered, this method returns an empty shared pointer.
+     *
+     * @param docType The document type whose handler to remove.
+     * @return The removed handler, if any.
+     */
+    HandlerSP
+    removeHandler(const DocTypeName &docTypeNameVer)
+    {
+        iterator it = _handlers.find(docTypeNameVer);
+        if (it == _handlers.end()) {
+            return HandlerSP();
+        }
+        HandlerSP ret = it->second;
+        _handlers.erase(it);
+        return ret;
+    }
+
+    /**
+     * Clear all handlers.
+     */
+    void
+    clear(void)
+    {
+        _handlers.clear();
+    }
+
+    /**
+     * Create a snapshot of the handlers currently contained in this
+     * map and return it as a sequence. The returned sequence will
+     * ensure that all object pointers stay valid until it is deleted.
+     *
+     * @return handler sequence
+     **/
+    std::unique_ptr<Snapshot>
+    snapshot()
+    {
+        return std::unique_ptr<Snapshot>(new Snapshot(_handlers));
+    }
+
+// we want to use snapshots rather than direct iteration to reduce locking;
+// the below functions should be deprecated when possible.
+
+    /**
+     * Returns a bidirectional iterator that points at the first element of the
+     * sequence (or just beyond the end of an empty sequence).
+     *
+     * @return The beginning of this map.
+     */
+    iterator
+    begin()
+    {
+        return _handlers.begin();
+    }
+
+    /**
+     * Returns a const bidirectional iterator that points at the first element
+     * of the sequence (or just beyond the end of an empty sequence).
+     *
+     * @return The beginning of this map.
+     */
+    const_iterator
+    begin() const
+    {
+        return _handlers.begin();
+    }
+
+    /**
+     * Returns a bidirectional iterator that points just beyond the end of the
+     * sequence.
+     *
+     * @return The end of this map.
+     */
+    iterator
+    end()
+    {
+        return _handlers.end();
+    }
+
+    /**
+     * Returns a const bidirectional iterator that points just beyond the end of
+     * the sequence.
+     *
+     * @return The end of this map.
+     */
+    const_iterator
+    end() const
+    {
+        return _handlers.end();
+    }
+
+    /**
+     * Returns the number of handlers in this map.
+     *
+     * @return the number of handlers.
+     */
+    size_t size() const { return _handlers.size(); }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/i_document_type_inspector.h b/searchcore/src/vespa/searchcore/proton/common/i_document_type_inspector.h
new file mode 100644
index 00000000000..8a6ad2755d9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/i_document_type_inspector.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+/**
+ * Interface used to inspect which fields are present in a document type.
+ */
+struct IDocumentTypeInspector
+{
+    typedef std::shared_ptr<IDocumentTypeInspector> SP;
+
+    virtual ~IDocumentTypeInspector() {}
+
+    virtual bool hasField(const vespalib::string &name) const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/indexsearchabletosearchableadapter.h b/searchcore/src/vespa/searchcore/proton/common/indexsearchabletosearchableadapter.h
new file mode 100644
index 00000000000..50fc371887f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/indexsearchabletosearchableadapter.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/index/indexsearchable.h>
+#include <vespa/searchlib/queryeval/searchable.h>
+
+namespace proton {
+
+class IndexSearchableToSearchableAdapter : public search::queryeval::Searchable {
+private:
+    searchcorespi::IndexSearchable::SP          _searchable;
+    const search::attribute::IAttributeContext &_attrCtx;
+
+public:
+    IndexSearchableToSearchableAdapter(const searchcorespi::IndexSearchable::SP &searchable,
+                                       const search::attribute::IAttributeContext &attrCtx)
+        : _searchable(searchable),
+          _attrCtx(attrCtx)
+    {
+    }
+
+    /**
+     * Implements search::queryeval::Searchable.
+     */
+    virtual search::queryeval::Blueprint::UP createBlueprint(const search::queryeval::IRequestContext & requestContext,
+                                                             const search::queryeval::FieldSpec &field,
+                                                             const search::query::Node &term) {
+        return _searchable->createBlueprint(requestContext, field, term, _attrCtx);
+    }
+
+    virtual search::queryeval::Blueprint::UP createBlueprint(const search::queryeval::IRequestContext & requestContext,
+                                                             const search::queryeval::FieldSpecList &fields,
+                                                             const search::query::Node &term) {
+        return _searchable->createBlueprint(requestContext, fields, term, _attrCtx);
+    }
+};
+
+
+} // namespace proton
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp b/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp
new file mode 100644
index 00000000000..4175925357d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp
@@ -0,0 +1,259 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.schemautil");
+
+#include "schemautil.h"
+
+using search::index::Schema;
+
+namespace proton {
+
+namespace
+{
+
+class FieldQuad
+{
+public:
+    vespalib::string _name;
+    vespalib::string _dataType;
+    vespalib::string _collectionType;
+    vespalib::string _location;
+
+    FieldQuad(const vespalib::string &name,
+              const vespalib::string &dataType,
+              const vespalib::string &collectionType,
+              const vespalib::string &location)
+        : _name(name),
+          _dataType(dataType),
+          _collectionType(collectionType),
+          _location(location)
+    {
+    }
+
+    bool
+    operator<(const FieldQuad &rhs) const
+    {
+        if (_name != rhs._name)
+            return _name < rhs._name;
+        if (_dataType != rhs._dataType)
+            return _dataType < rhs._dataType;
+        if (_collectionType != rhs._collectionType)
+            return _collectionType < rhs._collectionType;
+        return _location < rhs._location;
+    }
+};
+
+}
+
+
+Schema::SP
+SchemaUtil::makeHistorySchema(const Schema &newSchema,
+                              const Schema &oldSchema,
+                              const Schema &oldHistory)
+{
+    return makeHistorySchema(newSchema, oldSchema, oldHistory,
+                             fastos::ClockSystem::now());
+}
+
+Schema::SP
+SchemaUtil::makeHistorySchema(const Schema &newSchema,
+                              const Schema &oldSchema,
+                              const Schema &oldHistory,
+                              int64_t now)
+{
+    Schema::SP history(new Schema);
+
+    uint32_t fields = oldHistory.getNumIndexFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        const Schema::IndexField &field = oldHistory.getIndexField(fieldId);
+        uint32_t nFieldId = newSchema.getIndexFieldId(field.getName());
+        if (nFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Field not re-added, keep it in history
+            history->addIndexField(field);
+        }
+    }
+    fields = oldHistory.getNumAttributeFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        const Schema::AttributeField &field =
+            oldHistory.getAttributeField(fieldId);
+        uint32_t nFieldId =
+            newSchema.getAttributeFieldId(field.getName());
+        if (nFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Field not re-added, keep it in history
+            history->addAttributeField(field);
+        }
+    }
+    fields = oldHistory.getNumSummaryFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        const Schema::SummaryField &field =
+            oldHistory.getSummaryField(fieldId);
+        uint32_t nFieldId =
+            newSchema.getSummaryFieldId(field.getName());
+        if (nFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Field not re-added, keep it in history
+            history->addSummaryField(field);
+        }
+    }
+    fields = oldSchema.getNumIndexFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        Schema::IndexField field = oldSchema.getIndexField(fieldId);
+        uint32_t nFieldId = newSchema.getIndexFieldId(field.getName());
+        if (nFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Field removed, add to history
+            uint32_t oFieldId = history->getIndexFieldId(field.getName());
+            if (oFieldId == Schema::UNKNOWN_FIELD_ID) {
+                field.setTimestamp(now);
+                history->addIndexField(field);
+            }
+        }
+    }
+    fields = oldSchema.getNumAttributeFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        Schema::AttributeField field = oldSchema.getAttributeField(fieldId);
+        uint32_t nFieldId =
+            newSchema.getAttributeFieldId(field.getName());
+        if (nFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Field removed, add to history
+            uint32_t oFieldId = history->getAttributeFieldId(field.getName());
+            if (oFieldId == Schema::UNKNOWN_FIELD_ID) {
+                field.setTimestamp(now);
+                history->addAttributeField(field);
+            }
+        }
+    }
+    fields = oldSchema.getNumSummaryFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        Schema::SummaryField field = oldSchema.getSummaryField(fieldId);
+        uint32_t nFieldId =
+            newSchema.getSummaryFieldId(field.getName());
+        if (nFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Field removed, add to history
+            uint32_t oFieldId = history->getSummaryFieldId(field.getName());
+            if (oFieldId == Schema::UNKNOWN_FIELD_ID) {
+                field.setTimestamp(now);
+                history->addSummaryField(field);
+            }
+        }
+    }
+    return history;
+}
+
+
+Schema::SP
+SchemaUtil::makeUnionSchema(const Schema &schema,
+                            const Schema &history)
+{
+    Schema::SP uSchema(new Schema);
+    *uSchema = schema;
+    uint32_t fields = history.getNumIndexFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        const Schema::IndexField &field = history.getIndexField(fieldId);
+        uint32_t uFieldId = uSchema->getIndexFieldId(field.getName());
+        if (uFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Add back field known by history.
+            uSchema->addIndexField(field);
+        } else {
+            LOG(error,
+                "Index field '%s' is in both schema and history",
+                field.getName().c_str());
+        }
+    }
+    fields = history.getNumAttributeFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        const Schema::AttributeField &field =
+            history.getAttributeField(fieldId);
+        uint32_t uFieldId =
+            uSchema->getAttributeFieldId(field.getName());
+        if (uFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Add back field known by history.
+            uSchema->addAttributeField(field);
+        } else {
+            LOG(error,
+                "Attribute field '%s' is in both schema and history",
+                field.getName().c_str());
+        }
+    }
+    fields = history.getNumSummaryFields();
+    for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) {
+        const Schema::SummaryField &field =
+            history.getSummaryField(fieldId);
+        uint32_t uFieldId =
+            uSchema->getSummaryFieldId(field.getName());
+        if (uFieldId == Schema::UNKNOWN_FIELD_ID) {
+            // Add back field known by history.
+            uSchema->addSummaryField(field);
+        } else {
+            LOG(error,
+                "Summary field '%s' is in both schema and history",
+                field.getName().c_str());
+        }
+    }
+    return uSchema;
+}
+
+
+void
+SchemaUtil::listSchema(const Schema &schema,
+                       std::vector<vespalib::string> &fieldNames,
+                       std::vector<vespalib::string> &fieldDataTypes,
+                       std::vector<vespalib::string> &fieldCollectionTypes,
+                       std::vector<vespalib::string> &fieldLocations)
+{
+    std::vector<FieldQuad> quads;
+    for (uint32_t i = 0; i < schema.getNumAttributeFields(); ++i) {
+        const Schema::AttributeField &field = schema.getAttributeField(i);
+        quads.push_back(
+                FieldQuad(field.getName(),
+                          Schema::getTypeName(field.getDataType()),
+                          Schema::getTypeName(field.getCollectionType()),
+                          "a"));
+    }
+    for (uint32_t i = 0; i < schema.getNumIndexFields(); ++i) {
+        const Schema::IndexField &field = schema.getIndexField(i);
+        quads.push_back(
+                FieldQuad(field.getName(),
+                          Schema::getTypeName(field.getDataType()),
+                          Schema::getTypeName(field.getCollectionType()),
+                          "i"));
+    }
+    for (uint32_t i = 0; i < schema.getNumSummaryFields(); ++i) {
+        const Schema::SummaryField &field = schema.getSummaryField(i);
+        quads.push_back(
+                FieldQuad(field.getName(),
+                          Schema::getTypeName(field.getDataType()),
+                          Schema::getTypeName(field.getCollectionType()),
+                          "s"));
+    }
+    std::sort(quads.begin(), quads.end());
+    std::string name;
+    std::string dataType;
+    std::string collectionType;
+    std::string location;
+    for (std::vector<FieldQuad>::const_iterator
+             it = quads.begin(), ite = quads.end(); it != ite; ++it) {
+        if (it->_name != name || it->_dataType != dataType ||
+            it->_collectionType != collectionType) {
+            if (!name.empty()) {
+                fieldNames.push_back(name);
+                fieldDataTypes.push_back(dataType);
+                fieldCollectionTypes.push_back(collectionType);
+                fieldLocations.push_back(location);
+            }
+            name = it->_name;
+            dataType = it->_dataType;
+            collectionType = it->_collectionType;
+            location.clear();
+        }
+        location += it->_location;
+    }
+    if (!name.empty()) {
+        fieldNames.push_back(name);
+        fieldDataTypes.push_back(dataType);
+        fieldCollectionTypes.push_back(collectionType);
+        fieldLocations.push_back(location);
+    }
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/schemautil.h b/searchcore/src/vespa/searchcore/proton/common/schemautil.h
new file mode 100644
index 00000000000..bf276ac49e6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/schemautil.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vector>
+
+namespace proton
+{
+
+class SchemaUtil
+{
+public:
+    /**
+     * Make new history schema based on new and old schema and old history.
+     */
+    static search::index::Schema::SP
+    makeHistorySchema(const search::index::Schema &newSchema,
+                      const search::index::Schema &oldSchema,
+                      const search::index::Schema &oldHistory);
+
+    static search::index::Schema::SP
+    makeHistorySchema(const search::index::Schema &newSchema,
+                      const search::index::Schema &oldSchema,
+                      const search::index::Schema &oldHistory,
+                      int64_t timestamp);
+
+    /**
+     * Make union of schema and history, suitable for index fusion.
+     */
+    static search::index::Schema::SP
+    makeUnionSchema(const search::index::Schema &schema,
+                    const search::index::Schema &history);
+
+    /**
+     * Iterate through the given schema and fill out the given string vectors.
+     */
+    static void
+    listSchema(const search::index::Schema &schema,
+               std::vector<vespalib::string> &fieldNames,
+               std::vector<vespalib::string> &fieldDataTypes,
+               std::vector<vespalib::string> &fieldCollectionTypes,
+               std::vector<vespalib::string> &fieldLocations);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp
new file mode 100644
index 00000000000..6cf214343d8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.selectcontext");
+#include "selectcontext.h"
+#include "cachedselect.h"
+#include <vespa/searchlib/attribute/attributevector.h>
+
+namespace proton
+{
+
+using document::select::Value;
+using document::select::Context;
+using search::AttributeGuard;
+using search::AttributeVector;
+
+
+SelectContext::SelectContext(const CachedSelect &cachedSelect)
+    : Context(),
+      _docId(0u),
+      _guards(cachedSelect._attributes.size()),
+      _cachedSelect(cachedSelect)
+{
+}
+
+
+void
+SelectContext::getAttributeGuards(void)
+{
+    _guards.resize(_cachedSelect._attributes.size());
+    std::vector<AttributeVector::SP>::const_iterator
+        j(_cachedSelect._attributes.begin());
+    for (std::vector<AttributeGuard>::iterator i(_guards.begin()),
+             ie(_guards.end()); i != ie; ++i, ++j) {
+        *i = AttributeGuard(*j);
+    }
+}
+
+
+void
+SelectContext::dropAttributeGuards(void)
+{
+    _guards.clear();
+}
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h
new file mode 100644
index 00000000000..bd71099060c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/select/context.h>
+#include <vespa/searchlib/attribute/attributeguard.h>
+
+namespace proton
+{
+
+
+class CachedSelect;
+
+class SelectContext : public document::select::Context
+{
+public:
+    uint32_t _docId;
+
+    std::vector<search::AttributeGuard> _guards;
+    const CachedSelect &_cachedSelect;
+
+    SelectContext(const CachedSelect &cachedSelect);
+
+    void getAttributeGuards(void);
+    void dropAttributeGuards(void);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
new file mode 100644
index 00000000000..f4ae3dc0dd4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
@@ -0,0 +1,648 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.selectpruner");
+
+#include "selectpruner.h"
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/base/fieldpath.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/select/compare.h>
+#include <vespa/document/select/operator.h>
+#include <vespa/document/select/constant.h>
+#include <vespa/document/select/branch.h>
+#include <vespa/document/select/doctype.h>
+#include <vespa/document/select/invalidconstant.h>
+#include <vespa/document/select/constant.h>
+
+
+using document::select::And;
+using document::select::Compare;
+using document::select::Constant;
+using document::select::DocType;
+using document::select::Not;
+using document::select::Or;
+using document::select::ArithmeticValueNode;
+using document::select::FunctionValueNode;
+using document::select::IdValueNode;
+using document::select::SearchColumnValueNode;
+using document::select::FieldValueNode;
+using document::select::FloatValueNode;
+using document::select::VariableValueNode;
+using document::select::IntegerValueNode;
+using document::select::InvalidConstant;
+using document::select::CurrentTimeValueNode;
+using document::select::StringValueNode;
+using document::select::NullValueNode;
+using document::select::InvalidValueNode;
+using document::select::Node;
+using document::select::Result;
+using document::select::ResultSet;
+using document::select::ResultList;
+using document::select::FunctionOperator;
+using document::select::Operator;
+using document::select::InvalidValue;
+using document::select::ValueNode;
+using document::FieldPath;
+using document::Field;
+using document::FieldNotFoundException;
+
+namespace proton
+{
+
+SelectPrunerBase::SelectPrunerBase(const vespalib::string &docType,
+                                   const search::index::Schema &schema,
+                                   const document::Document &emptyDoc,
+                                   const document::DocumentTypeRepo &repo,
+                                   bool hasFields)
+    : _docType(docType),
+      _schema(schema),
+      _emptyDoc(emptyDoc),
+      _repo(repo),
+      _hasFields(hasFields)
+{
+}
+
+SelectPrunerBase::SelectPrunerBase(const SelectPrunerBase &rhs)
+    : _docType(rhs._docType),
+      _schema(rhs._schema),
+      _emptyDoc(rhs._emptyDoc),
+      _repo(rhs._repo),
+      _hasFields(rhs._hasFields)
+{
+}
+
+SelectPruner::SelectPruner(const vespalib::string &docType,
+                           const search::index::Schema &schema,
+                           const document::Document &emptyDoc,
+                           const document::DocumentTypeRepo &repo,
+                           bool hasFields)
+    : CloningVisitor(),
+      SelectPrunerBase(docType, schema, emptyDoc, repo, hasFields),
+      _inverted(false),
+      _wantInverted(false),
+      _attrFieldNodes(0u)
+{
+}
+
+SelectPruner::SelectPruner(const SelectPruner *rhs)
+    : CloningVisitor(),
+      SelectPrunerBase(*rhs),
+      _inverted(false),
+      _wantInverted(false),
+      _attrFieldNodes(0u)
+{
+}
+
+
+SelectPruner::~SelectPruner()
+{
+}
+
+
+void
+SelectPruner::visitAndBranch(const And &expr)
+{
+    SelectPruner lhs(this);
+    SelectPruner rhs(this);
+    if (_wantInverted) {
+        lhs._wantInverted = true;
+        rhs._wantInverted = true;
+    }
+    expr.getLeft().visit(lhs);
+    expr.getRight().visit(rhs);
+    if (lhs.getFieldNodes() - lhs.getAttrFieldNodes() >
+        rhs.getFieldNodes() - rhs.getAttrFieldNodes()) {
+        lhs.swap(rhs);
+    }
+    ResultSet lhsSet(lhs._resultSet);
+    ResultSet rhsSet(rhs._resultSet);
+    if (lhs._inverted)
+        lhsSet = lhsSet.calcNot();
+    if (rhs._inverted)
+        rhsSet = rhsSet.calcNot();
+    _resultSet = lhsSet.calcAnd(rhsSet);
+    _priority = AndPriority;
+    if (lhs._inverted && rhs._inverted) {
+        // De Morgan's laws
+        _inverted = true;
+        _priority = OrPriority;
+        _resultSet = _resultSet.calcNot();
+    }
+    _constVal = lhs._constVal && rhs._constVal;
+    lhs.resolveTernaryConst(_inverted);
+    rhs.resolveTernaryConst(_inverted);
+    if (lhs.isFalse() || rhs.isFalse()) {
+        // false and foo ==> false
+        // foo and false ==> false
+        setTernaryConst(_inverted);
+        return;
+    }
+    if (lhs.isTrue()) {
+        if (rhs.isTrue()) {
+            // true and true ==> true
+            setTernaryConst(!_inverted);
+            return;
+        }
+        // true and foo ==> foo
+        _node = std::move(rhs._node);
+        _priority = rhs._priority;
+        _inverted = rhs._inverted;
+        _resultSet = rhs._resultSet;
+        addNodeCount(rhs);
+        return;
+    } else if (rhs.isTrue()) {
+        // foo and true ==> foo
+        _node = std::move(lhs._node);
+        _priority = lhs._priority;
+        _inverted = lhs._inverted;
+        _resultSet = lhs._resultSet;
+        addNodeCount(lhs);
+        return;
+    }
+    if (lhs.isInvalid() && rhs.isInvalid()) {
+        setInvalidConst();
+        return;
+    }
+    if (lhs._inverted != _inverted) {
+        lhs.invertNode();
+    }
+    if (rhs._inverted != _inverted) {
+        rhs.invertNode();
+    }
+    if (lhs._priority < _priority)
+        lhs._node->setParentheses();
+    if (rhs._priority < _priority)
+        rhs._node->setParentheses();
+    if (_inverted) {
+        _node.reset(new Or(std::move(lhs._node), std::move(rhs._node), "or"));
+    } else {
+        _node.reset(new And(std::move(lhs._node), std::move(rhs._node), "and"));
+    }
+    addNodeCount(lhs);
+    addNodeCount(rhs);
+}
+
+
+void
+SelectPruner::visitComparison(const Compare &expr)
+{
+    SelectPruner lhs(this);
+    SelectPruner rhs(this);
+    expr.getLeft().visit(lhs);
+    expr.getRight().visit(rhs);
+    _constVal = lhs._constVal && rhs._constVal;
+    if (lhs.isInvalidVal() || rhs.isInvalidVal()) {
+        _inverted = _wantInverted;
+        _resultSet.add(Result::Invalid);
+        setInvalidConst();
+        return;
+    }
+    bool lhsNullVal = lhs.isNullVal();
+    bool rhsNullVal = rhs.isNullVal();
+    const Operator &op(getOperator(expr.getOperator()));
+    _node.reset(new Compare(std::move(lhs._valueNode),
+                            op,
+                            std::move(rhs._valueNode),
+                            expr.getBucketIdFactory()));
+    _priority = ComparePriority;
+    if (_constVal && (lhsNullVal || rhsNullVal)) {
+        if (!lhsNullVal || !rhsNullVal) {
+            // One null value
+            _inverted = _wantInverted;
+            _resultSet.add(Result::Invalid);
+            setInvalidConst();
+            return;
+        }
+        // Two null values
+        resolveTernaryConst(_wantInverted);
+        if (isInvalid()) {
+            _resultSet.add(Result::Invalid);
+        } else {
+            _resultSet.add(isTrue() != _inverted ?
+                           Result::True : Result::False);
+        }
+        return;
+    }
+    _resultSet.fill();	// should be less if const
+    addNodeCount(lhs);
+    addNodeCount(rhs);
+}
+
+
+void
+SelectPruner::visitDocumentType(const DocType &expr)
+{
+    _constVal = true;
+    bool res = expr.contains(_emptyDoc) == Result::True;
+    if (_wantInverted) {
+        _inverted = true;
+        res = !res;
+    }
+    _node.reset(new Constant(res ? "true" : "false"));
+    _resultSet.add(res ? Result::True : Result::False);
+    _priority = DocumentTypePriority;
+}
+
+
+void
+SelectPruner::visitNotBranch(const Not &expr)
+{
+    _wantInverted = !_wantInverted;
+    expr.getChild().visit(*this);
+    _inverted = !_inverted;
+    _wantInverted = !_wantInverted;
+}
+
+
+void
+SelectPruner::visitOrBranch(const Or &expr)
+{
+    SelectPruner lhs(this);
+    SelectPruner rhs(this);
+    if (_wantInverted) {
+        lhs._wantInverted = true;
+        rhs._wantInverted = true;
+    }
+    expr.getLeft().visit(lhs);
+    expr.getRight().visit(rhs);
+    if (lhs.getFieldNodes() - lhs.getAttrFieldNodes() >
+        rhs.getFieldNodes() - rhs.getAttrFieldNodes()) {
+        lhs.swap(rhs);
+    }
+    ResultSet lhsSet(lhs._resultSet);
+    ResultSet rhsSet(rhs._resultSet);
+    if (lhs._inverted)
+        lhsSet = lhsSet.calcNot();
+    if (rhs._inverted)
+        rhsSet = rhsSet.calcNot();
+    _resultSet = lhsSet.calcOr(rhsSet);
+    _priority = OrPriority;
+    if (lhs._inverted && rhs._inverted) {
+        // De Morgan's laws
+        _inverted = true;
+        _priority = AndPriority;
+        _resultSet = _resultSet.calcNot();
+    }
+    _constVal = lhs._constVal && rhs._constVal;
+    lhs.resolveTernaryConst(_inverted);
+    rhs.resolveTernaryConst(_inverted);
+    if (lhs.isTrue() || rhs.isTrue()) {
+        // true or foo ==> true
+        // foo or true ==> true
+        setTernaryConst(!_inverted);
+        return;
+    }
+    if (lhs.isFalse()) {
+        if (rhs.isFalse()) {
+            // false or false ==> false
+            setTernaryConst(_inverted);
+            return;
+        }
+        // false or foo ==> foo
+        _node = std::move(rhs._node);
+        _priority = rhs._priority;
+        _inverted = rhs._inverted;
+        _resultSet = rhs._resultSet;
+        addNodeCount(rhs);
+        return;
+    } else if (rhs.isFalse()) {
+        // foo or false ==> foo
+        _node = std::move(lhs._node);
+        _priority = lhs._priority;
+        _inverted = lhs._inverted;
+        _resultSet = lhs._resultSet;
+        addNodeCount(lhs);
+        return;
+    }
+    if (lhs.isInvalid() && rhs.isInvalid()) {
+        setInvalidConst();
+        return;
+    }
+    if (lhs._inverted != _inverted) {
+        lhs.invertNode();
+    }
+    if (rhs._inverted != _inverted) {
+        rhs.invertNode();
+    }
+    if (lhs._priority < _priority)
+        lhs._node->setParentheses();
+    if (rhs._priority < _priority)
+        rhs._node->setParentheses();
+    if (_inverted) {
+        _node.reset(new And(std::move(lhs._node), std::move(rhs._node), "and"));
+    } else {
+        _node.reset(new Or(std::move(lhs._node), std::move(rhs._node), "or"));
+    }
+    addNodeCount(lhs);
+    addNodeCount(rhs);
+}
+
+
+void
+SelectPruner::visitArithmeticValueNode(const ArithmeticValueNode &expr)
+{
+    SelectPruner lhs(this);
+    SelectPruner rhs(this);
+    expr.getLeft().visit(lhs);
+    expr.getRight().visit(rhs);
+    if (lhs.isInvalidVal() || rhs.isInvalidVal()) {
+        setInvalidVal();
+        return;
+    }
+    setArithmeticValueNode(expr,
+                           std::move(lhs._valueNode), lhs._priority, lhs._constVal,
+                           std::move(rhs._valueNode), rhs._priority, rhs._constVal);
+    addNodeCount(lhs);
+    addNodeCount(rhs);
+}
+
+
+void
+SelectPruner::visitFunctionValueNode(const FunctionValueNode &expr)
+{
+    expr.getChild().visit(*this);
+    if (isInvalidVal()) {
+        return; // Can shortcut evaluation when function argument is invalid
+    }
+    ValueNode::UP child(std::move(_valueNode));
+    const vespalib::string &funcName(expr.getFunctionName());
+    _valueNode.reset(new FunctionValueNode(funcName, std::move(child)));
+    if (_priority < FuncPriority) {
+        _valueNode->setParentheses();
+    }
+    _priority = FuncPriority;
+}
+
+
+void
+SelectPruner::visitFieldValueNode(const FieldValueNode &expr)
+{
+    if (_docType != expr.getDocType()) {
+        setInvalidVal();
+        return;
+    }
+    const document::DocumentType *docType = _repo.getDocumentType(_docType);
+    vespalib::string name(expr.getFieldName());
+    for (uint32_t i = 0; i < name.size(); ++i) {
+        if (name[i] == '.' || name[i] == '{' || name[i] == '[') {
+            // TODO: Check for struct, array, map or weigthed set
+            name = expr.getFieldName().substr(0, i);
+            break;
+        }
+    }
+    try {
+        std::unique_ptr<Field> fp(new Field(docType->getField(name)));
+        if (fp.get() == NULL) {
+            setInvalidVal();
+            return;
+        }
+    } catch (FieldNotFoundException &) {
+        setInvalidVal();
+        return;
+    }
+    try {
+        FieldPath::UP path(docType->buildFieldPath(expr.getFieldName()));
+        if (path.get() == NULL) {
+            setInvalidVal();
+            return;
+        }
+    } catch (vespalib::IllegalArgumentException &) {
+        setInvalidVal();
+        return;
+    } catch (FieldNotFoundException &) {
+        setInvalidVal();
+        return;
+    }
+    _constVal = false;
+    if (!_hasFields) {
+        // If we're working on removed document sub db then we have no fields.
+        _constVal = true;
+        _valueNode.reset(new NullValueNode("null"));
+        _priority = NullValPriority;
+        return;
+    }
+    
+    _valueNode = expr.clone(); // Replace with different node type for attrs ?
+    _valueNode->clearParentheses();
+    ++_fieldNodes;
+    if (_schema.isAttributeField(name)) {
+        ++_attrFieldNodes;
+    }
+    _priority = FieldValuePriority;
+}
+
+
+void
+SelectPruner::invertNode(void)
+{
+    _resultSet = _resultSet.calcNot();
+    if (isInvalid()) {
+        _inverted = !_inverted;
+        return;
+    }
+    if (_priority < NotPriority) {
+        _node->setParentheses();
+    }
+    NodeUP node(std::move(_node));
+    _node.reset(new Not(std::move(node), "not"));
+    _priority = NotPriority;
+    _inverted = !_inverted;
+}
+
+
+const Operator &
+SelectPruner::getOperator(const Operator &op)
+{
+    if (!_wantInverted)
+        return op;
+    if (op == FunctionOperator::GT) {
+        _inverted = true;
+        return FunctionOperator::LEQ;
+    }
+    if (op == FunctionOperator::GEQ) {
+        _inverted = true;
+        return FunctionOperator::LT;
+    }
+    if (op == FunctionOperator::EQ) {
+        _inverted = true;
+        return FunctionOperator::NE;
+    }
+    if (op == FunctionOperator::LEQ) {
+        _inverted = true;
+        return FunctionOperator::GT;
+    }
+    if (op == FunctionOperator::LT) {
+        _inverted = true;
+        return FunctionOperator::GEQ;
+    }
+    if (op == FunctionOperator::NE) {
+        _inverted = true;
+        return FunctionOperator::EQ;
+    }
+    return op;
+}
+
+
+void
+SelectPruner::addNodeCount(const SelectPruner &rhs)
+{
+    _fieldNodes += rhs._fieldNodes;
+    _attrFieldNodes += rhs._attrFieldNodes;
+}
+
+
+void
+SelectPruner::setInvalidVal(void)
+{
+    _constVal = true;
+    _priority = InvalidValPriority;
+    _valueNode.reset(new InvalidValueNode("invalidval"));
+}
+
+
+void
+SelectPruner::setInvalidConst(void)
+{
+    _constVal = true;
+    _priority = InvalidConstPriority;
+    _node.reset(new InvalidConstant("invalid"));
+}
+
+
+void
+SelectPruner::setTernaryConst(bool val)
+{
+    _constVal = true;
+    _priority = ConstPriority;
+    _node.reset(new Constant(val ? "true" : "false"));
+}
+
+void
+SelectPruner::resolveTernaryConst(bool wantInverted)
+{
+    if (!_constVal)
+        return;
+    const Result &res1(_node->contains(_emptyDoc).combineResults());
+    const Result &res = _inverted == wantInverted ? res1 : !res1;
+    if (res == Result::Invalid) {
+        setInvalidConst();
+    } else {
+        setTernaryConst(res == Result::True);
+        if (_inverted != wantInverted)
+            _resultSet = _resultSet.calcNot();
+        _inverted = wantInverted;
+    }
+}
+
+
+bool
+SelectPruner::isFalse(void) const
+{
+    if (!_constVal)
+        return false;
+    Constant *c(dynamic_cast<Constant *>(_node.get()));
+    if (c != NULL) {
+        return _inverted == c->getConstantValue();
+    }
+    InvalidConstant *ic(dynamic_cast<InvalidConstant *>(_node.get()));
+    if (ic != NULL) {
+        return false;
+    }
+    const Result &res(_node->contains(_emptyDoc).combineResults());
+    return _inverted ? res == Result::True : res == Result::False;
+}
+
+
+bool
+SelectPruner::isTrue(void) const
+{
+    if (!_constVal)
+        return false;
+    Constant *c(dynamic_cast<Constant *>(_node.get()));
+    if (c != NULL) {
+        return _inverted != c->getConstantValue();
+    }
+    InvalidConstant *ic(dynamic_cast<InvalidConstant *>(_node.get()));
+    if (ic != NULL) {
+        return false;
+    }
+    const Result &res(_node->contains(_emptyDoc).combineResults());
+    return _inverted ? res == Result::False : res == Result::True;
+}
+
+
+bool
+SelectPruner::isInvalid(void) const
+{
+    if (!_constVal)
+        return false;
+    Constant *c(dynamic_cast<Constant *>(_node.get()));
+    if (c != NULL) {
+        return false;
+    }
+    InvalidConstant *ic(dynamic_cast<InvalidConstant *>(_node.get()));
+    if (ic != NULL) {
+        return true;
+    }
+    const Result &res(_node->contains(_emptyDoc).combineResults());
+    return res == Result::Invalid;
+}
+
+
+bool
+SelectPruner::isInvalidVal(void) const
+{
+    if (!_constVal)
+        return false;
+    InvalidValueNode *iv(dynamic_cast<InvalidValueNode *>(_valueNode.get()));
+    return iv != NULL;
+}
+
+
+bool
+SelectPruner::isNullVal(void) const
+{
+    if (!_constVal)
+        return false;
+    NullValueNode *nv(dynamic_cast<NullValueNode *>(_valueNode.get()));
+    return nv != NULL;
+}
+
+
+bool
+SelectPruner::isConst(void) const
+{
+    return _constVal;
+}
+
+
+void
+SelectPruner::trace(std::ostream &t)
+{
+    _node->trace(_emptyDoc, t);
+}
+
+
+void
+SelectPruner::process(const Node &node)
+{
+    node.visit(*this);
+    resolveTernaryConst(false);
+    if (_inverted) {
+        invertNode();
+    }
+}
+
+
+void
+SelectPruner::swap(SelectPruner &rhs)
+{
+    CloningVisitor::swap(rhs);
+    std::swap(_inverted, rhs._inverted);
+    std::swap(_wantInverted, rhs._wantInverted);
+    std::swap(_attrFieldNodes, rhs._attrFieldNodes);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
new file mode 100644
index 00000000000..61c61644d6a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/select/node.h>
+#include <vespa/document/select/valuenode.h>
+#include <vespa/document/select/cloningvisitor.h>
+#include <vespa/document/select/operator.h>
+#include <vespa/document/select/resultset.h>
+#include <vespa/searchcommon/common/schema.h>
+
+namespace proton
+{
+
+class SelectPrunerBase
+{
+protected:
+    const vespalib::string &_docType;
+    const search::index::Schema &_schema;
+    const document::Document &_emptyDoc;
+    const document::DocumentTypeRepo &_repo;
+    bool _hasFields;
+
+public:
+    SelectPrunerBase(const vespalib::string &docType,
+                     const search::index::Schema &schema,
+                     const document::Document &emptyDoc,
+                     const document::DocumentTypeRepo &repo,
+                     bool hasFields);
+
+    SelectPrunerBase(const SelectPrunerBase &rhs);
+};
+
+
+class SelectPruner : public document::select::CloningVisitor,
+                      public SelectPrunerBase
+{
+public:
+private:
+    bool _inverted;
+    bool _wantInverted;
+    typedef document::select::Node::UP NodeUP;
+    typedef document::select::ValueNode::UP ValueNodeUP;
+    uint32_t _attrFieldNodes;
+public:
+    SelectPruner(const vespalib::string &docType,
+                 const search::index::Schema &schema,
+                 const document::Document &emptyDoc,
+                 const document::DocumentTypeRepo &repo,
+                 bool hasFields);
+
+    SelectPruner(const SelectPruner *rhs);
+
+    virtual
+    ~SelectPruner(void);
+
+    uint32_t
+    getFieldNodes(void) const
+    {
+        return _fieldNodes;
+    }
+
+    uint32_t
+    getAttrFieldNodes(void) const
+    {
+        return _attrFieldNodes;
+    }
+
+    const document::select::ResultSet &
+    getResultSet(void) const
+    {
+        return _resultSet;
+    }
+
+    bool
+    isFalse(void) const;
+
+    bool
+    isTrue(void) const;
+
+    bool
+    isInvalid(void) const;
+
+    bool
+    isConst(void) const;
+
+    void
+    trace(std::ostream &t);
+
+    void
+    process(const document::select::Node &node);
+private:
+    virtual void
+    visitAndBranch(const document::select::And &expr);
+
+    virtual void
+    visitComparison(const document::select::Compare &expr);
+
+    virtual void
+    visitDocumentType(const document::select::DocType &expr);
+
+    virtual void
+    visitNotBranch(const document::select::Not &expr);
+
+    virtual void
+    visitOrBranch(const document::select::Or &expr);
+
+    virtual void
+    visitArithmeticValueNode(const document::select::ArithmeticValueNode &
+                             expr);
+
+    virtual void
+    visitFunctionValueNode(const document::select::FunctionValueNode &expr);
+
+    virtual void
+    visitFieldValueNode(const document::select::FieldValueNode &expr);
+
+    void
+    invertNode(void);
+
+    const document::select::Operator &
+    getOperator(const document::select::Operator &op);
+
+    void
+    addNodeCount(const SelectPruner &rhs);
+
+    void
+    setInvalidVal(void);
+
+    void
+    setInvalidConst(void);
+
+    void
+    setTernaryConst(bool val);
+
+    void
+    resolveTernaryConst(bool wantInverted);
+
+    bool
+    isInvalidVal(void) const;
+
+    bool
+    isNullVal(void) const;
+
+    void
+    swap(SelectPruner &rhs);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.cpp b/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.cpp
new file mode 100644
index 00000000000..01f202fe1f4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.cpp
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.common.state_reporter_utils");
+
+#include "state_reporter_utils.h"
+#include <vespa/vespalib/data/slime/cursor.h>
+
+using namespace vespalib::slime;
+
+namespace proton {
+
+void
+StateReporterUtils::convertToSlime(const StatusReport &statusReport,
+                                   const Inserter &inserter)
+{
+    Cursor &object = inserter.insertObject();
+    object.setString("state", statusReport.getInternalState());
+    if (statusReport.hasProgress()) {
+        object.setDouble("progress", statusReport.getProgress());
+    }
+    if (!statusReport.getInternalConfigState().empty()) {
+        object.setString("configState", statusReport.getInternalConfigState());
+    }
+    if (!statusReport.getMessage().empty()) {
+        object.setString("message", statusReport.getMessage());
+    }
+}
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.h b/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.h
new file mode 100644
index 00000000000..219c5e358bc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.h
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "statusreport.h"
+#include <vespa/vespalib/data/slime/inserter.h>
+
+namespace proton {
+
+/**
+ * Utilities for converting state related objects to slime.
+ */
+struct StateReporterUtils
+{
+    static void convertToSlime(const StatusReport &statusReport,
+                               const vespalib::slime::Inserter &inserter);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/statusreport.h b/searchcore/src/vespa/searchcore/proton/common/statusreport.h
new file mode 100644
index 00000000000..ecb3b696a1f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/statusreport.h
@@ -0,0 +1,137 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <cmath>
+#include <limits>
+#include <memory>
+
+namespace proton {
+
+/**
+ * A StatusReport describes the status of a search component.
+ */
+class StatusReport {
+public:
+    typedef std::unique_ptr<StatusReport> UP;
+    typedef std::shared_ptr<StatusReport> SP;
+    typedef std::vector<SP> List;
+
+    enum State {
+        DOWN = 0,
+        PARTIAL,
+        UPOK
+    };
+
+    struct Params {
+        vespalib::string _component;
+        State _state;
+        vespalib::string _internalState;
+        vespalib::string _internalConfigState;
+        float _progress;
+        vespalib::string _message;
+
+        Params(const vespalib::string &component)
+            : _component(component),
+              _state(DOWN),
+              _internalState(),
+              _internalConfigState(),
+              _progress(std::numeric_limits<float>::quiet_NaN()),
+              _message()
+        {}
+        Params &state(State value) {
+            _state = value;
+            return *this;
+        }
+        Params &internalState(const vespalib::string &value) {
+            _internalState = value;
+            return *this;
+        }
+        Params &internalConfigState(const vespalib::string &value) {
+            _internalConfigState = value;
+            return *this;
+        }
+        Params &progress(float value) {
+            _progress = value;
+            return *this;
+        }
+        Params &message(const vespalib::string &value) {
+            _message = value;
+            return *this;
+        }
+    };
+
+private:
+    vespalib::string _component;
+    State _state;
+    vespalib::string _internalState;
+    vespalib::string _internalConfigState;
+    float _progress;
+    vespalib::string _message;
+
+public:
+    StatusReport(const Params &params)
+        : _component(params._component),
+          _state(params._state),
+          _internalState(params._internalState),
+          _internalConfigState(params._internalConfigState),
+          _progress(params._progress),
+          _message(params._message)
+    {}
+
+    static StatusReport::UP create(const Params &params) {
+        return StatusReport::UP(new StatusReport(params));
+    }
+
+    const vespalib::string &getComponent() const {
+        return _component;
+    }
+
+    State getState() const {
+        return _state;
+    }
+
+    const vespalib::string &getInternalState() const {
+        return _internalState;
+    }
+
+    const vespalib::string &getInternalConfigState() const {
+        return _internalConfigState;
+    }
+
+    bool hasProgress() const {
+        return !std::isnan(_progress);
+    }
+
+    float getProgress() const {
+        return _progress;
+    }
+
+    const vespalib::string &getMessage() const {
+        return _message;
+    }
+
+    vespalib::string getInternalStatesStr() const {
+        vespalib::string retval = "state=" + _internalState;
+        if (!_internalConfigState.empty()) {
+            retval = retval + " configstate=" + _internalConfigState;
+        }
+        return retval;
+    }
+
+};
+
+
+
+
+/**
+ * A StatusProducer is able to produce a list of StatusReport objects
+ * when needed.
+ **/
+struct StatusProducer {
+    virtual StatusReport::List getStatusReports() const = 0;
+    virtual ~StatusProducer() {}
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/common/subdbtype.h b/searchcore/src/vespa/searchcore/proton/common/subdbtype.h
new file mode 100644
index 00000000000..0864b0a8fe7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/subdbtype.h
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+/**
+ * Enumeration of the different kinds of sub databases within a
+ * document db.
+ */
+enum class SubDbType {
+    READY = 0,
+    REMOVED = 1,
+    NOTREADY = 2,
+    COUNT = 3/* number of valid subdb types, this value by itself is invalid */
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/create-base.sh b/searchcore/src/vespa/searchcore/proton/create-base.sh
new file mode 100644
index 00000000000..faf622de694
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/create-base.sh
@@ -0,0 +1,30 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+module="proton"
+
+if [ $# -lt 1 ]; then
+	echo "Code generation script for the $module module"
+	echo ""
+	echo "usage: $0 <class or interface name>"
+	echo ""
+	echo "Generates class files or interface files"
+	echo "depending on which script was invoked."
+	echo ""
+	echo "The current directory is used to generate"
+	echo "appropriate include guards."
+	echo ""
+	echo "Generated code is written to stdout."
+	echo ""
+	exit 1
+fi
+
+class=$1
+name=`echo $class | tr 'A-Z' 'a-z'`
+prefix=`pwd | sed -e "s|.*/${module}||" | tr '/' '_'`
+guard=`echo H_${module}${prefix}_${class}_H | tr 'a-z' 'A-Z'`
+ns_open="namespace $module {"
+ns_close="} // namespace $module"
+
+cat <<EOF
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+EOF
diff --git a/searchcore/src/vespa/searchcore/proton/create-class-cpp.sh b/searchcore/src/vespa/searchcore/proton/create-class-cpp.sh
new file mode 100755
index 00000000000..1b0b2440c45
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/create-class-cpp.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+dir=`dirname $0`
+. "$dir/create-base.sh"
+
+cat <<EOF
+#include <vespa/log/log.h>
+LOG_SETUP(".$name");
+#include <vespa/fastos/fastos.h>
+#include "$name.h"
+
+$ns_open
+
+$class::$class()
+{
+}
+
+$class::~$class()
+{
+}
+
+$ns_close
+EOF
diff --git a/searchcore/src/vespa/searchcore/proton/create-class-h.sh b/searchcore/src/vespa/searchcore/proton/create-class-h.sh
new file mode 100644
index 00000000000..5c5c7b94861
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/create-class-h.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+dir=`dirname $0`
+. "$dir/create-base.sh"
+
+cat <<EOF
+#pragma once
+
+$ns_open
+
+class $class
+{
+private:
+    $class(const $class &);
+    $class &operator=(const $class &);
+public:
+    $class();
+    virtual ~$class();
+};
+
+$ns_close
+
+EOF
diff --git a/searchcore/src/vespa/searchcore/proton/create-interface.sh b/searchcore/src/vespa/searchcore/proton/create-interface.sh
new file mode 100644
index 00000000000..12f1273e064
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/create-interface.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+dir=`dirname $0`
+. "$dir/create-base.sh"
+
+cat <<EOF
+#pragma once
+
+$ns_open
+
+class $class
+{
+public:
+    virtual ~$class() {}
+};
+
+$ns_close
+
+EOF
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/.gitignore b/searchcore/src/vespa/searchcore/proton/docsummary/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/docsummary/CMakeLists.txt
new file mode 100644
index 00000000000..37e2e691a4f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_docsummary STATIC
+    SOURCES
+    docsumcontext.cpp
+    document_store_explorer.cpp
+    documentstoreadapter.cpp
+    fieldcache.cpp
+    fieldcacherepo.cpp
+    linguisticsannotation.cpp
+    searchdatatype.cpp
+    summarycompacttarget.cpp
+    summaryfieldconverter.cpp
+    summaryflushtarget.cpp
+    summarymanager.cpp
+    summarymanagerinitializer.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/OWNERS b/searchcore/src/vespa/searchcore/proton/docsummary/OWNERS
new file mode 100644
index 00000000000..7ae1acb1be9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/OWNERS
@@ -0,0 +1 @@
+geirst
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp
new file mode 100644
index 00000000000..142516d08ab
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp
@@ -0,0 +1,206 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.docsumcontext");
+#include "docsumcontext.h"
+#include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+#include <vespa/searchlib/common/location.h>
+
+using document::PositionDataType;
+using search::common::Location;
+using vespalib::string;
+using vespalib::slime::SymbolTable;
+using vespalib::slime::NIX;
+using vespalib::slime::Memory;
+using vespalib::slime::Cursor;
+using vespalib::slime::Symbol;
+using vespalib::slime::Inserter;
+using vespalib::slime::ObjectSymbolInserter;
+using vespalib::Slime;
+using namespace search;
+using namespace search::attribute;
+using namespace search::engine;
+using namespace search::docsummary;
+
+namespace proton {
+
+using namespace matching;
+
+namespace {
+
+Memory DOCSUMS("docsums");
+Memory DOCSUM("docsum");
+
+}
+
+void
+DocsumContext::initState()
+{
+    const DocsumRequest & req = _request;
+    _docsumState._args.initFromDocsumRequest(req);
+    _docsumState._args.SetQueryFlags(req.queryFlags & ~search::fs4transport::QFLAG_DROP_SORTDATA);
+    _docsumState._docsumcnt = req.hits.size();
+
+    if (_docsumState._docsumcnt > 0) {
+        _docsumState._docsumbuf = (uint32_t*)malloc(sizeof(uint32_t) * _docsumState._docsumcnt);
+    } else {
+        _docsumState._docsumbuf = NULL;
+    }
+    for (uint32_t i = 0; i < _docsumState._docsumcnt; i++) {
+        _docsumState._docsumbuf[i] = req.hits[i].docid;
+    }
+}
+
+DocsumReply::UP
+DocsumContext::createReply()
+{
+    DocsumReply::UP reply(new DocsumReply());
+    search::RawBuf buf(4096);
+    _docsumWriter.InitState(_attrMgr, &_docsumState);
+    reply->docsums.resize(_docsumState._docsumcnt);
+    SymbolTable::UP symbols = std::make_unique<SymbolTable>();
+    IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(), _docsumStore.getSummaryClassId());
+    for (uint32_t i = 0; i < _docsumState._docsumcnt; ++i) {
+        buf.reset();
+        uint32_t docId = _docsumState._docsumbuf[i];
+        reply->docsums[i].docid = docId;
+        if (docId != search::endDocId && !rci.mustSkip) {
+            if ((_docsumState._args.getFlags() & ::search::fs4transport::GDFLAG_ALLOW_SLIME) != 0) {
+                Slime slime(Slime::Params(std::move(symbols)));
+                vespalib::slime::SlimeInserter inserter(slime);
+                _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, slime, inserter);
+                uint32_t docsumLen = (slime.get().type().getId() != NIX::ID)
+                                   ? IDocsumWriter::slime2RawBuf(slime, buf)
+                                   : 0;
+                reply->docsums[i].setData(buf.GetDrainPos(), docsumLen);
+                symbols = Slime::reclaimSymbols(std::move(slime));
+            } else {
+                uint32_t docsumLen = _docsumWriter.WriteDocsum(docId, &_docsumState, &_docsumStore, &buf);
+                reply->docsums[i].setData(buf.GetDrainPos(), docsumLen);
+            }
+        }
+    }
+    return reply;
+}
+
+namespace {
+
+vespalib::Slime::Params
+makeSlimeParams(size_t chunkSize) {
+    Slime::Params params;
+    params.setChunkSize(chunkSize);
+    return params;
+}
+
+}
+
+vespalib::Slime::UP
+DocsumContext::createSlimeReply()
+{
+    _docsumWriter.InitState(_attrMgr, &_docsumState);
+    const size_t estimatedChunkSize(std::min(0x200000ul, _docsumState._docsumcnt*0x400ul));
+    vespalib::Slime::UP response(std::make_unique<vespalib::Slime>(makeSlimeParams(estimatedChunkSize)));
+    Cursor & root = response->setObject();
+    Cursor & array = root.setArray(DOCSUMS);
+    const Symbol docsumSym = response->insert(DOCSUM);
+    IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(), _docsumStore.getSummaryClassId());
+    for (uint32_t i = 0; i < _docsumState._docsumcnt; ++i) {
+        uint32_t docId = _docsumState._docsumbuf[i];
+        Cursor & docSumC = array.addObject();
+        ObjectSymbolInserter inserter(docSumC, docsumSym);
+        if (docId != search::endDocId && !rci.mustSkip) {
+            _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, *response, inserter);
+        }
+    }
+    return response;
+}
+
+DocsumContext::DocsumContext(const DocsumRequest & request,
+                             IDocsumWriter & docsumWriter,
+                             IDocsumStore & docsumStore,
+                             const Matcher::SP & matcher,
+                             ISearchContext & searchCtx,
+                             IAttributeContext & attrCtx,
+                             search::IAttributeManager & attrMgr,
+                             SessionManager & sessionMgr) :
+    _request(request),
+    _docsumWriter(docsumWriter),
+    _docsumStore(docsumStore),
+    _matcher(matcher),
+    _searchCtx(searchCtx),
+    _attrCtx(attrCtx),
+    _attrMgr(attrMgr),
+    _docsumState(*this),
+    _sessionMgr(sessionMgr)
+{
+    initState();
+}
+
+DocsumReply::UP
+DocsumContext::getDocsums()
+{
+    if (_request.useRootSlime()) {
+        return std::make_unique<DocsumReply>(createSlimeReply());
+    }
+    return createReply();
+}
+
+void
+DocsumContext::FillSummaryFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment *)
+{
+    assert(&_docsumState == state);
+    if (_matcher->canProduceSummaryFeatures()) {
+        state->_summaryFeatures =
+            _matcher->getSummaryFeatures(_request, _searchCtx, _attrCtx,
+                                         _sessionMgr);
+    }
+    state->_summaryFeaturesCached = false;
+}
+
+void
+DocsumContext::FillRankFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment *)
+{
+    assert(&_docsumState == state);
+    // check if we are allowed to run
+    if ((state->_args.GetQueryFlags() & search::fs4transport::QFLAG_DUMP_FEATURES) == 0) {
+        return;
+    }
+    state->_rankFeatures =
+        _matcher->getRankFeatures(_request, _searchCtx, _attrCtx, _sessionMgr);
+}
+
+namespace {
+Location *getLocation(const string &loc_str, search::IAttributeManager &attrMgr)
+{
+    LOG(debug, "Filling document locations from location string: %s", loc_str.c_str());
+
+    Location *loc = new Location;
+    string location;
+    string::size_type pos = loc_str.find(':');
+    if (pos != string::npos) {
+        string view = loc_str.substr(0, pos);
+        AttributeGuard::UP vec = attrMgr.getAttribute(view);
+        if (!vec->valid()) {
+            view = PositionDataType::getZCurveFieldName(view);
+            vec = attrMgr.getAttribute(view);
+        }
+        loc->setVecGuard(std::move(vec));
+        location = loc_str.substr(pos + 1);
+    } else {
+        LOG(warning, "Location string lacks attribute vector specification. "
+            "loc='%s'", loc_str.c_str());
+        location = loc_str;
+    }
+    loc->parse(location);
+    return loc;
+}
+}  // namespace
+
+void
+DocsumContext::ParseLocation(search::docsummary::GetDocsumsState *state)
+{
+    state->_parsedLocation.reset(getLocation(_request.location, _attrMgr));
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h
new file mode 100644
index 00000000000..d0ea8e089d1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/matching/matcher.h>
+#include <vespa/searchsummary/docsummary/docsumstate.h>
+#include <vespa/searchsummary/docsummary/docsumstore.h>
+#include <vespa/searchsummary/docsummary/docsumwriter.h>
+#include <vespa/searchlib/engine/docsumrequest.h>
+#include <vespa/searchlib/engine/docsumreply.h>
+
+namespace proton {
+
+/**
+ * The DocsumContext class is responsible for performing a docsum request and
+ * creating a docsum reply.
+ **/
+class DocsumContext : public boost::noncopyable,
+                      public search::docsummary::GetDocsumsStateCallback {
+private:
+    const search::engine::DocsumRequest  & _request;
+    search::docsummary::IDocsumWriter    & _docsumWriter;
+    search::docsummary::IDocsumStore     & _docsumStore;
+    matching::Matcher::SP                  _matcher;
+    matching::ISearchContext             & _searchCtx;
+    search::attribute::IAttributeContext & _attrCtx;
+    search::IAttributeManager            & _attrMgr;
+    search::docsummary::GetDocsumsState    _docsumState;
+    matching::SessionManager             & _sessionMgr;
+
+    void initState();
+    search::engine::DocsumReply::UP createReply();
+    vespalib::Slime::UP createSlimeReply();
+
+public:
+    typedef std::unique_ptr<DocsumContext> UP;
+
+    DocsumContext(const search::engine::DocsumRequest & request,
+                  search::docsummary::IDocsumWriter & docsumWriter,
+                  search::docsummary::IDocsumStore & docsumStore,
+                  const matching::Matcher::SP & matcher,
+                  matching::ISearchContext & searchCtx,
+                  search::attribute::IAttributeContext & attrCtx,
+                  search::IAttributeManager & attrMgr,
+                  matching::SessionManager & sessionMgr);
+
+    search::engine::DocsumReply::UP getDocsums();
+
+    // Implements GetDocsumsStateCallback
+    virtual void FillSummaryFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env);
+    virtual void FillRankFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env);
+    virtual void ParseLocation(search::docsummary::GetDocsumsState * state);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.cpp
new file mode 100644
index 00000000000..ddc885a2b0b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.cpp
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.document_store_explorer");
+#include "document_store_explorer.h"
+
+using vespalib::slime::Cursor;
+using vespalib::slime::Inserter;
+using search::DataStoreFileChunkStats;
+using search::DataStoreStorageStats;
+
+namespace proton {
+
+DocumentStoreExplorer::DocumentStoreExplorer(ISummaryManager::SP mgr)
+    : _mgr(std::move(mgr))
+{
+}
+
+void
+DocumentStoreExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    Cursor &object = inserter.insertObject();
+    search::IDocumentStore &store = _mgr->getBackingStore();
+    DataStoreStorageStats storageStats(store.getStorageStats());
+    object.setLong("diskUsage", storageStats.diskUsage());
+    object.setLong("diskBloat", storageStats.diskBloat());
+    object.setDouble("maxBucketSpread", storageStats.maxBucketSpread());
+    object.setLong("lastFlushedSerialNum", storageStats.lastFlushedSerialNum());
+    object.setLong("lastSerialNum", storageStats.lastSerialNum());
+    if (full) {
+        const vespalib::string &baseDir = store.getBaseDir();
+        std::vector<DataStoreFileChunkStats> chunks;
+        chunks = store.getFileChunkStats();
+        Cursor &fileChunksArrayCursor = object.setArray("fileChunks");
+        for (const auto &chunk : chunks) {
+            Cursor &chunkCursor = fileChunksArrayCursor.addObject();
+            chunkCursor.setLong("diskUsage", chunk.diskUsage());
+            chunkCursor.setLong("diskBloat", chunk.diskBloat());
+            chunkCursor.setDouble("bucketSpread", chunk.maxBucketSpread());
+            chunkCursor.setLong("lastFlushedSerialNum",
+                                chunk.lastFlushedSerialNum());
+            chunkCursor.setLong("lastSerialNum", chunk.lastSerialNum());
+            chunkCursor.setLong("nameid", chunk.nameId());
+            chunkCursor.setString("name", chunk.createName(baseDir));
+        }
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.h b/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.h
new file mode 100644
index 00000000000..4dcfbcd4097
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "isummarymanager.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of a document store.
+ */
+class DocumentStoreExplorer : public vespalib::StateExplorer
+{
+private:
+    ISummaryManager::SP _mgr;
+
+public:
+    DocumentStoreExplorer(ISummaryManager::SP mgr);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
new file mode 100644
index 00000000000..eb44ac89e32
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
@@ -0,0 +1,202 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.documentstoreadapter");
+#include "documentstoreadapter.h"
+#include "summaryfieldconverter.h"
+
+#include <vespa/document/fieldvalue/literalfieldvalue.h>
+
+using namespace document;
+using namespace search::docsummary;
+
+namespace proton {
+
+namespace {
+
+const vespalib::string DOCUMENT_ID_FIELD("documentid");
+
+}
+
+bool
+DocumentStoreAdapter::writeStringField(const char * buf,
+                                       uint32_t buflen,
+                                       ResType type)
+{
+    switch (type) {
+    case RES_STRING:
+        return _resultPacker.AddString(buf, buflen);
+    case RES_LONG_STRING:
+    case RES_XMLSTRING:
+    case RES_JSONSTRING:
+        return _resultPacker.AddLongString(buf, buflen);
+    default:
+        break;
+    }
+    return false;
+}
+
+bool
+DocumentStoreAdapter::writeField(const FieldValue &value, ResType type)
+{
+    switch (type) {
+    case RES_BYTE:
+        return _resultPacker.AddByte(value.getAsInt());
+    case RES_SHORT:
+        return _resultPacker.AddShort(value.getAsInt());
+    case RES_INT:
+        return _resultPacker.AddInteger(value.getAsInt());
+    case RES_INT64:
+        return _resultPacker.AddInt64(value.getAsLong());
+    case RES_FLOAT:
+        return _resultPacker.AddFloat(value.getAsFloat());
+    case RES_DOUBLE:
+        return _resultPacker.AddDouble(value.getAsDouble());
+    case RES_STRING:
+    case RES_LONG_STRING:
+    case RES_XMLSTRING:
+    case RES_JSONSTRING:
+        {
+            if (value.getClass().inherits(LiteralFieldValueB::classId)) {
+                const LiteralFieldValueB & lfv =
+                    static_cast<const LiteralFieldValueB &>(value);
+                vespalib::stringref s = lfv.getValueRef();
+                return writeStringField(s.c_str(), s.size(), type);
+            } else {
+                vespalib::string s = value.getAsString();
+                return writeStringField(s.c_str(), s.size(), type);
+            }
+        }
+    case RES_DATA:
+        {
+            std::pair<const char *, size_t> buf = value.getAsRaw();
+            return _resultPacker.AddData(buf.first, buf.second);
+        }
+    case RES_LONG_DATA:
+        {
+            std::pair<const char *, size_t> buf = value.getAsRaw();
+            return _resultPacker.AddLongData(buf.first, buf.second);
+        }
+    default:
+        LOG(warning,
+            "Unknown docsum field type: %s. Add empty field",
+            ResultConfig::GetResTypeName(type));
+        return _resultPacker.AddEmpty();
+    }
+    return false;
+}
+
+
+void
+DocumentStoreAdapter::convertFromSearchDoc(Document &doc, uint32_t docId, bool useSlimeInsideFields)
+{
+    for (size_t i = 0; i < _resultClass->GetNumEntries(); ++i) {
+        const ResConfigEntry * entry = _resultClass->GetEntry(i);
+        const vespalib::string fieldName(entry->_bindname);
+        bool markup = _markupFields.find(fieldName) != _markupFields.end();
+        if (fieldName == DOCUMENT_ID_FIELD) {
+            StringFieldValue value(doc.getId().toString());
+            if (!writeField(value, entry->_type)) {
+                LOG(warning, "Error while writing field '%s' for docId %u",
+                    fieldName.c_str(), docId);
+            }
+            continue;
+        }
+        const Field *field = _fieldCache->getField(i);
+        if (!field) {
+            LOG(debug,
+                "Did not find field '%s' in the document "
+                "for docId %u. Adding empty field",
+                fieldName.c_str(), docId);
+            _resultPacker.AddEmpty();
+            continue;
+        }
+        FieldValue::UP fieldValue = doc.getValue(*field);
+        if (fieldValue.get() == NULL) {
+            LOG(spam,
+                "No field value for field '%s' in the document "
+                "for docId %u. Adding empty field",
+                fieldName.c_str(), docId);
+            _resultPacker.AddEmpty();
+            continue;
+        }
+        LOG(spam,
+            "writeField(%s): value(%s), type(%d)",
+            fieldName.c_str(), fieldValue->toString().c_str(),
+            entry->_type);
+        FieldValue::UP convertedFieldValue =
+            SummaryFieldConverter::convertSummaryField(markup, *fieldValue, useSlimeInsideFields);
+        if (convertedFieldValue.get() != NULL) {
+            if (!writeField(*convertedFieldValue, entry->_type)) {
+                LOG(warning,
+                    "Error while writing field '%s' for docId %u",
+                    fieldName.c_str(), docId);
+            }
+        } else {
+            LOG(spam,
+                "No converted field value for field '%s' "
+                " in the document "
+                "for docId %u. Adding empty field",
+                fieldName.c_str(), docId);
+            _resultPacker.AddEmpty();
+        }
+    }
+}
+
+
+DocumentStoreAdapter::
+DocumentStoreAdapter(const search::IDocumentStore & docStore,
+                     const DocumentTypeRepo &repo,
+                     const ResultConfig & resultConfig,
+                     const vespalib::string & resultClassName,
+                     const FieldCache::CSP & fieldCache,
+                     const std::set<vespalib::string> &markupFields)
+    : _docStore(docStore),
+      _repo(repo),
+      _resultConfig(resultConfig),
+      _resultClass(resultConfig.
+                   LookupResultClass(resultConfig.
+                                     LookupResultClassId(resultClassName.
+                                             c_str()))),
+      _resultPacker(&_resultConfig),
+      _fieldCache(fieldCache),
+      _markupFields(markupFields)
+{
+}
+
+DocsumStoreValue
+DocumentStoreAdapter::getMappedDocsum(uint32_t docId, bool useSlimeInsideFields)
+{
+    if (!_resultPacker.Init(getSummaryClassId())) {
+        LOG(warning,
+            "Error during init of result class '%s' with class id %u",
+            _resultClass->GetClassName(), getSummaryClassId());
+        return DocsumStoreValue();
+    }
+    Document::UP document = _docStore.read(docId, _repo);
+    if (document.get() == NULL) {
+        LOG(debug,
+            "Did not find summary document for docId %u. "
+            "Returning empty docsum",
+            docId);
+        return DocsumStoreValue();
+    }
+    LOG(spam,
+        "getMappedDocSum(%u): document={\n%s\n}",
+        docId,
+        document->toString(true).c_str());
+    convertFromSearchDoc(*document, docId, useSlimeInsideFields);
+    const char * buf;
+    uint32_t buflen;
+    if (!_resultPacker.GetDocsumBlob(&buf, &buflen)) {
+        LOG(warning,
+            "Error while getting the docsum blob for docId %u. "
+            "Returning empty docsum",
+            docId);
+        return DocsumStoreValue();
+    }
+    return DocsumStoreValue(buf, buflen);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h
new file mode 100644
index 00000000000..b71cf007899
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/searchsummary/docsummary/docsumstore.h>
+#include <vespa/searchsummary/docsummary/resultconfig.h>
+#include <vespa/searchsummary/docsummary/resultpacker.h>
+#include "fieldcache.h"
+
+namespace proton {
+
+class DocumentStoreAdapter : public search::docsummary::IDocsumStore
+{
+private:
+    const search::IDocumentStore           & _docStore;
+    const document::DocumentTypeRepo       & _repo;
+    const search::docsummary::ResultConfig & _resultConfig;
+    const search::docsummary::ResultClass  * _resultClass;
+    search::docsummary::ResultPacker         _resultPacker;
+    FieldCache::CSP                          _fieldCache;
+    const std::set<vespalib::string>       & _markupFields;
+
+    bool
+    writeStringField(const char * buf,
+                     uint32_t buflen,
+                     search::docsummary::ResType type);
+
+    bool
+    writeField(const document::FieldValue &value,
+               search::docsummary::ResType type);
+
+    void
+    convertFromSearchDoc(document::Document &doc, uint32_t docId,
+                         bool useSlimeInsideFields);
+
+public:
+    DocumentStoreAdapter(const search::IDocumentStore &docStore,
+                         const document::DocumentTypeRepo &repo,
+                         const search::docsummary::ResultConfig &resultConfig,
+                         const vespalib::string &resultClassName,
+                         const FieldCache::CSP &fieldCache,
+                         const std::set<vespalib::string> &markupFields);
+
+    const search::docsummary::ResultClass *getResultClass() const {
+        return _resultClass;
+    }
+
+    uint32_t getNumDocs() override { return _docStore.nextId(); }
+    search::docsummary::DocsumStoreValue getMappedDocsum(uint32_t docId, bool useSlimeInsideFields) override;
+    uint32_t getSummaryClassId() const override { return _resultClass->GetClassID(); }
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp
new file mode 100644
index 00000000000..15aacdd4589
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.fieldcache");
+#include "fieldcache.h"
+#include <vespa/document/fieldvalue/document.h>
+
+using namespace document;
+using namespace search::docsummary;
+
+namespace proton {
+
+FieldCache::FieldCache() :
+    _cache()
+{
+}
+
+FieldCache::FieldCache(const ResultClass &resClass,
+                       const DocumentType &docType) :
+    _cache()
+{
+    LOG(debug, "Creating field cache for summary class '%s'", resClass.GetClassName());
+    for (uint32_t i = 0; i < resClass.GetNumEntries(); ++i) {
+        const ResConfigEntry *entry = resClass.GetEntry(i);
+        const vespalib::string fieldName(entry->_bindname);
+        if (docType.hasField(fieldName)) {
+            const Field &field = docType.getField(fieldName);
+            LOG(debug, "Caching Field instance for field '%s': %s.%u",
+                fieldName.c_str(), field.getName().c_str(),
+                field.getId(Document::getNewestSerializationVersion()));
+            _cache.push_back(Field::CSP(new Field(field)));
+        } else {
+            _cache.push_back(Field::CSP());
+        }
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.h b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.h
new file mode 100644
index 00000000000..b8264158ca4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/searchsummary/docsummary/resultclass.h>
+
+namespace proton {
+
+/**
+ * A cache of document::Field instances that is associated
+ * with a summary result class.
+ **/
+class FieldCache
+{
+private:
+    typedef std::vector<document::Field::CSP> Cache;
+
+    Cache _cache;
+
+public:
+    typedef std::shared_ptr<const FieldCache> CSP;
+
+    FieldCache();
+
+    FieldCache(const search::docsummary::ResultClass &resClass,
+               const document::DocumentType &docType);
+
+    size_t size() const { return _cache.size(); }
+
+    const document::Field *getField(size_t idx) const {
+        return _cache[idx].get();
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp
new file mode 100644
index 00000000000..a38bee61480
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.fieldcacherepo");
+#include "fieldcacherepo.h"
+
+using namespace document;
+using namespace search::docsummary;
+
+namespace proton {
+
+FieldCacheRepo::FieldCacheRepo() :
+    _repo(),
+    _defaultCache(new FieldCache())
+{
+}
+
+FieldCacheRepo::FieldCacheRepo(const ResultConfig &resConfig,
+                               const DocumentType &docType) :
+    _repo(),
+    _defaultCache(new FieldCache())
+{
+    for (ResultConfig::const_iterator it(resConfig.begin()), mt(resConfig.end()); it != mt; it++) {
+        FieldCache::CSP cache(new FieldCache(*it, docType));
+        vespalib::string className(it->GetClassName());
+        LOG(debug, "Adding field cache for summary class '%s' to repo",
+            className.c_str());
+        _repo.insert(std::make_pair(className, cache));
+    }
+    const ResultClass *defaultClass = resConfig.LookupResultClass(resConfig.LookupResultClassId(""));
+    if (defaultClass != NULL) {
+        _defaultCache = getFieldCache(defaultClass->GetClassName());
+    }
+}
+
+FieldCache::CSP
+FieldCacheRepo::getFieldCache(const vespalib::string &resultClass) const
+{
+    Repo::const_iterator itr = _repo.find(resultClass);
+    if (itr != _repo.end()) {
+        return itr->second;
+    }
+    return _defaultCache;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.h b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.h
new file mode 100644
index 00000000000..e9c1cd971bc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/searchsummary/docsummary/resultconfig.h>
+#include "fieldcache.h"
+
+namespace proton {
+
+/**
+ * A repository of FieldCache instances,
+ * one for each summary result class that we have in the summary result config.
+ **/
+class FieldCacheRepo
+{
+private:
+    typedef std::map<vespalib::string, FieldCache::CSP> Repo;
+
+    Repo _repo;
+    FieldCache::CSP _defaultCache; // corresponds to the default summary class
+
+public:
+    typedef std::unique_ptr<FieldCacheRepo> UP;
+
+    FieldCacheRepo();
+
+    FieldCacheRepo(const search::docsummary::ResultConfig &resConfig,
+                   const document::DocumentType &docType);
+
+    FieldCache::CSP getFieldCache(const vespalib::string &resultClass) const;
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h
new file mode 100644
index 00000000000..b9f105291fc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchsummary/config/config-juniperrc.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/searchsummary/docsummary/docsumstore.h>
+#include <vespa/searchsummary/docsummary/docsumwriter.h>
+#include <vespa/searchsummary/docsummary/idocsumenvironment.h>
+#include <vespa/searchsummary/docsummary/resultconfig.h>
+#include <vespa/config-summary.h>
+#include <vespa/config-summarymap.h>
+
+namespace proton {
+
+/**
+ * Interface for a summary manager.
+ */
+class ISummaryManager : public boost::noncopyable
+{
+public:
+    /**
+     * Interface for a summary setup.
+     */
+    class ISummarySetup : public search::docsummary::IDocsumEnvironment {
+    public:
+        typedef std::unique_ptr<ISummarySetup> UP;
+        typedef std::shared_ptr<ISummarySetup> SP;
+
+        virtual ~ISummarySetup() {}
+
+        virtual search::docsummary::IDocsumWriter &getDocsumWriter() const = 0;
+        virtual search::docsummary::ResultConfig &getResultConfig() = 0;
+        virtual search::docsummary::IDocsumStore::UP createDocsumStore(const vespalib::string &resultClassName) = 0;
+
+        // Inherit doc from IDocsumEnvironment
+        virtual search::IAttributeManager *getAttributeManager() = 0;
+        virtual vespalib::string lookupIndex(const vespalib::string & s) const = 0;
+        virtual juniper::Juniper *getJuniper() = 0;
+    };
+
+    typedef std::unique_ptr<ISummaryManager> UP;
+    typedef std::shared_ptr<ISummaryManager> SP;
+
+    virtual ~ISummaryManager() {}
+
+    virtual ISummarySetup::SP
+    createSummarySetup(const vespa::config::search::SummaryConfig &summaryCfg,
+                       const vespa::config::search::SummarymapConfig &summarymapCfg,
+                       const vespa::config::search::summary::JuniperrcConfig &juniperCfg,
+                       const document::DocumentTypeRepo::SP &repo,
+                       const std::shared_ptr<search::IAttributeManager> &attributeMgr) = 0;
+
+    virtual search::IDocumentStore &getBackingStore() = 0;
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/linguisticsannotation.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/linguisticsannotation.cpp
new file mode 100644
index 00000000000..47f046bfd42
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/linguisticsannotation.cpp
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.linguisticsannotation");
+
+#include "linguisticsannotation.h"
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/datatype/primitivedatatype.h>
+
+using document::AnnotationType;
+using document::DataType;
+using document::PrimitiveDataType;
+using vespalib::string;
+
+namespace proton {
+namespace linguistics {
+namespace {
+AnnotationType makeType(int id, string name, const DataType &type) {
+    AnnotationType annotation_type(id, name);
+    annotation_type.setDataType(type);
+    return annotation_type;
+}
+
+const PrimitiveDataType STRING_OBJ(DataType::T_STRING);
+AnnotationType TERM_OBJ(makeType(1, "term", STRING_OBJ));
+}  // namespace
+
+const string SPANTREE_NAME("linguistics");
+const AnnotationType *const TERM(&TERM_OBJ);
+
+}  // namespace linguistics
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/linguisticsannotation.h b/searchcore/src/vespa/searchcore/proton/docsummary/linguisticsannotation.h
new file mode 100644
index 00000000000..ef9d3cc29f2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/linguisticsannotation.h
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/datatype/annotationtype.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+namespace linguistics {
+
+extern const vespalib::string SPANTREE_NAME;
+extern const document::AnnotationType *const TERM;
+
+}  // namespace linguistics;
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/searchdatatype.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/searchdatatype.cpp
new file mode 100644
index 00000000000..c2b7dc68b5c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/searchdatatype.cpp
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.searchdatatype");
+
+#include "searchdatatype.h"
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/document/datatype/structdatatype.h>
+
+using document::DataType;
+using document::Field;
+using document::PrimitiveDataType;
+using document::StructDataType;
+
+namespace proton {
+
+namespace {
+
+PrimitiveDataType STRING_OBJ(DataType::T_STRING);
+StructDataType URI_OBJ("url");
+
+const StructDataType *setUpUriType() {
+    URI_OBJ.addField(Field("all", STRING_OBJ, true));
+    URI_OBJ.addField(Field("scheme", STRING_OBJ, true));
+    URI_OBJ.addField(Field("host", STRING_OBJ, true));
+    URI_OBJ.addField(Field("port", STRING_OBJ, true));
+    URI_OBJ.addField(Field("path", STRING_OBJ, true));
+    URI_OBJ.addField(Field("query", STRING_OBJ, true));
+    URI_OBJ.addField(Field("fragment", STRING_OBJ, true));
+    return &URI_OBJ;
+}
+}  // namespace
+
+const DataType *SearchDataType::URI(setUpUriType());
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/searchdatatype.h b/searchcore/src/vespa/searchcore/proton/docsummary/searchdatatype.h
new file mode 100644
index 00000000000..7bc8875584d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/searchdatatype.h
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/datatype/datatype.h>
+
+namespace proton {
+
+struct SearchDataType {
+    static const document::DataType *URI;
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp
new file mode 100644
index 00000000000..fdbe60eb762
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.summarycompacttarget");
+
+#include "summarycompacttarget.h"
+#include <vespa/searchlib/util/dirtraverse.h>
+
+using search::IDocumentStore;
+using search::SerialNum;
+
+namespace proton {
+
+namespace {
+
+class Compacter : public searchcorespi::FlushTask {
+private:
+    IDocumentStore & _docStore;
+    FlushStats     & _stats;
+    SerialNum        _currSerial;
+public:
+    Compacter(IDocumentStore & docStore, FlushStats & stats, SerialNum currSerial) :
+        _docStore(docStore), _stats(stats), _currSerial(currSerial) {}
+    void run() override {
+        _docStore.compact(_currSerial);
+        updateStats();
+    }
+    void updateStats() {
+        // the target must live until this task is done (handled by flush engine).
+        _stats.setPath(_docStore.getBaseDir());
+    }
+
+    SerialNum getFlushSerial() const override {
+        return 0u; // Zero means no sync of transaction log is needed
+    }
+};
+
+}
+
+SummaryCompactTarget::SummaryCompactTarget(IDocumentStore & docStore)
+    : IFlushTarget("summary.compact", Type::GC, Component::DOCUMENT_STORE),
+      _docStore(docStore),
+      _lastStats()
+{
+    _lastStats.setPathElementsToLog(6);
+}
+
+IFlushTarget::MemoryGain
+SummaryCompactTarget::getApproxMemoryGain() const
+{
+    return MemoryGain::noGain(_docStore.memoryUsed());
+}
+
+IFlushTarget::DiskGain
+SummaryCompactTarget::getApproxDiskGain() const
+{
+    uint64_t total(_docStore.getDiskFootprint());
+    return DiskGain(total, total - std::min(total, _docStore.getMaxCompactGain()));
+}
+
+IFlushTarget::Time
+SummaryCompactTarget::getLastFlushTime() const
+{
+    return fastos::ClockSystem::now();
+}
+
+SerialNum
+SummaryCompactTarget::getFlushedSerialNum() const
+{
+    return _docStore.tentativeLastSyncToken();
+}
+
+IFlushTarget::Task::UP
+SummaryCompactTarget::initFlush(SerialNum currentSerial)
+{
+    return Task::UP(new Compacter(_docStore, _lastStats, currentSerial));
+}
+
+uint64_t
+SummaryCompactTarget::getApproxBytesToWriteToDisk() const
+{
+    return 0;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h
new file mode 100644
index 00000000000..e6e78a5f871
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+
+namespace proton {
+
+using searchcorespi::FlushStats;
+using searchcorespi::IFlushTarget;
+
+/**
+ * This class implements the IFlushTarget interface to proxy a summary manager.
+ */
+class SummaryCompactTarget : public IFlushTarget {
+private:
+    search::IDocumentStore & _docStore;
+    FlushStats _lastStats;
+
+public:
+    SummaryCompactTarget(search::IDocumentStore & docStore);
+
+    // Implements IFlushTarget
+    virtual MemoryGain getApproxMemoryGain() const;
+    virtual   DiskGain getApproxDiskGain() const;
+    virtual  SerialNum getFlushedSerialNum() const;
+    virtual       Time getLastFlushTime() const;
+
+    virtual Task::UP initFlush(SerialNum currentSerial);
+
+    virtual FlushStats getLastFlushStats() const { return _lastStats; }
+    virtual uint64_t getApproxBytesToWriteToDisk() const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.cpp
new file mode 100644
index 00000000000..58839cd7f04
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.cpp
@@ -0,0 +1,689 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.summaryfieldconverter");
+
+#include "summaryfieldconverter.h"
+
+#include "linguisticsannotation.h"
+#include "searchdatatype.h"
+#include <vespa/document/annotation/alternatespanlist.h>
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/document/annotation/span.h>
+#include <vespa/document/annotation/spanlist.h>
+#include <vespa/document/annotation/spannode.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/annotation/spantreevisitor.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/fieldvaluevisitor.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+#include <vespa/document/fieldvalue/shortfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/annotationreferencefieldvalue.h>
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchlib/util/url.h>
+#include <vespa/vespalib/encoding/base64.h>
+#include <vespa/vespalib/geo/zcurve.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/jsonwriter.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/data/slime/convenience.h>
+#include <vespa/vespalib/data/slime/binary_format.h>
+#include <vespa/vespalib/data/slime/json_format.h>
+#include <vespa/vespalib/tensor/serialization/slime_binary_format.h>
+#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
+
+
+using document::AlternateSpanList;
+using document::Annotation;
+using document::AnnotationReferenceFieldValue;
+using document::ArrayDataType;
+using document::ArrayFieldValue;
+using document::ByteFieldValue;
+using document::DataType;
+using document::Document;
+using document::DocumentType;
+using document::DoubleFieldValue;
+using document::FieldValue;
+using document::FixedTypeRepo;
+using document::ConstFieldValueVisitor;
+using document::FloatFieldValue;
+using document::IntFieldValue;
+using document::LongFieldValue;
+using document::MapFieldValue;
+using document::PredicateFieldValue;
+using document::RawFieldValue;
+using document::ShortFieldValue;
+using document::Span;
+using document::SpanList;
+using document::SimpleSpanList;
+using document::SpanNode;
+using document::SpanTree;
+using document::SpanTreeVisitor;
+using document::StringFieldValue;
+using document::StructFieldValue;
+using document::WeightedSetDataType;
+using document::WeightedSetFieldValue;
+using document::TensorFieldValue;
+using search::index::Schema;
+using search::util::URL;
+using std::make_pair;
+using std::pair;
+using std::vector;
+using vespalib::JSONWriter;
+using vespalib::asciistream;
+using vespalib::geo::ZCurve;
+using vespalib::make_string;
+using vespalib::string;
+using vespalib::stringref;
+
+namespace proton {
+
+namespace {
+string getSpanString(const string &s, const Span &span) {
+    return string(&s[span.from()], &s[span.from() + span.length()]);
+}
+
+struct SpanFinder : SpanTreeVisitor {
+    int32_t begin_pos;
+    int32_t end_pos;
+
+    SpanFinder() : begin_pos(0x7fffffff), end_pos(-1) {}
+    Span span() { return Span(begin_pos, end_pos - begin_pos); }
+
+    void visit(const Span &node) override {
+        begin_pos = std::min(begin_pos, node.from());
+        end_pos = std::max(end_pos, node.from() + node.length());
+    }
+    void visit(const SpanList &node) override {
+        for (const auto & span_ : node) {
+            span_->accept(*this);
+        }
+    }
+    void visit(const SimpleSpanList &node) override {
+        for (const auto & span_ : node) {
+            span_.accept(*this);
+        }
+    }
+    void visit(const AlternateSpanList &node) override {
+        for (size_t i = 0; i < node.getNumSubtrees(); ++i) {
+            visit(node.getSubtree(i));
+        }
+    }
+};
+
+Span getSpan(const SpanNode &span_node) {
+    SpanFinder finder;
+    span_node.accept(finder);
+    return finder.span();
+}
+
+// Extract the FieldValues from all TERM annotations. For each span
+// with such annotations, the Handler is invoked with a set of
+// iterators over the FieldValues for that span.
+template <typename Handler>
+void handleIndexingTerms(Handler &handler, const StringFieldValue &value) {
+        StringFieldValue::SpanTrees trees = value.getSpanTrees();
+    const SpanTree *tree = StringFieldValue::findTree(trees, linguistics::SPANTREE_NAME);
+    typedef pair<Span, const FieldValue *> SpanTerm;
+    typedef vector<SpanTerm> SpanTermVector;
+    if (!tree) {
+        // Treat a string without annotations as a single span.
+        SpanTerm str(Span(0, handler.text.size()),
+                     static_cast<const FieldValue*>(0));
+        handler.handleAnnotations(str.first, &str, &str + 1);
+        return;
+    }
+    SpanTermVector terms;
+    for (const Annotation & annotation : *tree) {
+        // For now, skip any composite spans.
+        const Span *span = dynamic_cast<const Span*>(annotation.getSpanNode());
+        if ((span != nullptr) && annotation.valid() &&
+            (annotation.getType() == *linguistics::TERM)) {
+            terms.push_back(make_pair(getSpan(*span),
+                                      annotation.getFieldValue()));
+        }
+    }
+    sort(terms.begin(), terms.end());
+    SpanTermVector::const_iterator it = terms.begin();
+    SpanTermVector::const_iterator ite = terms.end();
+    int32_t endPos = 0;
+    for (; it != ite; ) {
+        SpanTermVector::const_iterator it_begin = it;
+        if (it_begin->first.from() >  endPos) {
+            Span tmpSpan(endPos, it_begin->first.from() - endPos);
+            handler.handleAnnotations(tmpSpan, it, it);
+            endPos = it_begin->first.from();
+        }
+        for (; it != ite && it->first == it_begin->first; ++it);
+        handler.handleAnnotations(it_begin->first, it_begin, it);
+        endPos = it_begin->first.from() + it_begin->first.length();
+    }
+    int32_t wantEndPos = handler.text.size();
+    if (endPos < wantEndPos) {
+        Span tmpSpan(endPos, wantEndPos - endPos);
+        handler.handleAnnotations(tmpSpan, ite, ite);
+    }
+}
+
+const StringFieldValue &ensureStringFieldValue(const FieldValue &value) __attribute__((noinline));
+
+const StringFieldValue &ensureStringFieldValue(const FieldValue &value) {
+    if (!value.inherits(IDENTIFIABLE_CLASSID(StringFieldValue))) {
+        throw vespalib::IllegalArgumentException("Illegal field type. " + value.toString(), VESPA_STRLOC);
+    }
+    return static_cast<const StringFieldValue &>(value);
+}
+
+struct FieldValueConverter {
+    virtual FieldValue::UP convert(const FieldValue &input) = 0;
+    virtual ~FieldValueConverter() {}
+};
+
+
+struct SummaryHandler {
+    const string text;
+    asciistream &out;
+
+    SummaryHandler(const string &s, asciistream &stream)
+        : text(s), out(stream) {}
+
+    template <typename ForwardIt>
+    void handleAnnotations(const Span &span, ForwardIt it, ForwardIt last) {
+        int annCnt = (last - it);
+        if (annCnt > 1 || (annCnt == 1 && it->second)) {
+            annotateSpans(span, it, last);
+        } else {
+            out << getSpanString(text, span) << '\037';
+        }
+    }
+
+    template <typename ForwardIt>
+    void annotateSpans(const Span &span, ForwardIt it, ForwardIt last) {
+        out << "\357\277\271"  // ANCHOR
+            << (getSpanString(text, span))
+            << "\357\277\272"; // SEPARATOR
+        while (it != last) {
+            if (it->second) {
+                out << ensureStringFieldValue(*it->second).getValue();
+            } else {
+                out << getSpanString(text, span);
+            }
+            if (++it != last) {
+                out << " ";
+            }
+        }
+        out << "\357\277\273"  // TERMINATOR
+            << "\037";
+    }
+};
+
+
+
+class JsonFiller : public ConstFieldValueVisitor {
+    JSONWriter           &_json;
+    bool                  _tokenize;
+
+    virtual void visit(const AnnotationReferenceFieldValue & v ) {
+        (void)v;
+        _json.beginObject();
+        _json.appendKey("error");
+        _json.appendString("cannot convert from annotation reference field");
+        _json.endObject();
+    }
+    virtual void visit(const Document & v) {
+        (void)v;
+        _json.beginObject();
+        _json.appendKey("error");
+        _json.appendString("cannot convert from field of type document");
+        _json.endObject();
+    }
+
+    virtual void visit(const MapFieldValue & v) {
+        _json.beginArray();
+        for (const auto & entry : v) {
+            _json.beginObject();
+
+            _json.appendKey("key");
+            const FieldValue &key = *(entry.first);
+            key.accept(*this);
+
+            const FieldValue &val = *(entry.second);
+            _json.appendKey("value");
+            val.accept(*this);
+
+            _json.endObject();
+        }
+        _json.endArray();
+    }
+
+    virtual void visit(const ArrayFieldValue &value) {
+        _json.beginArray();
+        if (value.size() > 0) {
+            for (const FieldValue &fv : value) {
+                fv.accept(*this);
+            }
+        }
+        _json.endArray();
+    }
+
+    virtual void visit(const StringFieldValue &value) {
+        if (_tokenize) {
+            asciistream tmp;
+            SummaryHandler handler(value.getValue(), tmp);
+            handleIndexingTerms(handler, value);
+            _json.appendString(tmp.str());
+        } else {
+            _json.appendString(value.getValue());
+        }
+    }
+
+    virtual void visit(const IntFieldValue &value) {
+        int32_t v = value.getValue(); _json.appendInt64(v);
+    }
+    virtual void visit(const LongFieldValue &value) {
+        int64_t v = value.getValue(); _json.appendInt64(v);
+    }
+    virtual void visit(const ShortFieldValue &value) {
+        int16_t v = value.getValue(); _json.appendInt64(v);
+    }
+    virtual void visit(const ByteFieldValue &value) {
+        int8_t v = value.getAsByte(); _json.appendInt64(v);
+    }
+    virtual void visit(const DoubleFieldValue &value) {
+        double v = value.getValue(); _json.appendDouble(v);
+    }
+    virtual void visit(const FloatFieldValue &value) {
+        float v = value.getValue(); _json.appendFloat(v);
+    }
+
+    virtual void
+    visit(const PredicateFieldValue &value)
+    {
+        _json.appendJSON(value.toString());
+    }
+
+    virtual void
+    visit(const RawFieldValue &value)
+    {
+        // Use base64 coding to represent raw values in json strings.
+        std::pair<const char *, size_t> buf = value.getAsRaw();
+        vespalib::string rawVal(buf.first, buf.first + buf.second);
+        _json.appendString(vespalib::Base64::encode(rawVal));
+    }
+
+    virtual void visit(const StructFieldValue &value) {
+        // stringref type_name = value.getDataType()->getName();
+        if (*value.getDataType() == *SearchDataType::URI) {
+            FieldValue::UP uriAllValue = value.getValue("all");
+            if (uriAllValue.get() != NULL &&
+                uriAllValue->inherits(IDENTIFIABLE_CLASSID(StringFieldValue)))
+            {
+                uriAllValue->accept(*this);
+                return;
+            }
+        }
+        _json.beginObject();
+        for (StructFieldValue::const_iterator itr = value.begin(); itr != value.end(); ++itr) {
+            _json.appendKey(itr.field().getName());
+            FieldValue::UP nextValue(value.getValue(itr.field()));
+            (*nextValue).accept(*this);
+        }
+        _json.endObject();
+    }
+
+    virtual void visit(const WeightedSetFieldValue &value) {
+        _json.beginArray();
+        if ( value.size() > 0) {
+            for (const auto & entry : value) {
+                _json.beginObject();
+                _json.appendKey("item");
+                entry.first->accept(*this);
+                _json.appendKey("weight");
+                int weight = static_cast<const IntFieldValue &>(*entry.second).getValue();
+                _json.appendInt64(weight);
+                _json.endObject();
+            }
+        }
+        _json.endArray();
+    }
+
+    virtual void visit(const TensorFieldValue &value) override {
+        const auto &tensor = value.getAsTensorPtr();
+        if (tensor) {
+            auto slime =
+                vespalib::tensor::SlimeBinaryFormat::serialize(*tensor);
+            vespalib::slime::SimpleBuffer buf;
+            vespalib::slime::JsonFormat::encode(*slime, buf, true);
+            _json.appendJSON(buf.get().make_string());
+        } else {
+            // No tensor value => empty object
+            _json.beginObject();
+            _json.endObject();
+        }
+    }
+
+public:
+    JsonFiller(bool markup, JSONWriter &json)
+        : _json(json), _tokenize(markup) {}
+};
+
+class JsonConverter : public FieldValueConverter {
+    bool                  _tokenize;
+public:
+    JsonConverter(bool tokenize)
+        : _tokenize(tokenize)
+    {}
+
+    FieldValue::UP convert(const FieldValue &input) {
+        asciistream target;
+        JSONWriter json(target);
+        JsonFiller visitor(_tokenize, json);
+        input.accept(visitor);
+        return FieldValue::UP(new StringFieldValue(target.str()));
+    }
+
+};
+
+class SummaryFieldValueConverter : protected ConstFieldValueVisitor
+{
+    asciistream          _str;
+    bool                 _tokenize;
+    FieldValue::UP       _field_value;
+    FieldValueConverter &_structuredFieldConverter;
+
+    virtual void visit(const ArrayFieldValue &value) {
+        _field_value = _structuredFieldConverter.convert(value);
+    }
+
+    template <typename T>
+    void visitPrimitive(const T &t) {
+        _field_value.reset(t.clone());
+    }
+    virtual void visit(const IntFieldValue &value) { visitPrimitive(value); }
+    virtual void visit(const LongFieldValue &value) { visitPrimitive(value); }
+    virtual void visit(const ShortFieldValue &value) { visitPrimitive(value); }
+    virtual void visit(const ByteFieldValue &value) {
+        int8_t signedValue = value.getAsByte();
+        _field_value.reset(new ShortFieldValue(signedValue));
+    }
+    virtual void visit(const DoubleFieldValue &value) { visitPrimitive(value); }
+    virtual void visit(const FloatFieldValue &value) { visitPrimitive(value); }
+
+    virtual void visit(const StringFieldValue &value) {
+        if (_tokenize) {
+            SummaryHandler handler(value.getValue(), _str);
+            handleIndexingTerms(handler, value);
+        } else {
+            _str << value.getValue();
+        }
+    }
+
+    virtual void visit(const AnnotationReferenceFieldValue & v ) {
+        _field_value = _structuredFieldConverter.convert(v);
+    }
+    virtual void visit(const Document & v) {
+        _field_value = _structuredFieldConverter.convert(v);
+    }
+
+    virtual void
+    visit(const PredicateFieldValue &value)
+    {
+        _str << value.toString();
+    }
+
+    virtual void
+    visit(const RawFieldValue &value)
+    {
+        visitPrimitive(value);
+    }
+
+    virtual void visit(const MapFieldValue & v) {
+        _field_value = _structuredFieldConverter.convert(v);
+    }
+
+    virtual void visit(const StructFieldValue &value) {
+        if (*value.getDataType() == *SearchDataType::URI) {
+            FieldValue::UP uriAllValue = value.getValue("all");
+            if (uriAllValue.get() != NULL &&
+                uriAllValue->inherits(IDENTIFIABLE_CLASSID(StringFieldValue)))
+            {
+                uriAllValue->accept(*this);
+                return;
+            }
+        }
+        _field_value = _structuredFieldConverter.convert(value);
+    }
+
+    virtual void visit(const WeightedSetFieldValue &value) {
+        _field_value = _structuredFieldConverter.convert(value);
+    }
+
+    virtual void visit(const TensorFieldValue &value) override {
+        _field_value = _structuredFieldConverter.convert(value);
+    }
+
+public:
+    SummaryFieldValueConverter(bool tokenize, FieldValueConverter &subConverter)
+        : _str(), _tokenize(tokenize),
+          _structuredFieldConverter(subConverter)
+    {}
+
+    FieldValue::UP convert(const FieldValue &input) {
+        input.accept(*this);
+        if (_field_value.get()) {
+            return std::move(_field_value);
+        }
+        return FieldValue::UP(new StringFieldValue(_str.str()));
+    }
+};
+
+
+
+using namespace vespalib::slime::convenience;
+
+class SlimeFiller : public ConstFieldValueVisitor {
+    Inserter    &_inserter;
+    bool         _tokenize;
+
+    virtual void visit(const AnnotationReferenceFieldValue & v ) {
+        (void)v;
+        Cursor &c = _inserter.insertObject();
+        Memory key("error");
+        Memory val("cannot convert from annotation reference field");
+        c.setString(key, val);
+    }
+    virtual void visit(const Document & v) {
+        (void)v;
+        Cursor &c = _inserter.insertObject();
+        Memory key("error");
+        Memory val("cannot convert from field of type document");
+        c.setString(key, val);
+    }
+
+    virtual void visit(const MapFieldValue & v) {
+        Cursor &a = _inserter.insertArray();
+        Memory keymem("key");
+        Memory valmem("value");
+        for (const auto & entry : v) {
+            Cursor &c = a.addObject();
+            ObjectInserter ki(c, keymem);
+            ObjectInserter vi(c, valmem);
+            SlimeFiller keyConv(ki, _tokenize);
+            SlimeFiller valConv(vi, _tokenize);
+
+            const FieldValue &key = *(entry.first);
+            key.accept(keyConv);
+            const FieldValue &val = *(entry.second);
+            val.accept(valConv);
+        }
+    }
+
+    virtual void visit(const ArrayFieldValue &value) {
+        Cursor &a = _inserter.insertArray();
+        if (value.size() > 0) {
+            ArrayInserter ai(a);
+            SlimeFiller conv(ai, _tokenize);
+            for (const FieldValue &fv : value) {
+                fv.accept(conv);
+            }
+        }
+    }
+
+    virtual void visit(const StringFieldValue &value) {
+        if (_tokenize) {
+            asciistream tmp;
+            SummaryHandler handler(value.getValue(), tmp);
+            handleIndexingTerms(handler, value);
+            _inserter.insertString(Memory(tmp.str()));
+        } else {
+            _inserter.insertString(Memory(value.getValue()));
+        }
+    }
+
+    virtual void visit(const IntFieldValue &value) {
+        int32_t v = value.getValue();
+        _inserter.insertLong(v);
+    }
+    virtual void visit(const LongFieldValue &value) {
+        int64_t v = value.getValue();
+        _inserter.insertLong(v);
+    }
+    virtual void visit(const ShortFieldValue &value) {
+        int16_t v = value.getValue();
+        _inserter.insertLong(v);
+    }
+    virtual void visit(const ByteFieldValue &value) {
+        int8_t v = value.getAsByte();
+        _inserter.insertLong(v);
+    }
+    virtual void visit(const DoubleFieldValue &value) {
+        double v = value.getValue();
+        _inserter.insertDouble(v);
+    }
+    virtual void visit(const FloatFieldValue &value) {
+        float v = value.getValue();
+        _inserter.insertDouble(v);
+    }
+
+    virtual void
+    visit(const PredicateFieldValue &value)
+    {
+        vespalib::slime::inject(value.getSlime().get(), _inserter);
+    }
+
+    virtual void
+    visit(const RawFieldValue &value)
+    {
+        // Use base64 coding to represent raw values
+        std::pair<const char *, size_t> buf = value.getAsRaw();
+        vespalib::string rawVal(buf.first, buf.first + buf.second);
+        vespalib::string encVal(vespalib::Base64::encode(rawVal));
+        _inserter.insertString(Memory(encVal.c_str()));
+    }
+
+    virtual void visit(const StructFieldValue &value) {
+        if (*value.getDataType() == *SearchDataType::URI) {
+            FieldValue::UP uriAllValue = value.getValue("all");
+            if (uriAllValue.get() != NULL &&
+                uriAllValue->inherits(IDENTIFIABLE_CLASSID(StringFieldValue)))
+            {
+                uriAllValue->accept(*this);
+                return;
+            }
+        }
+        Cursor &c = _inserter.insertObject();
+        for (StructFieldValue::const_iterator itr = value.begin(); itr != value.end(); ++itr) {
+            Memory keymem(itr.field().getName());
+            ObjectInserter vi(c, keymem);
+            SlimeFiller conv(vi, _tokenize);
+            FieldValue::UP nextValue(value.getValue(itr.field()));
+            (*nextValue).accept(conv);
+        }
+    }
+
+    virtual void visit(const WeightedSetFieldValue &value) {
+        Cursor &a = _inserter.insertArray();
+        if (value.size() > 0) {
+            Memory imem("item");
+            Memory wmem("weight");
+            for (const auto & entry : value) {
+                Cursor &o = a.addObject();
+                ObjectInserter ki(o, imem);
+                SlimeFiller conv(ki, _tokenize);
+                entry.first->accept(conv);
+                int weight = static_cast<const IntFieldValue &>(*entry.second).getValue();
+                o.setLong(wmem, weight);
+            }
+        }
+    }
+
+    virtual void visit(const TensorFieldValue &value) override {
+        const auto &tensor = value.getAsTensorPtr();
+        if (tensor) {
+            vespalib::tensor::SlimeBinaryFormat::serialize(_inserter, *tensor);
+        } else {
+            // No tensor value => empty object
+            _inserter.insertObject();
+        }
+    }
+
+public:
+    SlimeFiller(Inserter &inserter, bool tokenize)
+        : _inserter(inserter), _tokenize(tokenize) {}
+};
+
+class SlimeConverter : public FieldValueConverter {
+    bool _tokenize;
+public:
+    SlimeConverter(bool tokenize)
+        : _tokenize(tokenize)
+    {}
+
+    FieldValue::UP convert(const FieldValue &input) {
+        vespalib::Slime slime;
+        SlimeInserter inserter(slime);
+        SlimeFiller visitor(inserter, _tokenize);
+        input.accept(visitor);
+        search::RawBuf rbuf(4096);
+        search::SlimeOutputRawBufAdapter adapter(rbuf);
+        vespalib::slime::BinaryFormat::encode(slime, adapter);
+        return FieldValue::UP(new RawFieldValue(rbuf.GetDrainPos(), rbuf.GetUsedLen()));
+    }
+};
+
+
+}  // namespace
+
+FieldValue::UP
+SummaryFieldConverter::convertSummaryField(bool markup,
+                                           const FieldValue &value,
+                                           bool useSlimeInsideFields)
+{
+    if (useSlimeInsideFields) {
+        SlimeConverter subConv(markup);
+        return SummaryFieldValueConverter(markup, subConv).convert(value);
+    } else {
+        JsonConverter subConv(markup);
+        return SummaryFieldValueConverter(markup, subConv).convert(value);
+    }
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.h b/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.h
new file mode 100644
index 00000000000..9507b8cab8d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.h
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/fieldvalue.h>
+
+namespace proton {
+
+/**
+ * This class converts a summary field for docsum fetching.
+ */
+class SummaryFieldConverter
+{
+public:
+    static document::FieldValue::UP
+    convertSummaryField(bool markup, const document::FieldValue &value, bool useSlimeInsideFields);
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp
new file mode 100644
index 00000000000..33bf18fea12
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.summaryflushtarget");
+
+#include "summaryflushtarget.h"
+#include <vespa/searchlib/util/dirtraverse.h>
+
+using search::IDocumentStore;
+using search::SerialNum;
+
+namespace proton {
+
+namespace {
+
+class Flusher : public searchcorespi::FlushTask {
+private:
+    IDocumentStore & _docStore;
+    FlushStats     & _stats;
+    SerialNum        _currSerial;
+public:
+    Flusher(IDocumentStore & docStore,
+            FlushStats & stats,
+            SerialNum currSerial)
+        : _docStore(docStore),
+          _stats(stats),
+          _currSerial(currSerial)
+    {
+        _currSerial = _docStore.initFlush(currSerial);
+    }
+    virtual void run() {
+        _docStore.flush(_currSerial);
+        updateStats();
+    }
+    void updateStats() {
+        // the target must live until this task is done (handled by flush engine).
+        _stats.setPath(_docStore.getBaseDir());
+    }
+
+    virtual SerialNum
+    getFlushSerial(void) const
+    {
+        return _currSerial;
+    }
+};
+
+}
+
+SummaryFlushTarget::SummaryFlushTarget(IDocumentStore & docStore)
+    : IFlushTarget("summary.flush", Type::SYNC, Component::DOCUMENT_STORE),
+      _docStore(docStore),
+      _lastStats()
+{
+    _lastStats.setPathElementsToLog(6);
+}
+
+IFlushTarget::MemoryGain
+SummaryFlushTarget::getApproxMemoryGain() const
+{
+    return MemoryGain(_docStore.memoryUsed(), _docStore.memoryMeta());
+}
+
+IFlushTarget::DiskGain
+SummaryFlushTarget::getApproxDiskGain() const
+{
+    return DiskGain(0, 0);
+}
+
+IFlushTarget::Time
+SummaryFlushTarget::getLastFlushTime() const
+{
+    return _docStore.getLastFlushTime();
+}
+
+SerialNum
+SummaryFlushTarget::getFlushedSerialNum() const
+{
+    return _docStore.lastSyncToken();
+}
+
+IFlushTarget::Task::UP
+SummaryFlushTarget::initFlush(SerialNum currentSerial)
+{
+    return Task::UP(new Flusher(_docStore, _lastStats, currentSerial));
+}
+
+uint64_t
+SummaryFlushTarget::getApproxBytesToWriteToDisk() const
+{
+    return 0;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h
new file mode 100644
index 00000000000..617a375d51c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+
+namespace proton {
+
+using searchcorespi::FlushStats;
+using searchcorespi::IFlushTarget;
+
+/**
+ * This class implements the IFlushTarget interface to proxy a summary manager.
+ */
+class SummaryFlushTarget : public IFlushTarget {
+private:
+    search::IDocumentStore & _docStore;
+    FlushStats _lastStats;
+
+public:
+    SummaryFlushTarget(search::IDocumentStore & docStore);
+
+    // Implements IFlushTarget
+    virtual MemoryGain getApproxMemoryGain() const;
+    virtual   DiskGain getApproxDiskGain() const;
+    virtual  SerialNum getFlushedSerialNum() const;
+    virtual       Time getLastFlushTime() const;
+
+    virtual Task::UP initFlush(SerialNum currentSerial);
+
+    virtual FlushStats getLastFlushStats() const { return _lastStats; }
+    virtual uint64_t getApproxBytesToWriteToDisk() const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
new file mode 100644
index 00000000000..41727b9fa2a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
@@ -0,0 +1,193 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.docsummary.summarymanager");
+#include "documentstoreadapter.h"
+#include "summarycompacttarget.h"
+#include "summaryflushtarget.h"
+#include "summarymanager.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/searchlib/docstore/logdocumentstore.h>
+#include <vespa/searchsummary/docsummary/docsumconfig.h>
+#include <vespa/config/print/ostreamconfigwriter.h>
+
+using namespace config;
+using namespace document;
+using namespace search::docsummary;
+using namespace vespa::config::search::core;
+using namespace vespa::config::search::summary;
+using namespace vespa::config::search;
+
+using search::TuneFileSummary;
+using search::common::FileHeaderContext;
+
+namespace proton {
+
+SummaryManager::SummarySetup::
+SummarySetup(const vespalib::string & baseDir,
+             const DocTypeName & docTypeName,
+             const SummaryConfig & summaryCfg,
+             const SummarymapConfig & summarymapCfg,
+             const JuniperrcConfig & juniperCfg,
+             const search::IAttributeManager::SP &attributeMgr,
+             const search::IDocumentStore::SP & docStore,
+             const DocumentTypeRepo::SP &repo)
+    : _docsumWriter(),
+      _wordFolder(),
+      _juniperProps(juniperCfg),
+      _juniperConfig(),
+      _attributeMgr(attributeMgr),
+      _docStore(docStore),
+      _fieldCacheRepo(),
+      _repo(repo),
+      _markupFields()
+{
+    std::unique_ptr<ResultConfig> resultConfig(new ResultConfig());
+    if (!resultConfig->ReadConfig(summaryCfg,
+                                  vespalib::make_string("SummaryManager(%s)",
+                                          baseDir.c_str()).c_str())) {
+        std::ostringstream oss;
+        config::OstreamConfigWriter writer(oss);
+        writer.write(summaryCfg);
+        throw vespalib::IllegalArgumentException
+            (vespalib::make_string("Could not initialize "
+                                   "summary result config for directory '%s' "
+                                   "based on summary config '%s'",
+                                   baseDir.c_str(), oss.str().c_str()));
+    }
+
+    _juniperConfig.reset(new juniper::Juniper(&_juniperProps, &_wordFolder));
+    _docsumWriter.reset(new DynamicDocsumWriter(resultConfig.release(), NULL));
+    DynamicDocsumConfig dynCfg(this, _docsumWriter.get());
+    dynCfg.configure(summarymapCfg);
+    for (size_t i = 0; i < summarymapCfg.override.size(); ++i) {
+        const SummarymapConfig::Override & o = summarymapCfg.override[i];
+        if (o.command == "dynamicteaser" ||
+            o.command == "textextractor") {
+            vespalib::string markupField = o.arguments;
+            if (markupField.empty())
+                continue;
+            // Assume just one argument: source field that must contain markup
+            _markupFields.insert(markupField);
+        }
+    }
+    const DocumentType *docType = repo->getDocumentType(docTypeName.getName());
+    if (docType != NULL) {
+        _fieldCacheRepo.reset(new FieldCacheRepo(getResultConfig(), *docType));
+    } else if (getResultConfig().GetNumResultClasses() == 0) {
+        LOG(debug, "Create empty field cache repo for document type '%s'",
+            docTypeName.toString().c_str());
+        _fieldCacheRepo.reset(new FieldCacheRepo());
+    } else {
+        throw vespalib::IllegalArgumentException
+            (vespalib::make_string("Did not find document type '%s' in current document type repo. "
+                                   "Cannot setup field cache repo for the summary setup",
+                                   docTypeName.toString().c_str()));
+    }
+}
+
+IDocsumStore::UP SummaryManager::SummarySetup::createDocsumStore(
+        const vespalib::string &resultClassName) {
+    return search::docsummary::IDocsumStore::UP(
+            new DocumentStoreAdapter(
+                    *_docStore, *_repo, getResultConfig(), resultClassName,
+                    _fieldCacheRepo->getFieldCache(resultClassName),
+                    _markupFields));
+}
+
+
+ISummaryManager::ISummarySetup::SP
+SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg,
+                                   const SummarymapConfig & summarymapCfg,
+                                   const JuniperrcConfig & juniperCfg,
+                                   const DocumentTypeRepo::SP &repo,
+                                   const search::IAttributeManager::SP &attributeMgr)
+{
+    ISummarySetup::SP newSetup(new SummarySetup(_baseDir, _docTypeName, summaryCfg,
+                                                summarymapCfg, juniperCfg,
+                                                attributeMgr, _docStore, repo));
+    return newSetup;
+}
+
+namespace {
+
+search::DocumentStore::Config getStoreConfig(const ProtonConfig::Summary::Cache & cache)
+{
+    document::CompressionConfig compression;
+    if (cache.compression.type == ProtonConfig::Summary::Cache::Compression::LZ4) {
+        compression.type = document::CompressionConfig::LZ4;
+    }
+    compression.compressionLevel = cache.compression.level;
+    return search::DocumentStore::Config(compression, cache.maxbytes, cache.initialentries);
+}
+
+}
+
+SummaryManager::SummaryManager(vespalib::ThreadStackExecutorBase & executor,
+                               const ProtonConfig::Summary & summary,
+                               const search::GrowStrategy & growStrategy,
+                               const vespalib::string &baseDir,
+                               const DocTypeName &docTypeName,
+                               const TuneFileSummary &tuneFileSummary,
+                               const FileHeaderContext &fileHeaderContext,
+                               search::transactionlog::SyncProxy &tlSyncer,
+                               const search::IBucketizer::SP & bucketizer)
+    : _baseDir(baseDir),
+      _docTypeName(docTypeName),
+      _docStore(),
+      _tuneFileSummary(tuneFileSummary),
+      _currentSerial(0u)
+{
+    search::DocumentStore::Config config(getStoreConfig(summary.cache));
+    const ProtonConfig::Summary::Log & log(summary.log);
+    const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk);
+    document::CompressionConfig compression;
+    if (chunk.compression.type == ProtonConfig::Summary::Log::Chunk::Compression::LZ4) {
+        compression.type = document::CompressionConfig::LZ4;
+    }
+    compression.compressionLevel = chunk.compression.level;
+    search::WriteableFileChunk::Config fileConfig(compression, chunk.maxbytes, chunk.maxentries);
+    search::LogDataStore::Config logConfig(log.maxfilesize,
+                                           log.maxdiskbloatfactor,
+                                           log.maxbucketspread,
+                                           log.minfilesizefactor,
+                                           log.numthreads,
+                                           log.compact2activefile,
+                                           fileConfig);
+    logConfig.disableCrcOnRead(chunk.skipcrconread);
+    _docStore.reset(
+            new search::LogDocumentStore(executor, baseDir,
+                                         search::LogDocumentStore::
+                                         Config(config, logConfig),
+                                         growStrategy,
+                                         tuneFileSummary,
+                                         fileHeaderContext,
+                                         tlSyncer,
+                                         summary.compact2buckets ? bucketizer : search::IBucketizer::SP()));
+}
+
+void
+SummaryManager::putDocument(uint64_t syncToken, const Document & doc, search::DocumentIdT lid)
+{
+    _docStore->write(syncToken, doc, lid);
+    _currentSerial = syncToken;
+}
+
+void
+SummaryManager::removeDocument(uint64_t syncToken, search::DocumentIdT lid)
+{
+    _docStore->remove(syncToken, lid);
+    _currentSerial = syncToken;
+}
+
+IFlushTarget::List SummaryManager::getFlushTargets()
+{
+    IFlushTarget::List ret;
+    ret.push_back(IFlushTarget::SP(new SummaryFlushTarget(getBackingStore())));
+    if (dynamic_cast<search::LogDocumentStore *>(_docStore.get()) != NULL) {
+        ret.push_back(IFlushTarget::SP(new SummaryCompactTarget(getBackingStore())));
+    }
+    return ret;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h
new file mode 100644
index 00000000000..5ade573887f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/utility.hpp>
+#include <vespa/document/document.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/config/config-proton.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchlib/docstore/idatastore.h>
+#include <vespa/searchlib/transactionlog/syncproxy.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include "fieldcacherepo.h"
+#include "isummarymanager.h"
+
+namespace search
+{
+
+namespace common
+{
+
+class FileHeaderContext;
+
+}
+
+}
+
+namespace proton
+{
+
+class SummaryManager : public ISummaryManager
+{
+public:
+    class SummarySetup : public ISummarySetup {
+    private:
+        std::unique_ptr<search::docsummary::DynamicDocsumWriter> _docsumWriter;
+        Fast_NormalizeWordFolder              _wordFolder;
+        search::docsummary::JuniperProperties _juniperProps;
+        std::unique_ptr<juniper::Juniper>     _juniperConfig;
+        search::IAttributeManager::SP         _attributeMgr;
+        search::IDocumentStore::SP            _docStore;
+        FieldCacheRepo::UP                    _fieldCacheRepo;
+        const document::DocumentTypeRepo::SP  _repo;
+        std::set<vespalib::string>            _markupFields;
+    public:
+        SummarySetup(const vespalib::string & baseDir,
+                     const DocTypeName & docTypeName,
+                     const vespa::config::search::SummaryConfig & summaryCfg,
+                     const vespa::config::search::SummarymapConfig & summarymapCfg,
+                     const vespa::config::search::summary::JuniperrcConfig & juniperCfg,
+                     const search::IAttributeManager::SP &attributeMgr,
+                     const search::IDocumentStore::SP & docStore,
+                     const document::DocumentTypeRepo::SP &repo);
+
+        /**
+         * Implements ISummarySetup.
+         */
+        search::docsummary::IDocsumWriter & getDocsumWriter() const { return *_docsumWriter; }
+        search::docsummary::ResultConfig & getResultConfig() { return *_docsumWriter->GetResultConfig(); }
+
+        search::docsummary::IDocsumStore::UP createDocsumStore(
+                const vespalib::string &resultClassName);
+
+        // Inherit doc from IDocsumEnvironment
+        virtual search::IAttributeManager * getAttributeManager() { return _attributeMgr.get(); }
+        virtual vespalib::string lookupIndex(const vespalib::string & s) const { (void) s; return ""; }
+        virtual juniper::Juniper * getJuniper() { return _juniperConfig.get(); }
+    };
+
+private:
+    vespalib::string                      _baseDir;
+    DocTypeName                        _docTypeName;
+    search::IDocumentStore::SP            _docStore;
+    const search::TuneFileSummary         _tuneFileSummary;
+    uint64_t 				  _currentSerial;
+
+public:
+    typedef std::shared_ptr<SummaryManager> SP;
+    SummaryManager(vespalib::ThreadStackExecutorBase & executor,
+                   const vespa::config::search::core::ProtonConfig::Summary & summary,
+                   const search::GrowStrategy & growStrategy,
+                   const vespalib::string &baseDir,
+                   const DocTypeName &docTypeName,
+                   const search::TuneFileSummary &tuneFileSummary,
+                   const search::common::FileHeaderContext &fileHeaderContext,
+                   search::transactionlog::SyncProxy &tlSyncer,
+                   const search::IBucketizer::SP & bucketizer);
+
+    void putDocument(uint64_t syncToken, const document::Document & doc,
+                     search::DocumentIdT lid);
+    void removeDocument(uint64_t syncToken, search::DocumentIdT lid);
+    IFlushTarget::List getFlushTargets();
+
+    /**
+     * Implements ISummaryManager.
+     */
+    virtual ISummarySetup::SP
+    createSummarySetup(const vespa::config::search::SummaryConfig &summaryCfg,
+                       const vespa::config::search::SummarymapConfig &summarymapCfg,
+                       const vespa::config::search::summary::JuniperrcConfig &juniperCfg,
+                       const document::DocumentTypeRepo::SP &repo,
+                       const search::IAttributeManager::SP &attributeMgr);
+
+    virtual search::IDocumentStore & getBackingStore() { return *_docStore; }
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp
new file mode 100644
index 00000000000..1a6c9c8a093
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "summarymanagerinitializer.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+namespace proton
+{
+
+SummaryManagerInitializer::
+SummaryManagerInitializer(const search::GrowStrategy &grow,
+                          const vespalib::string baseDir,
+                          const vespalib::string &subDbName,
+                          const DocTypeName &docTypeName,
+                          vespalib::ThreadStackExecutorBase &
+                          summaryExecutor,
+                          const ProtonConfig::Summary protonSummaryCfg,
+                          const search::TuneFileSummary &tuneFile,
+                          const search::common::FileHeaderContext &
+                          fileHeaderContext,
+                          search::transactionlog::SyncProxy &tlSyncer,
+                          search::IBucketizer::SP bucketizer,
+                          std::shared_ptr<SummaryManager::SP> result)
+    : proton::initializer::InitializerTask(),
+      _grow(grow),
+      _baseDir(baseDir),
+      _subDbName(subDbName),
+      _docTypeName(docTypeName),
+      _summaryExecutor(summaryExecutor),
+      _protonSummaryCfg(protonSummaryCfg),
+      _tuneFile(tuneFile),
+      _fileHeaderContext(fileHeaderContext),
+      _tlSyncer(tlSyncer),
+      _bucketizer(bucketizer),
+      _result(result)
+{
+}
+
+
+void
+SummaryManagerInitializer::run()
+{
+    vespalib::mkdir(_baseDir, false);
+    fastos::TimeStamp startTime = fastos::ClockSystem::now();
+    EventLogger::loadDocumentStoreStart(_subDbName);
+    *_result = std::make_shared<SummaryManager>
+               (_summaryExecutor,
+                _protonSummaryCfg, _grow,
+                _baseDir,
+                _docTypeName,
+                _tuneFile,
+                _fileHeaderContext,
+                _tlSyncer, _bucketizer);
+    fastos::TimeStamp endTime = fastos::ClockSystem::now();
+    int64_t elapsedTimeMs = (endTime - startTime).ms();
+    EventLogger::loadDocumentStoreComplete(_subDbName, elapsedTimeMs);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h
new file mode 100644
index 00000000000..c67b0f48a10
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "summarymanager.h"
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+#include <vespa/searchcommon/common/growstrategy.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton
+{
+
+/*
+ * Class representing an initializer task for constructing summary manager
+ * during proton startup.
+ */
+class SummaryManagerInitializer : public initializer::InitializerTask
+{
+    using ProtonConfig = vespa::config::search::core::ProtonConfig;
+    const search::GrowStrategy         _grow;
+    const vespalib::string             _baseDir;
+    const vespalib::string             _subDbName;
+    const DocTypeName                  _docTypeName;
+    vespalib::ThreadStackExecutorBase &_summaryExecutor;
+    const ProtonConfig::Summary        _protonSummaryCfg;
+    const search::TuneFileSummary      _tuneFile;
+    const search::common::FileHeaderContext &_fileHeaderContext;
+    search::transactionlog::SyncProxy &_tlSyncer;
+    const search::IBucketizer::SP      _bucketizer;
+    std::shared_ptr<SummaryManager::SP> _result;
+
+public:
+    using SP = std::shared_ptr<SummaryManagerInitializer>;
+
+    // Note: lifetime of result must be handled by caller.
+    SummaryManagerInitializer(const search::GrowStrategy &grow,
+                              const vespalib::string baseDir,
+                              const vespalib::string &subDbName,
+                              const DocTypeName &docTypeName,
+                              vespalib::ThreadStackExecutorBase &
+                              summaryExecutor,
+                              const ProtonConfig::Summary protonSummaryCfg,
+                              const search::TuneFileSummary &tuneFile,
+                              const search::common::FileHeaderContext &
+                              fileHeaderContext,
+                              search::transactionlog::SyncProxy &tlSyncer,
+                              search::IBucketizer::SP bucketizer,
+                              std::shared_ptr<SummaryManager::SP> result);
+    virtual void run() override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/documentmetastore/CMakeLists.txt
new file mode 100644
index 00000000000..ce94ca74d66
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/CMakeLists.txt
@@ -0,0 +1,22 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_documentmetastore STATIC
+    SOURCES
+    document_meta_store_explorer.cpp
+    document_meta_store_initializer_result.cpp
+    documentmetastore.cpp
+    documentmetastoreattribute.cpp
+    documentmetastorecontext.cpp
+    documentmetastoreflushtarget.cpp
+    documentmetastoreinitializer.cpp
+    documentmetastoresaver.cpp
+    search_context.cpp
+    lid_allocator.cpp
+    lid_gid_key_comparator.cpp
+    lid_reuse_delayer_config.cpp
+    lidreusedelayer.cpp
+    lidstatevector.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_bucketdb
+    searchcore_initializer
+)
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS b/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS
new file mode 100644
index 00000000000..7c03446a5d4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS
@@ -0,0 +1,2 @@
+geirst
+tegge
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_adapter.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_adapter.h
new file mode 100644
index 00000000000..974e9111306
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_adapter.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_document_meta_store.h"
+
+namespace proton {
+
+/**
+ * Class that maps functions in IDocumentMetaStore that also are found
+ * in search::AttributeVector to functions that DocumentMetaStore can implement.
+ */
+class DocumentMetaStoreAdapter : public IDocumentMetaStore
+{
+protected:
+    virtual void doCommit(search::SerialNum firstSerialNum,
+                          search::SerialNum lastSerialNum) = 0;
+    virtual DocId doGetCommittedDocIdLimit() const = 0;
+    virtual void doRemoveAllOldGenerations() = 0;
+
+public:
+    virtual void commit(search::SerialNum firstSerialNum,
+                        search::SerialNum lastSerialNum) override {
+        doCommit(firstSerialNum, lastSerialNum);
+    }
+    virtual DocId getCommittedDocIdLimit() const override {
+        return doGetCommittedDocIdLimit();
+    }
+    virtual void removeAllOldGenerations() override {
+        doRemoveAllOldGenerations();
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp
new file mode 100644
index 00000000000..bcd9302cde6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.document_meta_store_explorer");
+#include "document_meta_store_explorer.h"
+
+#include <vespa/vespalib/data/slime/cursor.h>
+
+using vespalib::slime::Cursor;
+using vespalib::slime::Inserter;
+
+namespace proton {
+
+DocumentMetaStoreExplorer::DocumentMetaStoreExplorer(IDocumentMetaStoreContext::IReadGuard::UP metaStore)
+    : _metaStore(std::move(metaStore))
+{
+}
+
+void
+DocumentMetaStoreExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    Cursor &object = inserter.insertObject();
+    if (full) {
+        search::LidUsageStats stats = _metaStore->get().getLidUsageStats();
+        object.setLong("usedLids", stats.getUsedLids());
+        object.setLong("activeLids", _metaStore->get().getNumActiveLids());
+        object.setLong("lidLimit", stats.getLidLimit());
+        object.setLong("lowestFreeLid", stats.getLowestFreeLid());
+        object.setLong("highestUsedLid", stats.getHighestUsedLid());
+        object.setLong("lidBloat", stats.getLidBloat());
+        object.setDouble("lidBloatFactor", stats.getLidBloatFactor());
+    } else {
+        object.setLong("usedLids", _metaStore->get().getNumUsedLids());
+        object.setLong("activeLids", _metaStore->get().getNumActiveLids());
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h
new file mode 100644
index 00000000000..e956b58408f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_document_meta_store_context.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of a document meta store.
+ */
+class DocumentMetaStoreExplorer : public vespalib::StateExplorer
+{
+private:
+    IDocumentMetaStoreContext::IReadGuard::UP _metaStore;
+
+public:
+    DocumentMetaStoreExplorer(IDocumentMetaStoreContext::IReadGuard::UP metaStore);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp
new file mode 100644
index 00000000000..20381e51412
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "document_meta_store_initializer_result.h"
+
+namespace proton {
+
+DocumentMetaStoreInitializerResult::
+DocumentMetaStoreInitializerResult(DocumentMetaStore::SP
+                                   documentMetaStore_in,
+                                   const search::TuneFileAttributes &
+                                   tuneFile_in)
+    : _documentMetaStore(documentMetaStore_in),
+      _tuneFile(tuneFile_in)
+{
+}
+
+
+DocumentMetaStoreInitializerResult::~DocumentMetaStoreInitializerResult()
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h
new file mode 100644
index 00000000000..aebd44b20ac
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentmetastore.h"
+#include <vespa/searchlib/common/tunefileinfo.h>
+
+namespace proton {
+
+
+/**
+ * The result after initializing document meta store component in a
+ * document sub database.
+ */
+class DocumentMetaStoreInitializerResult
+{
+private:
+    DocumentMetaStore::SP _documentMetaStore;
+    const search::TuneFileAttributes _tuneFile;
+
+public:
+    using SP = std::shared_ptr<DocumentMetaStoreInitializerResult>;
+
+    DocumentMetaStoreInitializerResult(DocumentMetaStore::SP
+                                       documentMetaStore_in,
+                                       const search::TuneFileAttributes &
+                                       tuneFile_in);
+
+    virtual ~DocumentMetaStoreInitializerResult();
+
+    DocumentMetaStore::SP documentMetaStore() const {
+        return _documentMetaStore;
+    }
+    const search::TuneFileAttributes &tuneFile() const { return _tuneFile; }
+};
+
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
new file mode 100644
index 00000000000..7b66a4b0105
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
@@ -0,0 +1,1026 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.documentmetastore");
+#include "documentmetastore.h"
+#include "search_context.h"
+
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/btree/btree.hpp>
+#include <vespa/searchlib/btree/btreenode.hpp>
+#include <vespa/searchlib/btree/btreenodestore.hpp>
+#include <vespa/searchlib/btree/btreenodeallocator.hpp>
+#include <vespa/searchlib/btree/btreeiterator.hpp>
+#include <vespa/searchlib/btree/btreeroot.hpp>
+#include <vespa/searchlib/btree/btreebuilder.hpp>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/searchcore/proton/bucketdb/bucketsessionbase.h>
+#include <vespa/searchcore/proton/bucketdb/joinbucketssession.h>
+#include <vespa/searchcore/proton/bucketdb/splitbucketsession.h>
+#include <vespa/searchlib/util/bufferwriter.h>
+#include "documentmetastoresaver.h"
+
+using document::GlobalId;
+using document::BucketId;
+using search::AttributeVector;
+using search::FileReader;
+using vespalib::GenerationHandler;
+using search::GrowStrategy;
+using search::IAttributeSaveTarget;
+using search::LidUsageStats;
+using search::MemoryUsage;
+using search::fef::TermFieldMatchData;
+using search::btree::BTreeNoLeafData;
+using search::queryeval::Blueprint;
+using search::queryeval::SearchIterator;
+using storage::spi::Timestamp;
+using proton::bucketdb::BucketState;
+using vespalib::IllegalStateException;
+using vespalib::make_string;
+using vespalib::GenerationHeldBase;
+
+
+namespace proton {
+
+namespace documentmetastore {
+
+vespalib::string DOCID_LIMIT("docIdLimit");
+
+class Reader {
+private:
+    std::unique_ptr<Fast_BufferedFile> _datFile;
+    FileReader<uint32_t> _lidReader;
+    FileReader<GlobalId> _gidReader;
+    FileReader<uint8_t> _bucketUsedBitsReader;
+    FileReader<Timestamp> _timestampReader;
+    vespalib::FileHeader _header;
+    uint32_t _headerLen;
+    uint32_t _docIdLimit;
+    uint64_t _datFileSize;
+
+public:
+    Reader(std::unique_ptr<Fast_BufferedFile> datFile)
+            : _datFile(std::move(datFile)),
+              _lidReader(*_datFile),
+              _gidReader(*_datFile),
+              _bucketUsedBitsReader(*_datFile),
+              _timestampReader(*_datFile),
+              _header(),
+              _headerLen(0u),
+              _docIdLimit(0),
+              _datFileSize(0u) {
+        _headerLen = _header.readFile(*_datFile);
+        _datFile->SetPosition(_headerLen);
+        if (!AttributeVector::ReaderBase::extractFileSize(_header, *_datFile,
+                                                          _datFileSize)) {
+            abort();
+        }
+        _docIdLimit = _header.getTag(DOCID_LIMIT).asInteger();
+    }
+
+    uint32_t getDocIdLimit() const { return _docIdLimit; }
+
+    uint32_t
+    getNextLid() {
+        return _lidReader.readHostOrder();
+    }
+
+    GlobalId
+    getNextGid() {
+        return _gidReader.readHostOrder();
+    }
+
+    uint8_t
+    getNextBucketUsedBits(void) {
+        return _bucketUsedBitsReader.readHostOrder();
+    }
+
+    Timestamp
+    getNextTimestamp(void) {
+        return _timestampReader.readHostOrder();
+    }
+
+    size_t
+    getNumElems() const {
+        return (_datFileSize - _headerLen) /
+               (sizeof(uint32_t) + sizeof(GlobalId) +
+                sizeof(uint8_t) + sizeof(Timestamp::Type));
+    }
+};
+
+}
+
+namespace {
+class ShrinkBlockHeld : public GenerationHeldBase
+{
+    DocumentMetaStore &_dms;
+
+public:
+    ShrinkBlockHeld(DocumentMetaStore &dms)
+        : GenerationHeldBase(0),
+          _dms(dms)
+    {
+    }
+
+    virtual
+    ~ShrinkBlockHeld()
+    {
+        _dms.unblockShrinkLidSpace();
+    }
+};
+
+}  // namespace
+
+
+DocumentMetaStore::DocId
+DocumentMetaStore::getFreeLid()
+{
+    return _lidAlloc.getFreeLid(_metaDataStore.size());
+}
+
+DocumentMetaStore::DocId
+DocumentMetaStore::peekFreeLid()
+{
+    return _lidAlloc.peekFreeLid(_metaDataStore.size());
+}
+
+void
+DocumentMetaStore::ensureSpace(DocId lid)
+{
+    while (lid >= _metaDataStore.size()) {
+        _metaDataStore.push_back(RawDocumentMetaData());
+    }
+    setNumDocs(_metaDataStore.size());
+    unsigned int newSize = _metaDataStore.size();
+    unsigned int newCapacity = _metaDataStore.capacity();
+    _lidAlloc.ensureSpace(lid, newSize, newCapacity);
+}
+
+bool
+DocumentMetaStore::insert(DocId lid,
+                          const RawDocumentMetaData &metaData)
+{
+    ensureSpace(lid);
+    _metaDataStore[lid] = metaData;
+    KeyComp comp(metaData, _metaDataStore, *_gidCompare);
+    if (!_gidToLidMap.insert(lid, BTreeNoLeafData(), comp)) {
+        return false;
+    }
+    // flush writes to meta store rcu vector before new entry is visible
+    // from frozen root or lid based scan
+    std::atomic_thread_fence(std::memory_order_release);
+    _lidAlloc.registerLid(lid);
+    updateUncommittedDocIdLimit(lid);
+    incGeneration();
+    const BucketState &state =
+        _bucketDB->takeGuard()->add(metaData.getGid(),
+                      metaData.getBucketId().stripUnused(),
+                      metaData.getTimestamp(),
+                      _subDbType);
+    if (state.isActive()) {
+        _lidAlloc.markAsActive(lid);
+    }
+    updateCommittedDocIdLimit();
+    return true;
+}
+
+void
+DocumentMetaStore::onUpdateStat()
+{
+    MemoryUsage usage = _metaDataStore.getMemoryUsage();
+    usage.incAllocatedBytesOnHold(getGenerationHolder().getHeldBytes());
+    size_t bvSize = _lidAlloc.getUsedLidsSize();
+    usage.incAllocatedBytes(bvSize);
+    usage.incUsedBytes(bvSize);
+    usage.merge(_gidToLidMap.getMemoryUsage());
+    // the free lists are not taken into account here
+    updateStatistics(_metaDataStore.size(),
+                     _metaDataStore.size(),
+                     usage.allocatedBytes(),
+                     usage.usedBytes(),
+                     usage.deadBytes(),
+                     usage.allocatedBytesOnHold());
+}
+
+void
+DocumentMetaStore::onGenerationChange(generation_t generation)
+{
+    _gidToLidMap.getAllocator().freeze();
+    _gidToLidMap.getAllocator().transferHoldLists(generation - 1);
+    getGenerationHolder().transferHoldLists(generation - 1);
+    updateStat(false);
+}
+
+void
+DocumentMetaStore::removeOldGenerations(generation_t firstUsed)
+{
+    _gidToLidMap.getAllocator().trimHoldLists(firstUsed);
+    _lidAlloc.trimHoldLists(firstUsed);
+    getGenerationHolder().trimHoldLists(firstUsed);
+}
+
+std::unique_ptr<search::AttributeSaver>
+DocumentMetaStore::onInitSave()
+{
+    GenerationHandler::Guard guard(getGuard());
+    return std::make_unique<DocumentMetaStoreSaver>
+        (std::move(guard), createSaveTargetConfig(),
+         _gidToLidMap.getFrozenView().begin(), _metaDataStore);
+}
+
+DocumentMetaStore::DocId
+DocumentMetaStore::readNextDoc(documentmetastore::Reader & reader, TreeType::Builder & treeBuilder)
+{
+    uint32_t lid(reader.getNextLid());
+    assert(lid < reader.getDocIdLimit());
+    RawDocumentMetaData & meta = _metaDataStore[lid];
+    meta.setGid(reader.getNextGid());
+    meta.setBucketUsedBits(reader.getNextBucketUsedBits());
+    meta.setTimestamp(reader.getNextTimestamp());
+    treeBuilder.insert(lid, BTreeNoLeafData());
+    assert(!validLid(lid));
+    _lidAlloc.registerLid(lid);
+    return lid;
+}
+
+bool
+DocumentMetaStore::onLoad()
+{
+    documentmetastore::Reader reader(openDAT());
+    unload();
+    size_t numElems = reader.getNumElems();
+    size_t docIdLimit = reader.getDocIdLimit();
+    _metaDataStore.unsafe_reserve(numElems);
+    TreeType::Builder treeBuilder(_gidToLidMap.getAllocator());
+    assert(docIdLimit > 0); // lid 0 is reserved
+    ensureSpace(docIdLimit - 1);
+
+    // insert gids (already sorted)
+    if (numElems > 0) {
+        DocId lid = readNextDoc(reader, treeBuilder);
+        const RawDocumentMetaData * meta = &_metaDataStore[lid];
+        BucketId prevId(meta->getBucketId());
+        BucketState state;
+        state.add(meta->getGid(), meta->getTimestamp(), _subDbType);
+        for (size_t i = 1; i < numElems; ++i) {
+            lid = readNextDoc(reader, treeBuilder);
+            meta = &_metaDataStore[lid];
+            BucketId bucketId = meta->getBucketId();
+            if (prevId != bucketId) {
+                _bucketDB->takeGuard()->add(prevId, state);
+                state = BucketState();
+                prevId = bucketId;
+            }
+            state.add(meta->getGid(), meta->getTimestamp(), _subDbType);
+        }
+        _bucketDB->takeGuard()->add(prevId, state);
+    }
+    _gidToLidMap.assign(treeBuilder);
+    _gidToLidMap.getAllocator().freeze(); // create initial frozen tree
+    generation_t generation = getGenerationHandler().getCurrentGeneration();
+    _gidToLidMap.getAllocator().transferHoldLists(generation);
+
+    setNumDocs(_metaDataStore.size());
+    setCommittedDocIdLimit(_metaDataStore.size());
+
+    return true;
+}
+
+bool
+DocumentMetaStore::checkBuckets(const GlobalId &gid,
+                                const BucketId &bucketId,
+                                const TreeType::Iterator &itr,
+                                bool found)
+{
+    bool success = true;
+#if 0
+    TreeType::Iterator p = itr;
+    --p;
+    if (p.valid()) {
+        DocId prevLid = p.getKey();
+        RawDocumentMetaData &prevMetaData = _metaDataStore[prevLid];
+        BucketId prevBucketId = prevMetaData.getBucketId();
+        if (bucketId != prevBucketId &&
+            (bucketId.contains(prevBucketId) ||
+             prevBucketId.contains(bucketId))) {
+            LOG(error,
+                "Bucket overlap, gid %s bucketId %s and prev gid %s bucket %s",
+                gid.toString().c_str(),
+                bucketId.toString().c_str(),
+                prevMetaData.getGid().toString().c_str(),
+                prevBucketId.toString().c_str());
+            success = false;
+        }
+    }
+    TreeType::Iterator n = itr;
+    if (found)
+        ++n;
+    if (n.valid()) {
+        DocId nextLid = n.getKey();
+        RawDocumentMetaData &nextMetaData = _metaDataStore[nextLid];
+        BucketId nextBucketId = nextMetaData.getBucketId();
+        if (bucketId != nextBucketId &&
+            (bucketId.contains(nextBucketId) ||
+             nextBucketId.contains(bucketId))) {
+            LOG(error,
+                "Bucket overlap, gid %s bucketId %s and next gid %s bucket %s",
+                gid.toString().c_str(),
+                bucketId.toString().c_str(),
+                nextMetaData.getGid().toString().c_str(),
+                nextBucketId.toString().c_str());
+            success = false;
+        }
+    }
+#else
+    (void) gid;
+    (void) bucketId;
+    (void) itr;
+    (void) found;
+#endif
+    return success;
+}
+
+
+template <typename TreeView>
+typename TreeView::Iterator
+DocumentMetaStore::lowerBound(const BucketId &bucketId,
+                              const TreeView &treeView) const
+{
+    document::GlobalId first(document::GlobalId::calculateFirstInBucket(bucketId));
+    KeyComp lowerComp(first, _metaDataStore, *_gidCompare);
+    return treeView.lowerBound(KeyComp::FIND_DOC_ID, lowerComp);
+}
+
+template <typename TreeView>
+typename TreeView::Iterator
+DocumentMetaStore::upperBound(const BucketId &bucketId,
+                              const TreeView &treeView) const
+{
+    document::GlobalId last(document::GlobalId::calculateLastInBucket(bucketId));
+    KeyComp upperComp(last, _metaDataStore, *_gidCompare);
+    return treeView.upperBound(KeyComp::FIND_DOC_ID, upperComp);
+}
+
+void
+DocumentMetaStore::updateMetaDataAndBucketDB(const GlobalId &gid,
+                                             DocId lid,
+                                             const RawDocumentMetaData &newMetaData)
+{
+    RawDocumentMetaData &oldMetaData = _metaDataStore[lid];
+    _bucketDB->takeGuard()->modify(gid,
+                     oldMetaData.getBucketId().stripUnused(),
+                     oldMetaData.getTimestamp(),
+                     newMetaData.getBucketId().stripUnused(),
+                     newMetaData.getTimestamp(),
+                     _subDbType);
+    oldMetaData.setBucketId(newMetaData.getBucketId());
+    std::atomic_thread_fence(std::memory_order_release);
+    oldMetaData.setTimestamp(newMetaData.getTimestamp());
+}
+
+
+namespace
+{
+
+void
+unloadBucket(BucketDBOwner &db, const BucketId &id, const BucketState &delta)
+{
+    if (!id.valid()) {
+        assert(delta.empty());
+        return;
+    }
+    assert(!delta.empty());
+    BucketDBOwner::Guard guard(db.takeGuard());
+    guard->unloadBucket(id, delta);
+}
+
+}
+
+void
+DocumentMetaStore::unload()
+{
+    TreeType::Iterator itr = _gidToLidMap.begin();
+    if ( ! itr.valid() ) return;
+    BucketId prev;
+    BucketState prevDelta;
+    for (; itr.valid(); ++itr) {
+        uint32_t lid = itr.getKey();
+        assert(validLid(lid));
+        RawDocumentMetaData &metaData = _metaDataStore[lid];
+        BucketId bucketId = metaData.getBucketId();
+        if (prev != bucketId) {
+            unloadBucket(*_bucketDB, prev, prevDelta);
+            prevDelta = BucketState();
+            prev = bucketId;
+        }
+        prevDelta.add(metaData.getGid(), metaData.getTimestamp(),
+                      _subDbType);
+    }
+    unloadBucket(*_bucketDB, prev, prevDelta);
+}
+
+
+DocumentMetaStore::DocumentMetaStore(BucketDBOwner::SP bucketDB,
+                                     const vespalib::string &name,
+                                     const GrowStrategy &grow,
+                                     const IGidCompare::SP &gidCompare,
+                                     SubDbType subDbType)
+    : DocumentMetaStoreAttribute(name),
+      _metaDataStore(grow.getDocsInitialCapacity(),
+                     grow.getDocsGrowPercent(),
+                     grow.getDocsGrowDelta(),
+                     getGenerationHolder()),
+      _gidToLidMap(),
+      _lidAlloc(_metaDataStore.size(),
+                _metaDataStore.capacity(),
+                getGenerationHolder()),
+      _gidCompare(gidCompare),
+      _bucketDB(bucketDB),
+      _shrinkLidSpaceBlockers(0),
+      _subDbType(subDbType)
+{
+    ensureSpace(0);			// lid 0 is reserved
+    setCommittedDocIdLimit(1u);         // lid 0 is reserved
+    _gidToLidMap.getAllocator().freeze(); // create initial frozen tree
+    generation_t generation = getGenerationHandler().getCurrentGeneration();
+    _gidToLidMap.getAllocator().transferHoldLists(generation);
+    updateStat(true);
+}
+
+DocumentMetaStore::~DocumentMetaStore()
+{
+    // TODO: Properly notify about modified buckets when using shared bucket db
+    // between document types
+    unload();
+    getGenerationHolder().clearHoldLists();
+    assert(_shrinkLidSpaceBlockers == 0);
+}
+
+DocumentMetaStore::Result
+DocumentMetaStore::inspectExisting(const GlobalId &gid) const
+{
+    assert(_lidAlloc.isFreeListConstructed());
+    Result res;
+    KeyComp comp(gid, _metaDataStore, *_gidCompare);
+    TreeType::Iterator itr = _gidToLidMap.lowerBound(KeyComp::FIND_DOC_ID,
+            comp);
+    bool found = itr.valid() && !comp(KeyComp::FIND_DOC_ID, itr.getKey());
+    if (found) {
+        res.setLid(itr.getKey());
+        res.fillPrev(_metaDataStore[res.getLid()].getTimestamp());
+        res.markSuccess();
+    }
+    return res;
+}
+
+DocumentMetaStore::Result
+DocumentMetaStore::inspect(const GlobalId &gid)
+{
+    assert(_lidAlloc.isFreeListConstructed());
+    Result res;
+    KeyComp comp(gid, _metaDataStore, *_gidCompare);
+    TreeType::Iterator itr = _gidToLidMap.lowerBound(KeyComp::FIND_DOC_ID,
+            comp);
+    bool found = itr.valid() && !comp(KeyComp::FIND_DOC_ID, itr.getKey());
+    if (!found) {
+        DocId myLid = peekFreeLid();
+        res.setLid(myLid);
+        res.markSuccess();
+    } else {
+        res.setLid(itr.getKey());
+        res.fillPrev(_metaDataStore[res.getLid()].getTimestamp());
+        res.markSuccess();
+    }
+    return res;
+}
+
+DocumentMetaStore::Result
+DocumentMetaStore::put(const GlobalId &gid,
+                       const BucketId &bucketId,
+                       const Timestamp &timestamp,
+                       DocId lid)
+{
+    Result res;
+    RawDocumentMetaData metaData(gid, bucketId, timestamp);
+    KeyComp comp(metaData, _metaDataStore, *_gidCompare);
+    TreeType::Iterator itr = _gidToLidMap.lowerBound(KeyComp::FIND_DOC_ID,
+            comp);
+    bool found = itr.valid() && !comp(KeyComp::FIND_DOC_ID, itr.getKey());
+    if (!checkBuckets(gid, bucketId, itr, found)) {
+        // Failure
+    } else if (!found) {
+        if (validLid(lid)) {
+            throw IllegalStateException(
+                    make_string(
+                            "document meta data store"
+                            " or transaction log is corrupted,"
+                            " cannot put"
+                            " document with lid '%u' and gid '%s',"
+                            " gid not found, but lid is used"
+                            " by another gid '%s'",
+                            lid,
+                            gid.toString().c_str(),
+                            _metaDataStore[lid].getGid().toString().c_str()));
+        }
+        if (_lidAlloc.isFreeListConstructed()) {
+            DocId freeLid = getFreeLid();
+            assert(freeLid == lid);
+            (void) freeLid;
+        }
+        if (insert(lid, metaData)) {
+            res.setLid(lid);
+            res.markSuccess();
+        }
+    } else if (lid != itr.getKey()) {
+        throw IllegalStateException(
+                make_string(
+                        "document meta data store"
+                        " or transaction log is corrupted,"
+                        " cannot put"
+                        " document with lid '%u' and gid '%s',"
+                        " gid found, but using another lid '%u'",
+                        lid,
+                        gid.toString().c_str(),
+                        itr.getKey()));
+    } else {
+        res.setLid(lid);
+        res.fillPrev(_metaDataStore[lid].getTimestamp());
+        updateMetaDataAndBucketDB(gid, lid, metaData);
+        res.markSuccess();
+    }
+    return res;
+}
+
+bool
+DocumentMetaStore::updateMetaData(DocId lid,
+                                  const BucketId &bucketId,
+                                  const Timestamp &timestamp)
+{
+    if (!validLid(lid)) {
+        return false;
+    }
+    RawDocumentMetaData &metaData = _metaDataStore[lid];
+    _bucketDB->takeGuard()->modify(metaData.getGid(),
+                     metaData.getBucketId().stripUnused(),
+                     metaData.getTimestamp(),
+                     bucketId.stripUnused(),
+                     timestamp,
+                     _subDbType);
+    metaData.setBucketId(bucketId);
+    std::atomic_thread_fence(std::memory_order_release);
+    metaData.setTimestamp(timestamp);
+    return true;
+}
+
+bool
+DocumentMetaStore::remove(DocId lid)
+{
+    if (!validLid(lid)) {
+        return false;
+    }
+    const GlobalId & gid = getRawGid(lid);
+    KeyComp comp(gid, _metaDataStore, *_gidCompare);
+    if (!_gidToLidMap.remove(lid, comp)) {
+        throw IllegalStateException(make_string(
+                        "document meta data store corrupted,"
+                        " cannot remove"
+                        " document with lid '%u' and gid '%s'",
+                        lid, gid.toString().c_str()));
+    }
+    _lidAlloc.unregisterLid(lid);
+    incGeneration();
+    RawDocumentMetaData &oldMetaData = _metaDataStore[lid];
+    _bucketDB->takeGuard()->remove(oldMetaData.getGid(),
+                     oldMetaData.getBucketId().stripUnused(),
+                     oldMetaData.getTimestamp(),
+                     _subDbType);
+    return true;
+}
+
+void
+DocumentMetaStore::removeComplete(DocId lid)
+{
+    assert(lid != 0);
+    assert(lid < _metaDataStore.size());
+    _lidAlloc.holdLid(lid, _metaDataStore.size(), getCurrentGeneration());
+    incGeneration();
+}
+
+void
+DocumentMetaStore::move(DocId fromLid, DocId toLid)
+{
+    assert(fromLid != 0);
+    assert(toLid != 0);
+    assert(fromLid > toLid);
+    assert(fromLid < getCommittedDocIdLimit());
+    assert(!validLid(toLid));
+    assert(validLid(fromLid));
+    _lidAlloc.moveLidBegin(fromLid, toLid);
+    _metaDataStore[toLid] = _metaDataStore[fromLid];
+    const GlobalId & gid = getRawGid(fromLid);
+    KeyComp comp(gid, _metaDataStore, *_gidCompare);
+    TreeType::Iterator it(_gidToLidMap.lowerBound(fromLid, comp));
+    assert(it.valid());
+    assert(it.getKey() == fromLid);
+    _gidToLidMap.thaw(it);
+    it.writeKey(toLid);
+    _lidAlloc.moveLidEnd(fromLid, toLid);
+    incGeneration();
+}
+
+void
+DocumentMetaStore::removeBatch(const std::vector<search::DocumentIdT> &lidsToRemove,
+                               const uint32_t docIdLimit)
+{
+    for (const auto &lid : lidsToRemove) {
+        assert(lid > 0 && lid < docIdLimit);
+        (void) docIdLimit;
+
+        bool removed = remove(lid);
+        assert(removed);
+        (void) removed;
+    }
+}
+
+void
+DocumentMetaStore::removeBatchComplete(const std::vector<DocId> &lidsToRemove)
+{
+    _lidAlloc.holdLids(lidsToRemove, _metaDataStore.size(), getCurrentGeneration());
+    incGeneration();
+}
+
+bool
+DocumentMetaStore::getGid(DocId lid, GlobalId &gid) const
+{
+    if (!validLid(lid)) {
+        return false;
+    }
+    gid = getRawGid(lid);
+    return true;
+}
+
+bool
+DocumentMetaStore::getLid(const GlobalId &gid, DocId &lid) const
+{
+    GlobalId value(gid);
+    KeyComp comp(value, _metaDataStore, *_gidCompare);
+    TreeType::ConstIterator itr =
+        _gidToLidMap.getFrozenView().find(KeyComp::FIND_DOC_ID, comp);
+    if (!itr.valid()) {
+        return false;
+    }
+    lid = itr.getKey();
+    return true;
+}
+
+void
+DocumentMetaStore::constructFreeList()
+{
+    _lidAlloc.constructFreeList(_metaDataStore.size());
+    incGeneration();
+    _lidAlloc.setFreeListConstructed();
+}
+
+search::DocumentMetaData
+DocumentMetaStore::getMetaData(const GlobalId &gid) const
+{
+    DocId lid = 0;
+    if (!getLid(gid, lid) || !validLid(lid)) {
+        return search::DocumentMetaData();
+    }
+    const RawDocumentMetaData &raw = getRawMetaData(lid);
+    Timestamp timestamp(raw.getTimestamp());
+    std::atomic_thread_fence(std::memory_order_acquire);
+    return search::DocumentMetaData(lid,
+                                    timestamp,
+                                    raw.getBucketId(),
+                                    raw.getGid(),
+                                    _subDbType == SubDbType::REMOVED);
+}
+
+void
+DocumentMetaStore::getMetaData(const BucketId &bucketId,
+                               search::DocumentMetaData::Vector &result) const
+{
+    TreeType::FrozenView frozenTreeView = _gidToLidMap.getFrozenView();
+    TreeType::ConstIterator itr = lowerBound(bucketId, frozenTreeView);
+    TreeType::ConstIterator end = upperBound(bucketId, frozenTreeView);
+    for (; itr != end; ++itr) {
+        DocId lid = itr.getKey();
+        if (validLid(lid)) {
+            const RawDocumentMetaData &rawData = getRawMetaData(lid);
+            if (bucketId.getUsedBits() != rawData.getBucketUsedBits())
+                continue; // Wrong bucket (due to overlapping buckets)
+            Timestamp timestamp(rawData.getTimestamp());
+            std::atomic_thread_fence(std::memory_order_acquire);
+            result.push_back(search::DocumentMetaData(lid, timestamp,
+                            rawData.getBucketId(),
+                            rawData.getGid(),
+                            _subDbType == SubDbType::REMOVED));
+        }
+    }
+}
+
+LidUsageStats
+DocumentMetaStore::getLidUsageStats() const
+{
+    uint32_t docIdLimit = getCommittedDocIdLimit();
+    uint32_t numDocs = getNumUsedLids();
+    uint32_t lowestFreeLid = _lidAlloc.getLowestFreeLid();
+    uint32_t highestUsedLid = _lidAlloc.getHighestUsedLid();
+    return LidUsageStats(docIdLimit,
+                         numDocs,
+                         lowestFreeLid,
+                         highestUsedLid);
+}
+
+Blueprint::UP
+DocumentMetaStore::createBlackListBlueprint() const
+{
+    return _lidAlloc.createBlackListBlueprint();
+}
+
+AttributeVector::SearchContext::UP
+DocumentMetaStore::getSearch(search::QueryTermSimple::UP qTerm, const AttributeVector::SearchContext::Params &) const
+{
+    return AttributeVector::SearchContext::UP
+            (new documentmetastore::SearchContext(std::move(qTerm), *this));
+}
+
+DocumentMetaStore::ConstIterator
+DocumentMetaStore::beginFrozen() const
+{
+    return _gidToLidMap.getFrozenView().begin();
+}
+
+DocumentMetaStore::Iterator
+DocumentMetaStore::begin() const
+{
+    // Called by writer thread
+    return _gidToLidMap.begin();
+}
+
+DocumentMetaStore::Iterator
+DocumentMetaStore::lowerBound(const BucketId &bucketId) const
+{
+    // Called by writer thread
+    return lowerBound(bucketId, _gidToLidMap);
+}
+
+DocumentMetaStore::Iterator
+DocumentMetaStore::upperBound(const BucketId &bucketId) const
+{
+    // Called by writer thread
+    return upperBound(bucketId, _gidToLidMap);
+}
+
+DocumentMetaStore::Iterator
+DocumentMetaStore::lowerBound(const GlobalId &gid) const
+{
+    // Called by writer thread
+    KeyComp comp(gid, _metaDataStore, *_gidCompare);
+    return _gidToLidMap.lowerBound(KeyComp::FIND_DOC_ID, comp);
+}
+
+DocumentMetaStore::Iterator
+DocumentMetaStore::upperBound(const GlobalId &gid) const
+{
+    // Called by writer thread
+    KeyComp comp(gid, _metaDataStore, *_gidCompare);
+    return _gidToLidMap.upperBound(KeyComp::FIND_DOC_ID, comp);
+}
+
+void
+DocumentMetaStore::getLids(const BucketId &bucketId, std::vector<DocId> &lids)
+{
+    // Called by writer thread
+    TreeType::Iterator itr = lowerBound(bucketId);
+    TreeType::Iterator end = upperBound(bucketId);
+    for (; itr != end; ++itr) {
+        DocId lid = itr.getKey();
+        assert(validLid(lid));
+        const RawDocumentMetaData &metaData = getRawMetaData(lid);
+        uint8_t bucketUsedBits = metaData.getBucketUsedBits();
+        assert(bucketUsedBits >= BucketId::minNumBits() &&
+               bucketUsedBits <= BucketId::maxNumBits());
+        if (bucketUsedBits != bucketId.getUsedBits())
+            continue;	// Skip document belonging to overlapping bucket
+        lids.push_back(lid);
+    }
+}
+
+bucketdb::BucketDeltaPair
+DocumentMetaStore::handleSplit(const bucketdb::SplitBucketSession &session)
+{
+    const BucketId &source(session.getSource());
+    const BucketId &target1(session.getTarget1());
+    const BucketId &target2(session.getTarget2());
+
+    if (_subDbType == SubDbType::READY) {
+        if (session.mustFixupTarget1ActiveLids()) {
+            updateActiveLids(target1, session.getSourceActive());
+        }
+        if (session.mustFixupTarget2ActiveLids()) {
+            updateActiveLids(target2, session.getSourceActive());
+        }
+    }
+
+    TreeType::Iterator itr = lowerBound(source);
+    TreeType::Iterator end = upperBound(source);
+    bucketdb::BucketDeltaPair deltas;
+    for (; itr != end; ++itr) {
+        DocId lid = itr.getKey();
+        assert(validLid(lid));
+        RawDocumentMetaData &metaData = _metaDataStore[lid];
+        uint8_t bucketUsedBits = metaData.getBucketUsedBits();
+        assert(bucketUsedBits >= BucketId::minNumBits() &&
+               bucketUsedBits <= BucketId::maxNumBits());
+        if (bucketUsedBits == source.getUsedBits()) {
+            BucketId t1(metaData.getGid().convertToBucketId());
+            BucketId t2(t1);
+            if (target1.valid()) {
+                t1.setUsedBits(target1.getUsedBits());
+            }
+            if (target2.valid()) {
+                t2.setUsedBits(target2.getUsedBits());
+            }
+            if (target1.valid() && t1 == target1) {
+                metaData.setBucketUsedBits(target1.getUsedBits());
+                deltas._delta1.add(metaData.getGid(),
+                                   metaData.getTimestamp(),
+                                   _subDbType);
+            } else if (target2.valid() && t2 == target2) {
+                metaData.setBucketUsedBits(target2.getUsedBits());
+                deltas._delta2.add(metaData.getGid(),
+                                   metaData.getTimestamp(),
+                                   _subDbType);
+            }
+        }
+    }
+    return deltas;
+    // Caller can remove source bucket if empty
+}
+
+
+bucketdb::BucketDeltaPair
+DocumentMetaStore::handleJoin(const bucketdb::JoinBucketsSession &session)
+{
+    const BucketId &source1(session.getSource1());
+    const BucketId &source2(session.getSource2());
+    const BucketId &target(session.getTarget());
+
+    TreeType::Iterator itr = lowerBound(target);
+    TreeType::Iterator end = upperBound(target);
+    bucketdb::BucketDeltaPair deltas;
+    for (; itr != end; ++itr) {
+        DocId lid = itr.getKey();
+        assert(validLid(lid));
+        RawDocumentMetaData &metaData = _metaDataStore[lid];
+        uint8_t bucketUsedBits = metaData.getBucketUsedBits();
+        assert(bucketUsedBits >= BucketId::minNumBits() &&
+               bucketUsedBits <= BucketId::maxNumBits());
+        BucketId s(metaData.getBucketId());
+        if (source1.valid() && s == source1) {
+            metaData.setBucketUsedBits(target.getUsedBits());
+            deltas._delta1.add(metaData.getGid(), metaData.getTimestamp(),
+                             _subDbType);
+        } else if (source2.valid() && s == source2) {
+            metaData.setBucketUsedBits(target.getUsedBits());
+            deltas._delta2.add(metaData.getGid(), metaData.getTimestamp(),
+                             _subDbType);
+        }
+    }
+    if (_subDbType == SubDbType::READY) {
+        bool movedSource1Docs = deltas._delta1.getReadyCount() != 0;
+        bool movedSource2Docs = deltas._delta2.getReadyCount() != 0;
+        if (session.mustFixupTargetActiveLids(movedSource1Docs,
+                                              movedSource2Docs)) {
+            updateActiveLids(target, session.getWantTargetActive());
+        }
+    }
+    return deltas;
+    // Caller can remove source buckets if they are empty
+}
+
+
+void
+DocumentMetaStore::setBucketState(const BucketId &bucketId, bool active)
+{
+    updateActiveLids(bucketId, active);
+    _bucketDB->takeGuard()->setBucketState(bucketId, active);
+}
+
+void
+DocumentMetaStore::updateActiveLids(const BucketId &bucketId, bool active)
+{
+    TreeType::Iterator itr = lowerBound(bucketId);
+    TreeType::Iterator end = upperBound(bucketId);
+    uint8_t bucketUsedBits = bucketId.getUsedBits();
+    for (; itr != end; ++itr) {
+        DocId lid = itr.getKey();
+        assert(validLid(lid));
+        RawDocumentMetaData &metaData = _metaDataStore[lid];
+        if (metaData.getBucketUsedBits() != bucketUsedBits) {
+            continue;
+        }
+        _lidAlloc.updateActiveLids(lid, active);
+    }
+    _lidAlloc.commitActiveLids();
+}
+
+void
+DocumentMetaStore::populateActiveBuckets(const BucketId::List &buckets)
+{
+    typedef BucketId::List BIV;
+    BIV fixupBuckets;
+
+    _bucketDB->takeGuard()->populateActiveBuckets(buckets, fixupBuckets);
+
+    for (const auto &bucketId : fixupBuckets) {
+        updateActiveLids(bucketId, true);
+    }
+    _lidAlloc.commitActiveLids();
+}
+
+void
+DocumentMetaStore::clearDocs(DocId lidLow, DocId lidLimit)
+{
+    assert(lidLow <= lidLimit);
+    assert(lidLimit <= getNumDocs());
+    _lidAlloc.clearDocs(lidLow, lidLimit);
+}
+
+void
+DocumentMetaStore::compactLidSpace(uint32_t wantedLidLimit)
+{
+    AttributeVector::compactLidSpace(wantedLidLimit);
+    _lidAlloc.compactLidSpace(wantedLidLimit);
+    ++_shrinkLidSpaceBlockers;
+}
+
+void
+DocumentMetaStore::holdUnblockShrinkLidSpace(void)
+{
+    assert(_shrinkLidSpaceBlockers > 0);
+    GenerationHeldBase::UP hold(new ShrinkBlockHeld(*this));
+    getGenerationHolder().hold(std::move(hold));
+    incGeneration();
+}
+
+void
+DocumentMetaStore::unblockShrinkLidSpace(void)
+{
+    assert(_shrinkLidSpaceBlockers > 0);
+    --_shrinkLidSpaceBlockers;
+}
+
+bool
+DocumentMetaStore::canShrinkLidSpace(void) const
+{
+    return AttributeVector::canShrinkLidSpace() &&
+        _shrinkLidSpaceBlockers == 0;
+}
+
+void
+DocumentMetaStore::onShrinkLidSpace()
+{
+    uint32_t committedDocIdLimit = this->getCommittedDocIdLimit();
+    _lidAlloc.shrinkLidSpace(committedDocIdLimit);
+    _metaDataStore.shrink(committedDocIdLimit);
+    setNumDocs(committedDocIdLimit);
+}
+
+uint64_t
+DocumentMetaStore::getBucketOf(const vespalib::GenerationHandler::Guard &, uint32_t lid) const
+{
+    if (__builtin_expect(lid < getCommittedDocIdLimit(), true)) {
+        if (__builtin_expect(validLidFast(lid), true)) {
+            return getRawMetaData(lid).getBucketId().getId();
+        }
+    }
+    return 0;
+}
+
+vespalib::GenerationHandler::Guard
+DocumentMetaStore::getGuard() const
+{
+    const vespalib::GenerationHandler & genHandler = getGenerationHandler();
+    return genHandler.takeGuard();
+}
+
+uint64_t
+DocumentMetaStore::getEstimatedSaveByteSize() const
+{
+    uint32_t numDocs = getNumUsedLids();
+    return minHeaderLen + numDocs * entrySize;
+}
+
+
+}  // namespace proton
+
+template class search::btree::
+BTreeIterator<proton::DocumentMetaStore::DocId,
+              search::btree::BTreeNoLeafData,
+              search::btree::NoAggregated,
+              const proton::DocumentMetaStore::KeyComp &>;
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
new file mode 100644
index 00000000000..c8a59136573
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
@@ -0,0 +1,339 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h>
+#include <vespa/searchcore/proton/common/subdbtype.h>
+#include <vespa/searchlib/attribute/iattributesavetarget.h>
+#include <vespa/searchlib/common/rcuvector.h>
+#include <vespa/searchlib/attribute/singlesmallnumericattribute.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/docstore/ibucketizer.h>
+#include "gid_compare.h"
+#include "document_meta_store_adapter.h"
+#include "documentmetastoreattribute.h"
+#include "lid_allocator.h"
+#include "lid_gid_key_comparator.h"
+#include "lid_hold_list.h"
+#include "lidstatevector.h"
+#include "raw_document_meta_data.h"
+
+namespace proton {
+
+namespace bucketdb
+{
+
+class SplitBucketSession;
+class JoinBucketsSession;
+
+}
+
+namespace documentmetastore {
+    class Reader;
+};
+
+/**
+ * This class provides a storage of <lid, meta data> pairs (local
+ * document id, meta data (including global document id)) and mapping
+ * from lid -> meta data (including gid) and gid -> lid.
+ **/
+class DocumentMetaStore final : public DocumentMetaStoreAttribute,
+                                public DocumentMetaStoreAdapter,
+                                public search::IBucketizer
+{
+public:
+    typedef std::shared_ptr<DocumentMetaStore> SP;
+    typedef documentmetastore::IStore::Result Result;
+    typedef documentmetastore::IStore::DocId DocId;
+    typedef documentmetastore::IStore::GlobalId GlobalId;
+    typedef documentmetastore::IStore::BucketId BucketId;
+    typedef documentmetastore::IStore::Timestamp Timestamp;
+    typedef documentmetastore::IGidCompare IGidCompare;
+    typedef documentmetastore::DefaultGidCompare DefaultGidCompare;
+
+    // If using proton::DocumentMetaStore directly, the
+    // DocumentMetaStoreAttribute functions here are used instead of
+    // the ones with the same signature in proton::IDocumentMetaStore.
+    using DocumentMetaStoreAttribute::commit;
+    using DocumentMetaStoreAttribute::getCommittedDocIdLimit;
+    using DocumentMetaStoreAttribute::removeAllOldGenerations;
+
+private:
+    // maps from lid -> meta data
+    typedef search::attribute::RcuVectorBase<RawDocumentMetaData> MetaDataStore;
+    typedef documentmetastore::LidGidKeyComparator KeyComp;
+
+    // Lids are stored as keys in the tree, sorted by their gid
+    // counterpart.  The LidGidKeyComparator class maps from lids -> metadata by
+    // using the metadata store.
+    typedef search::btree::BTree<DocId,
+                                 search::btree::BTreeNoLeafData,
+                                 search::btree::NoAggregated,
+                                 const KeyComp &> TreeType;
+    // Bit attribute vector used to keep track of active & inactive documents.
+    // Inactive documents will be black-listed during search.
+    typedef search::SingleValueBitNumericAttribute BitAttribute;
+
+    MetaDataStore       _metaDataStore;
+    TreeType            _gidToLidMap;
+    documentmetastore::LidAllocator _lidAlloc;
+    IGidCompare::SP     _gidCompare;
+    BucketDBOwner::SP   _bucketDB;
+    uint32_t            _shrinkLidSpaceBlockers;
+    const SubDbType     _subDbType;
+
+    DocId getFreeLid();
+    DocId peekFreeLid();
+    VESPA_DLL_LOCAL void ensureSpace(DocId lid);
+    bool insert(DocId lid, const RawDocumentMetaData &metaData);
+
+    const GlobalId &
+    getRawGid(DocId lid) const
+    {
+        return getRawMetaData(lid).getGid();
+    }
+
+    virtual void onUpdateStat() override;
+
+    // Implements AttributeVector
+    virtual void onGenerationChange(generation_t generation) override;
+    virtual void removeOldGenerations(generation_t firstUsed) override;
+    virtual std::unique_ptr<search::AttributeSaver> onInitSave() override;
+    virtual bool onLoad() override;
+
+    bool
+    checkBuckets(const GlobalId &gid,
+                 const BucketId &bucketId,
+                 const TreeType::Iterator &itr,
+                 bool found);
+
+    template <typename TreeView>
+    typename TreeView::Iterator
+    lowerBound(const BucketId &bucketId,
+               const TreeView &treeView) const;
+
+    template <typename TreeView>
+    typename TreeView::Iterator
+    upperBound(const BucketId &bucketId,
+               const TreeView &treeView) const;
+
+    void updateMetaDataAndBucketDB(const GlobalId &gid,
+                                   DocId lid,
+                                   const RawDocumentMetaData &newMetaData);
+
+    void unload();
+
+    virtual void
+    updateActiveLids(const BucketId &bucketId, bool active) override;
+
+    /**
+     * Implements DocumentMetaStoreAdapter
+     */
+    virtual void doCommit(search::SerialNum firstSerialNum,
+                          search::SerialNum lastSerialNum) override {
+        commit(firstSerialNum, lastSerialNum);
+    }
+    virtual DocId doGetCommittedDocIdLimit() const override {
+        return getCommittedDocIdLimit();
+    }
+    virtual void doRemoveAllOldGenerations() override {
+        removeAllOldGenerations();
+    }
+
+    VESPA_DLL_LOCAL DocId readNextDoc(documentmetastore::Reader & reader, TreeType::Builder & treeBuilder);
+
+public:
+    typedef TreeType::Iterator Iterator;
+    typedef TreeType::ConstIterator ConstIterator;
+    static constexpr size_t minHeaderLen = 0x1000;
+    static constexpr size_t entrySize =
+        sizeof(uint32_t) + GlobalId::LENGTH + sizeof(uint8_t) +
+        sizeof(Timestamp::Type);
+
+    DocumentMetaStore(BucketDBOwner::SP bucketDB,
+                      const vespalib::string & name=getFixedName(),
+                      const search::GrowStrategy & grow=search::GrowStrategy(),
+                      const IGidCompare::SP &gidCompare =
+                      IGidCompare::SP(new documentmetastore::DefaultGidCompare),
+                      SubDbType subDbType = SubDbType::READY);
+    ~DocumentMetaStore();
+
+    /**
+     * Implements documentmetastore::IStore.
+     */
+    virtual Result inspectExisting(const GlobalId &gid) const override;
+    virtual Result inspect(const GlobalId &gid) override;
+    /**
+     * Puts the given <lid, meta data> pair to this store.
+     * This function should only be called before constructFreeList()
+     * and typically after a load(). The use case is replaying of a
+     * transaction log where the lids are stored in the log. The gid
+     * map is then re-built the same way it was originally where add()
+     * was used to create the <lid, gid> pairs.
+     **/
+    virtual Result put(const GlobalId &gid,
+                       const BucketId &bucketId,
+                       const Timestamp &timestamp,
+                       DocId lid) override;
+    virtual bool updateMetaData(DocId lid,
+                                const BucketId &bucketId,
+                                const Timestamp &timestamp) override;
+    virtual bool remove(DocId lid) override;
+
+    virtual uint64_t getBucketOf(const vespalib::GenerationHandler::Guard & guard, uint32_t lid) const override;
+    virtual vespalib::GenerationHandler::Guard getGuard() const override;
+
+    /**
+     * Put lid on a hold list, for later reuse.  Typically called
+     * after remove() has been called and related structures for
+     * document has been torn down (memory index, attribute vectors,
+     * document store).
+     */
+    virtual void removeComplete(DocId lid) override;
+    virtual void move(DocId fromLid, DocId toLid) override;
+    bool validLidFast(DocId lid) const { return _lidAlloc.validLid(lid); }
+    virtual bool validLid(DocId lid) const override {
+        return validLidFast(lid);
+    }
+    virtual void removeBatch(const std::vector<DocId> &lidsToRemove,
+                             const DocId docIdLimit) override;
+    /**
+     * Put lids on a hold list, for laster reuse.
+     */
+    virtual void
+    removeBatchComplete(const std::vector<DocId> &lidsToRemove) override;
+    virtual const RawDocumentMetaData &
+    getRawMetaData(DocId lid) const override { return _metaDataStore[lid]; }
+
+
+
+    /**
+     * Implements search::IDocumentMetaStore
+     **/
+    virtual bool getGid(DocId lid, GlobalId &gid) const override;
+    virtual bool getLid(const GlobalId & gid, DocId &lid) const override;
+    virtual search::DocumentMetaData
+    getMetaData(const GlobalId &gid) const override;
+    virtual void
+    getMetaData(const BucketId &bucketId,
+                search::DocumentMetaData::Vector &result) const override;
+    virtual DocId getNumUsedLids() const override {
+        return _lidAlloc.getNumUsedLids();
+    }
+    virtual DocId getNumActiveLids() const override {
+        return _lidAlloc.getNumActiveLids();
+    }
+    virtual search::LidUsageStats getLidUsageStats() const override;
+    virtual search::queryeval::Blueprint::UP
+    createBlackListBlueprint() const override;
+
+
+
+    /**
+     * Implements search::AttributeVector
+     */
+    SearchContext::UP
+    getSearch(search::QueryTermSimple::UP qTerm,
+              const search::AttributeVector::SearchContext::Params & params)
+        const override;
+
+
+
+    /**
+     * Implements proton::IDocumentMetaStore
+     */
+    virtual void constructFreeList() override;
+
+    virtual Iterator begin() const override;
+
+    virtual Iterator lowerBound(const BucketId &bucketId) const override;
+
+    virtual Iterator upperBound(const BucketId &bucketId) const override;
+
+    virtual Iterator lowerBound(const GlobalId &gid) const override;
+
+    virtual Iterator upperBound(const GlobalId &gid) const override;
+
+    virtual void
+    getLids(const BucketId &bucketId, std::vector<DocId> &lids) override;
+
+    virtual search::AttributeGuard getActiveLidsGuard() const override {
+        return _lidAlloc.getActiveLidsGuard();
+    }
+
+    virtual bool getFreeListActive() const override {
+        return _lidAlloc.isFreeListConstructed();
+    }
+
+    virtual void compactLidSpace(DocId wantedLidLimit) override;
+
+    virtual void holdUnblockShrinkLidSpace() override;
+
+    virtual bool canShrinkLidSpace() const override;
+
+    virtual search::SerialNum getLastSerialNum() const override {
+        return getStatus().getLastSyncToken();
+    }
+
+
+
+    /**
+     * Implements documentmetastore::IBucketHandler.
+     */
+    virtual BucketDBOwner &getBucketDB() const override {
+        return *_bucketDB;
+    }
+
+    virtual bucketdb::BucketDeltaPair
+    handleSplit(const bucketdb::SplitBucketSession &session) override;
+
+    virtual bucketdb::BucketDeltaPair
+    handleJoin(const bucketdb::JoinBucketsSession &session) override;
+
+    virtual void
+    setBucketState(const BucketId &bucketId, bool active) override;
+
+    virtual void
+    populateActiveBuckets(const BucketId::List &buckets) override;
+
+
+
+    ConstIterator
+    beginFrozen() const;
+
+    const vespalib::GenerationHandler & getGenerationHandler() const {
+        return AttributeVector::getGenerationHandler();
+    }
+
+    vespalib::GenerationHandler & getGenerationHandler() {
+        return AttributeVector::getGenerationHandler();
+    }
+
+    const BitAttribute &getActiveLids() const { return _lidAlloc.getActiveLids(); }
+
+    virtual void
+    clearDocs(DocId lidLow, DocId lidLimit) override;
+
+    /*
+     * Called by document db executor to unblock shrinking of lid
+     * space after all lids held by holdLid() operations have been
+     * unheld.
+     */
+    void
+    unblockShrinkLidSpace();
+
+    virtual void
+    onShrinkLidSpace() override;
+
+    virtual uint64_t getEstimatedSaveByteSize() const override;
+};
+
+}
+
+extern template class search::btree::
+BTreeIterator<proton::DocumentMetaStore::DocId,
+              search::btree::BTreeNoLeafData,
+              search::btree::NoAggregated,
+              const proton::DocumentMetaStore::KeyComp &>;
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp
new file mode 100644
index 00000000000..3de5c4c84d1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.documentmetastoreattribute");
+#include "documentmetastoreattribute.h"
+
+namespace proton {
+
+namespace {
+
+const vespalib::string _G_documentMetaStoreName("[documentmetastore]");
+
+}
+
+const vespalib::string &
+DocumentMetaStoreAttribute::getFixedName()
+{
+    return _G_documentMetaStoreName;
+}
+
+
+void
+DocumentMetaStoreAttribute::notImplemented() const
+{
+    throw vespalib::IllegalStateException(
+            "The function is not implemented for DocumentMetaStoreAttribute");
+}
+
+
+DocumentMetaStoreAttribute::DocumentMetaStoreAttribute(const vespalib::string &name)
+    : NotImplementedAttribute(name, Config(BasicType::NONE))
+{
+}
+
+
+DocumentMetaStoreAttribute::~DocumentMetaStoreAttribute()
+{
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h
new file mode 100644
index 00000000000..54686f79c01
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/base/globalid.h>
+#include <vespa/searchlib/attribute/not_implemented_attribute.h>
+
+namespace proton {
+
+/**
+ * Abstract implementation of the IDocumentMetaStore interface
+ * as an attribute vector.
+ **/
+class DocumentMetaStoreAttribute : public search::NotImplementedAttribute
+{
+protected:
+    virtual void notImplemented() const __attribute__((noinline));
+
+public:
+    DocumentMetaStoreAttribute(const vespalib::string &name=getFixedName());
+    virtual ~DocumentMetaStoreAttribute();
+
+    static const vespalib::string &getFixedName();
+
+    // Implements IAttributeVector
+    virtual size_t
+    getFixedWidth() const override
+    {
+        return document::GlobalId::LENGTH;
+    }
+
+    virtual void
+    onCommit()
+    {
+    }
+};
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp
new file mode 100644
index 00000000000..65ab730f7f4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.documentmetastorecontext");
+#include "documentmetastorecontext.h"
+
+namespace proton {
+
+DocumentMetaStoreContext::ReadGuard::ReadGuard(const search::AttributeVector::SP &metaStoreAttr) :
+    _guard(metaStoreAttr),
+    _store(static_cast<const DocumentMetaStore &>(_guard.get())),
+    _activeLidsGuard(_store.getActiveLidsGuard())
+{
+}
+
+
+DocumentMetaStoreContext::DocumentMetaStoreContext(BucketDBOwner::SP bucketDB,
+                                                   const vespalib::string &name,
+                                                   const search::GrowStrategy &grow,
+                                                   const DocumentMetaStore::IGidCompare::SP &gidCompare) :
+    _metaStoreAttr(new DocumentMetaStore(bucketDB, name, grow, gidCompare)),
+    _metaStore(std::dynamic_pointer_cast<IDocumentMetaStore>(_metaStoreAttr))
+{
+}
+
+
+DocumentMetaStoreContext::DocumentMetaStoreContext(const search::AttributeVector::SP &metaStoreAttr) :
+    _metaStoreAttr(metaStoreAttr),
+    _metaStore(std::dynamic_pointer_cast<IDocumentMetaStore>(_metaStoreAttr))
+{
+}
+
+void
+DocumentMetaStoreContext::constructFreeList(void)
+{
+    _metaStore->constructFreeList();
+}
+
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h
new file mode 100644
index 00000000000..01fa6c8c4aa
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentmetastore.h"
+#include "i_document_meta_store_context.h"
+#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h>
+
+namespace proton {
+
+/**
+ * Class providing write and read interface to the document meta store.
+ */
+class DocumentMetaStoreContext : public IDocumentMetaStoreContext
+{
+public:
+    class ReadGuard : public IDocumentMetaStoreContext::IReadGuard
+    {
+    private:
+        search::AttributeGuard   _guard;
+        const DocumentMetaStore &_store;
+        search::AttributeGuard   _activeLidsGuard;
+    public:
+        ReadGuard(const search::AttributeVector::SP &metaStoreAttr);
+        virtual const search::IDocumentMetaStore &get() const { return _store; }
+    };
+private:
+    search::AttributeVector::SP _metaStoreAttr;
+    IDocumentMetaStore::SP      _metaStore;
+public:
+
+    /**
+     * Create a new context instantiating a document meta store
+     * with the given name, grow strategy, and comparator.
+     */
+    DocumentMetaStoreContext(BucketDBOwner::SP bucketDB,
+                             const vespalib::string &name = DocumentMetaStore::getFixedName(),
+                             const search::GrowStrategy &grow = search::GrowStrategy(),
+                             const DocumentMetaStore::IGidCompare::SP &gidCompare =
+                             DocumentMetaStore::IGidCompare::SP(new DocumentMetaStore::DefaultGidCompare));
+
+    /**
+     * Create a new context with the given document meta store encapsulated
+     * as an attribute vector.
+     */
+    DocumentMetaStoreContext(const search::AttributeVector::SP &metaStoreAttr);
+
+    // Implements IDocumentMetaStoreContext
+    proton::IDocumentMetaStore::SP   getSP() const override { return _metaStore; }
+    proton::IDocumentMetaStore &       get()       override { return *_metaStore; }
+
+    IReadGuard::UP getReadGuard() const override {
+        return IReadGuard::UP(new ReadGuard(_metaStoreAttr));
+    }
+
+    void constructFreeList() override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
new file mode 100644
index 00000000000..4768b847fd4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
@@ -0,0 +1,272 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.documentmetastoreflushtarget");
+
+#include <vespa/searchcore/proton/attribute/attributedisklayout.h>
+#include "documentmetastoreflushtarget.h"
+#include <vespa/searchlib/attribute/attributefilesavetarget.h>
+#include <vespa/searchlib/attribute/attributesaver.h>
+#include <vespa/searchlib/util/dirtraverse.h>
+#include <vespa/searchlib/util/filekit.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <fstream>
+#include <vespa/searchlib/common/serialnumfileheadercontext.h>
+#include <vespa/searchcore/proton/server/itlssyncer.h>
+
+using namespace search;
+using namespace vespalib;
+using search::common::FileHeaderContext;
+using search::common::SerialNumFileHeaderContext;
+using vespalib::makeTask;
+using vespalib::makeClosure;
+
+namespace proton {
+
+DocumentMetaStoreFlushTarget::Flusher::
+Flusher(DocumentMetaStoreFlushTarget &dmsft,
+        SerialNum syncToken)
+    : _dmsft(dmsft),
+      _saver(),
+      _syncToken(syncToken),
+      _flushDir("")
+{
+    DocumentMetaStore &dms = *_dmsft._dms;
+    // Called by document db executor
+    if (dms.canShrinkLidSpace()) {
+        dms.shrinkLidSpace();
+    }
+    _flushDir = _dmsft.getSnapshotDir(syncToken);
+    vespalib::string newBaseFileName(_flushDir + "/" + dms.getName());
+    dms.setBaseFileName(newBaseFileName);
+    _saver = dms.initSave();
+    assert(_saver);
+}
+
+DocumentMetaStoreFlushTarget::Flusher::~Flusher()
+{
+    // empty
+}
+
+bool
+DocumentMetaStoreFlushTarget::Flusher::saveSnapInfo()
+{
+    if (!_dmsft._snapInfo.save()) {
+        LOG(warning,
+            "Could not save meta-info file for document meta store"
+            " '%s' to disk",
+            _dmsft._dms->getBaseFileName().c_str());
+        return false;
+    }
+    return true;
+}
+
+bool
+DocumentMetaStoreFlushTarget::Flusher::flush()
+{
+    IndexMetaInfo::Snapshot newSnap(false, _syncToken,
+                                    getSnapshotName(_syncToken));
+    {
+        vespalib::LockGuard guard(_dmsft._snapInfoLock);
+        _dmsft._snapInfo.addSnapshot(newSnap);
+    }
+    if (!saveSnapInfo()) {
+        return false;
+    }
+    vespalib::mkdir(_flushDir, false);
+    vespalib::string flushFile(_flushDir + "/" + _dmsft._dms->getName());
+
+    SerialNumFileHeaderContext fileHeaderContext(_dmsft._fileHeaderContext,
+                                                 _syncToken);
+    search::AttributeFileSaveTarget saveTarget(_dmsft._tuneFileAttributes,
+                             fileHeaderContext);
+    if (!_saver->save(saveTarget)) {
+        LOG(warning, "Could not write document meta store '%s' to disk",
+            _dmsft._dms->getBaseFileName().c_str());
+        _saver.reset();
+        return false;
+    }
+    _saver.reset();
+    /*
+     * Sync transaction log again.  This is needed when background
+     * flush is activated to ensure that same future will occur that has
+     * already been observable in the saved document meta store (future
+     * timestamp or bucket id).
+     */
+    _dmsft._tlsSyncer.sync();
+    {
+        vespalib::LockGuard guard(_dmsft._snapInfoLock);
+        _dmsft._snapInfo.validateSnapshot(_syncToken);
+    }
+    if (!saveSnapInfo()) {
+        return false;
+    }
+    _dmsft._lastFlushTime = search::FileKit::getModificationTime(_flushDir);
+    return true;
+}
+
+void
+DocumentMetaStoreFlushTarget::Flusher::updateStats()
+{
+    _dmsft._lastStats.setPath(_flushDir);
+}
+
+bool
+DocumentMetaStoreFlushTarget::Flusher::cleanUp()
+{
+    if (_dmsft._cleanUpAfterFlush) {
+        if (!AttributeDiskLayout::removeOldSnapshots(_dmsft._snapInfo,
+                    _dmsft._snapInfoLock)) {
+            LOG(warning,
+                "Encountered problems when removing old snapshot directories"
+                "after flushing document meta store '%s' to disk",
+                _dmsft._dms->getBaseFileName().c_str());
+            return false;
+        }
+    }
+    return true;
+}
+
+void
+DocumentMetaStoreFlushTarget::Flusher::run()
+{
+    vespalib::LockGuard guard(_dmsft._flusherLock);
+    if (_syncToken <= _dmsft.getFlushedSerialNum()) {
+        // another flusher has created an equal or better snapshot
+        // after this flusher was created
+        return;
+    }
+    if (!flush()) {
+        // TODO (geirst): throw exception ?
+    }
+    updateStats();
+    if (!cleanUp()) {
+        // TODO (geirst): throw exception ?
+    }
+}
+
+DocumentMetaStoreFlushTarget::
+DocumentMetaStoreFlushTarget(const DocumentMetaStore::SP dms,
+                             ITlsSyncer &tlsSyncer,
+                             const vespalib::string & baseDir,
+                             const TuneFileAttributes &tuneFileAttributes,
+                             const FileHeaderContext &fileHeaderContext)
+    : IFlushTarget("documentmetastore", Type::SYNC, Component::ATTRIBUTE),
+      _dms(dms),
+      _tlsSyncer(tlsSyncer),
+      _baseDir(baseDir),
+      _snapInfo(_baseDir),
+      _snapInfoLock(),
+      _flusherLock(),
+      _cleanUpAfterFlush(true),
+      _lastStats(),
+      _tuneFileAttributes(tuneFileAttributes),
+      _fileHeaderContext(fileHeaderContext),
+      _lastFlushTime()
+
+{
+    if (!_snapInfo.load()) {
+        _snapInfo.save();
+    } else {
+        vespalib::string dirName(getSnapshotDir(getFlushedSerialNum()));
+        _lastFlushTime = search::FileKit::getModificationTime(dirName);
+    }
+    _lastStats.setPathElementsToLog(8);
+}
+
+
+DocumentMetaStoreFlushTarget::~DocumentMetaStoreFlushTarget()
+{
+}
+
+
+vespalib::string
+DocumentMetaStoreFlushTarget::getSnapshotName(uint64_t syncToken)
+{
+    return vespalib::make_string("snapshot-%" PRIu64, syncToken);
+}
+
+
+vespalib::string
+DocumentMetaStoreFlushTarget::getSnapshotDir(uint64_t syncToken)
+{
+    return vespalib::make_string("%s/%s",
+                                 _baseDir.c_str(),
+                                 getSnapshotName(syncToken).c_str());
+}
+
+
+IFlushTarget::SerialNum
+DocumentMetaStoreFlushTarget::getFlushedSerialNum() const
+{
+    vespalib::LockGuard guard(_snapInfoLock);
+    IndexMetaInfo::Snapshot bestSnap = _snapInfo.getBestSnapshot();
+    return bestSnap.valid ? bestSnap.syncToken : 0;
+}
+
+
+IFlushTarget::MemoryGain
+DocumentMetaStoreFlushTarget::getApproxMemoryGain() const
+{
+    int64_t used(_dms->getStatus().getUsed());
+    int64_t canFree = 0;
+    if (_dms->canShrinkLidSpace()) {
+        uint32_t committedDocIdLimit = _dms->getCommittedDocIdLimit();
+        uint32_t numDocs = _dms->getNumDocs();
+        if (committedDocIdLimit < numDocs) {
+            canFree = sizeof(RawDocumentMetaData) *
+                      (numDocs - committedDocIdLimit);
+            if (canFree > used)
+                canFree = used;
+        }
+    }
+    return MemoryGain(used, used - canFree);
+}
+
+
+IFlushTarget::DiskGain
+DocumentMetaStoreFlushTarget::getApproxDiskGain() const
+{
+    return DiskGain(0, 0);
+}
+
+
+IFlushTarget::Time
+DocumentMetaStoreFlushTarget::getLastFlushTime() const
+{
+    return _lastFlushTime;
+}
+
+
+IFlushTarget::Task::UP
+DocumentMetaStoreFlushTarget::initFlush(SerialNum currentSerial)
+{
+    // Called by document db executor
+    (void)currentSerial;
+    _dms->removeAllOldGenerations();
+    SerialNum syncToken = currentSerial;
+    syncToken = std::max(currentSerial,
+                         _dms->getStatus().getLastSyncToken());
+    if (syncToken <= getFlushedSerialNum()) {
+        vespalib::LockGuard guard(_flusherLock);
+        _lastFlushTime = fastos::ClockSystem::now();
+        LOG(debug,
+            "No document meta store to flush."
+            " Update flush time to current: lastFlushTime(%f)",
+            _lastFlushTime.sec());
+        return Task::UP();
+    }
+    return Task::UP(new Flusher(*this, syncToken));
+}
+
+
+uint64_t
+DocumentMetaStoreFlushTarget::getApproxBytesToWriteToDisk() const
+{
+    return _dms->getEstimatedSaveByteSize();
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h
new file mode 100644
index 00000000000..b3851149412
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h
@@ -0,0 +1,115 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/common/indexmetainfo.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include "documentmetastore.h"
+
+namespace search
+{
+
+namespace common
+{
+
+class FileHeaderContext;
+
+}
+
+}
+
+namespace proton
+{
+
+class ITlsSyncer;
+
+using searchcorespi::FlushStats;
+using searchcorespi::IFlushTarget;
+
+/**
+ * Implementation of IFlushTarget interface for document meta store.
+ **/
+class DocumentMetaStoreFlushTarget : public IFlushTarget
+{
+private:
+    /**
+     * Task performing the actual flushing to disk.
+     **/
+    class Flusher : public Task {
+    private:
+        DocumentMetaStoreFlushTarget     &_dmsft;
+        std::unique_ptr<search::AttributeSaver> _saver;
+        uint64_t                          _syncToken;
+        vespalib::string                  _flushDir;
+    public:
+        Flusher(DocumentMetaStoreFlushTarget &dmsft, uint64_t syncToken);
+        ~Flusher();
+        uint64_t getSyncToken() const { return _syncToken; }
+        bool saveSnapInfo();
+        bool flush();
+        void updateStats();
+        bool cleanUp();
+        // Implements vespalib::Executor::Task
+        virtual void run();
+
+        virtual SerialNum
+        getFlushSerial(void) const
+        {
+            return _syncToken;
+        }
+    };
+
+    DocumentMetaStore::SP       _dms;
+    ITlsSyncer                 &_tlsSyncer;
+    vespalib::string            _baseDir;
+    search::IndexMetaInfo       _snapInfo;
+    vespalib::Lock		_snapInfoLock;
+    vespalib::Lock              _flusherLock;
+    bool                        _cleanUpAfterFlush;
+    FlushStats                  _lastStats;
+    const search::TuneFileAttributes _tuneFileAttributes;
+    const search::common::FileHeaderContext &_fileHeaderContext;
+    fastos::TimeStamp           _lastFlushTime;
+    
+    static vespalib::string
+    getSnapshotName(uint64_t syncToken);
+
+    vespalib::string
+    getSnapshotDir(uint64_t syncToken);
+
+public:
+    typedef std::shared_ptr<DocumentMetaStoreFlushTarget> SP;
+
+    /**
+     * Creates a new instance using the given attribute vector and the
+     * given base dir where all attribute vectors are located.
+     **/
+    DocumentMetaStoreFlushTarget(const DocumentMetaStore::SP dms,
+                                 ITlsSyncer &tlsSyncer,
+                                 const vespalib::string &baseDir,
+                                 const search::TuneFileAttributes &
+                                 tuneFileAttributes,
+                                 const search::common::FileHeaderContext &
+                                 fileHeaderContext);
+
+    virtual
+    ~DocumentMetaStoreFlushTarget();
+
+    void setCleanUpAfterFlush(bool cleanUp) { _cleanUpAfterFlush = cleanUp; }
+
+    // Implements IFlushTarget
+    virtual MemoryGain getApproxMemoryGain() const;
+    virtual DiskGain getApproxDiskGain() const;
+    virtual Time getLastFlushTime() const;
+    virtual SerialNum getFlushedSerialNum() const;
+    virtual Task::UP initFlush(SerialNum currentSerial);
+    virtual FlushStats getLastFlushStats() const { return _lastStats; }
+
+    static void initCleanup(const vespalib::string &baseDir);
+    virtual uint64_t getApproxBytesToWriteToDisk() const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.cpp
new file mode 100644
index 00000000000..f0e38cba9bf
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.cpp
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "documentmetastoreinitializer.h"
+#include "documentmetastore.h"
+#include <vespa/searchlib/common/indexmetainfo.h>
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+using search::GrowStrategy;
+using search::IndexMetaInfo;
+using vespalib::IllegalStateException;
+using proton::initializer::InitializerTask;
+
+namespace proton
+{
+
+namespace documentmetastore
+{
+
+DocumentMetaStoreInitializer::
+DocumentMetaStoreInitializer(const vespalib::string baseDir,
+                             const vespalib::string &subDbName,
+                             const vespalib::string &docTypeName,
+                             DocumentMetaStore::SP dms)
+    : _baseDir(baseDir),
+      _subDbName(subDbName),
+      _docTypeName(docTypeName),
+      _dms(dms)
+{
+}
+
+
+void
+DocumentMetaStoreInitializer::run()
+{
+    vespalib::string name = DocumentMetaStore::getFixedName();
+    IndexMetaInfo info(_baseDir);
+    if (info.load()) {
+        IndexMetaInfo::Snapshot snap = info.getBestSnapshot();
+        if (snap.valid) {
+            vespalib::string attrFileName = _baseDir + "/" +
+                                            snap.dirName + "/" +
+                                            name;
+            _dms->setBaseFileName(attrFileName);
+            assert(_dms->hasLoadData());
+            fastos::TimeStamp startTime = fastos::ClockSystem::now();
+            EventLogger::loadDocumentMetaStoreStart(_subDbName);
+            if (!_dms->load()) {
+                throw IllegalStateException(vespalib::make_string(
+                                                    "Failed to load"
+                                                    " document meta store "
+                                                    "for document "
+                                                    "type '%s' from disk",
+                                                    _docTypeName.c_str()));
+            } else {
+                _dms->commit(snap.syncToken,
+                             snap.syncToken);
+            }
+            fastos::TimeStamp endTime = fastos::ClockSystem::now();
+            int64_t elapsedTimeMs = (endTime - startTime).ms();
+            EventLogger::loadDocumentMetaStoreComplete(_subDbName, elapsedTimeMs);
+        }
+    } else {
+        vespalib::mkdir(_baseDir, false);
+        info.save();
+    }
+}
+
+
+} // namespace proton::documentmetastore
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h
new file mode 100644
index 00000000000..8d25f3b52c2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentmetastore.h"
+#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h>
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+#include <vespa/searchcommon/common/growstrategy.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton
+{
+
+namespace documentmetastore
+{
+
+/*
+ * Class representing an Initializer task for loading document meta store
+ * from disk to memory during proton startup.
+ */
+class DocumentMetaStoreInitializer : public initializer::InitializerTask
+{
+    vespalib::string          _baseDir;
+    vespalib::string          _subDbName;
+    vespalib::string          _docTypeName;
+    DocumentMetaStore::SP     _dms;
+
+public:
+    using SP = std::shared_ptr<DocumentMetaStoreInitializer>;
+
+    // Note: lifetime of result must be handled by caller.
+    DocumentMetaStoreInitializer(const vespalib::string baseDir,
+                                 const vespalib::string &subDbName,
+                                 const vespalib::string &docTypeName,
+                                 DocumentMetaStore::SP dms);
+    virtual void run() override;
+};
+
+
+} // namespace proton::documentmetastore
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp
new file mode 100644
index 00000000000..569677cf6a7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.documentmetastoresaver");
+#include "documentmetastoresaver.h"
+#include <vespa/searchlib/util/bufferwriter.h>
+
+using vespalib::GenerationHandler;
+using search::IAttributeSaveTarget;
+
+namespace proton {
+
+namespace {
+
+/*
+ * Functor class to write meta data for a single lid. Note that during
+ * a background save with active feeding, timestamp and bucketused
+ * bits might reflect future values due to missing snapshot properties
+ * in MetaDataStore.
+ */
+class WriteMetaData
+{
+    search::BufferWriter &_datWriter;
+    const RawDocumentMetaData *_metaDataStore;
+    uint32_t _metaDataStoreSize;
+    using MetaDataStore = DocumentMetaStoreSaver::MetaDataStore;
+    using GlobalId = documentmetastore::IStore::GlobalId;
+    using BucketId = documentmetastore::IStore::BucketId;
+    using Timestamp = documentmetastore::IStore::Timestamp;
+public:
+    WriteMetaData(search::BufferWriter &datWriter,
+                 const MetaDataStore &metaDataStore)
+        : _datWriter(datWriter),
+          _metaDataStore(&metaDataStore[0]),
+          _metaDataStoreSize(metaDataStore.size())
+    {
+    }
+
+    void operator()(uint32_t lid) {
+        assert(lid < _metaDataStoreSize);
+        const RawDocumentMetaData &metaData = _metaDataStore[lid];
+        const GlobalId &gid = metaData.getGid();
+        // 6 bits used for bucket bits
+        uint8_t bucketUsedBits = metaData.getBucketUsedBits();
+        assert(bucketUsedBits >= BucketId::minNumBits() &&
+               bucketUsedBits <= BucketId::maxNumBits());
+        assert((bucketUsedBits >> BucketId::CountBits) == 0);
+        Timestamp::Type timestamp = metaData.getTimestamp();
+        search::BufferWriter &datWriter(_datWriter);
+        datWriter.write(&lid, sizeof(lid));
+        datWriter.write(gid.get(), GlobalId::LENGTH);
+        datWriter.write(&bucketUsedBits, sizeof(bucketUsedBits));
+        datWriter.write(&timestamp, sizeof(timestamp));
+    }
+};
+
+
+}
+
+
+DocumentMetaStoreSaver::
+DocumentMetaStoreSaver(vespalib::GenerationHandler::Guard &&guard,
+                             const search::IAttributeSaveTarget::Config &cfg,
+                             const GidIterator &gidIterator,
+                             const MetaDataStore &metaDataStore)
+    : AttributeSaver(std::move(guard), cfg),
+      _gidIterator(gidIterator),
+      _metaDataStore(metaDataStore)
+{
+}
+
+
+DocumentMetaStoreSaver::~DocumentMetaStoreSaver()
+{
+}
+
+
+bool
+DocumentMetaStoreSaver::onSave(IAttributeSaveTarget &saveTarget)
+{
+    // write <lid,gid> pairs, sorted on gid
+    std::unique_ptr<search::BufferWriter>
+        datWriter(saveTarget.datWriter().allocBufferWriter());
+    _gidIterator.foreach_key(WriteMetaData(*datWriter, _metaDataStore));
+    datWriter->flush();
+    return true;
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h
new file mode 100644
index 00000000000..bfa5e15c4c6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/attribute/attributesaver.h>
+#include "documentmetastore.h"
+
+namespace proton {
+
+/*
+ * Class holding the necessary context for saving a document meta
+ * store.  The generation guard in the parent class prevents lids from
+ * being reused during the save operation, but timestamp and
+ * bucketusedbits can reflect future operations relative to when the
+ * document meta store was logically saved.  Thus it is important to
+ * replay the same operations at startup.
+ */
+class DocumentMetaStoreSaver : public search::AttributeSaver
+{
+public:
+    using KeyComp = documentmetastore::LidGidKeyComparator;
+    using DocId = documentmetastore::IStore::DocId;
+    using GidIterator = search::btree::BTreeConstIterator<
+        DocId,
+        search::btree::BTreeNoLeafData,
+        search::btree::NoAggregated,
+        const KeyComp &>;
+    using MetaDataStore = search::attribute::RcuVectorBase<RawDocumentMetaData>;
+
+private:
+    GidIterator _gidIterator; // iterator over frozen tree
+    const MetaDataStore &_metaDataStore;
+
+    virtual bool onSave(search::IAttributeSaveTarget &saveTarget) override;
+public:
+    DocumentMetaStoreSaver(vespalib::GenerationHandler::Guard &&guard,
+                                 const search::IAttributeSaveTarget::Config &cfg,
+                                 const GidIterator &gidIterator,
+                                 const MetaDataStore &metaDataStore);
+
+    virtual ~DocumentMetaStoreSaver();
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/gid_compare.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/gid_compare.h
new file mode 100644
index 00000000000..0d1c2e99dda
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/gid_compare.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/base/globalid.h>
+
+namespace proton {
+namespace documentmetastore {
+
+/**
+ * Interface for comparing global document ids for ordering.
+ */
+class IGidCompare
+{
+public:
+    typedef std::shared_ptr<IGidCompare> SP;
+
+    virtual ~IGidCompare() {}
+
+    virtual bool operator()(const document::GlobalId &lhs,
+                            const document::GlobalId &rhs) const = 0;
+};
+
+
+/**
+ * Default ordering using bucket id order.
+ */
+class DefaultGidCompare : public IGidCompare
+{
+private:
+    document::GlobalId::BucketOrderCmp _comp;
+
+public:
+    DefaultGidCompare()
+        : IGidCompare(),
+          _comp()
+    {
+    }
+
+    virtual bool operator()(const document::GlobalId &lhs,
+                            const document::GlobalId &rhs) const {
+        return _comp(lhs, rhs);
+    }
+};
+
+
+} // namespace documentmetastore
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_bucket_handler.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_bucket_handler.h
new file mode 100644
index 00000000000..601bb66785a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_bucket_handler.h
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h>
+#include <vespa/searchcore/proton/bucketdb/bucketstate.h>
+#include <vespa/searchcore/proton/bucketdb/bucketdeltapair.h>
+#include <vector>
+
+namespace proton {
+
+namespace bucketdb
+{
+
+class SplitBucketSession;
+class JoinBucketsSession;
+
+}
+
+
+namespace documentmetastore {
+
+/**
+ * Interface for handling bucket changes relevant to the document meta store.
+ */
+struct IBucketHandler
+{
+    typedef document::BucketId BucketId;
+
+    virtual ~IBucketHandler() {}
+
+    virtual BucketDBOwner &getBucketDB() const = 0;
+
+    /**
+     * Split the source bucket into two target buckets.
+     */
+    virtual bucketdb::BucketDeltaPair
+    handleSplit(const bucketdb::SplitBucketSession &session) = 0;
+
+    /**
+     * Join the two source buckets into a target bucket.
+     */
+    virtual bucketdb::BucketDeltaPair
+    handleJoin(const bucketdb::JoinBucketsSession &session) = 0;
+
+    /*
+     * Adjust active flag on all lids belonging to given bucket
+     */
+    virtual void updateActiveLids(const BucketId &bucketId, bool active) = 0;
+
+    /**
+     * Sets the bucket state to active / inactive.
+     * Documents that are inactive will be black-listed during search.
+     **/
+    virtual void setBucketState(const BucketId &bucketId, bool active) = 0;
+
+    /**
+     * Sets the bucket state to active, used when adding document db as
+     * part of live reconfig.
+     **/
+    virtual void populateActiveBuckets(const BucketId::List &buckets) = 0;
+
+};
+
+} // namespace documentmetastore
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_document_meta_store.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_document_meta_store.h
new file mode 100644
index 00000000000..9a3d2dbae90
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_document_meta_store.h
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "lid_gid_key_comparator.h"
+#include "i_simple_document_meta_store.h"
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/btree/btree.h>
+#include <vespa/searchlib/btree/btreenodeallocator.h>
+#include <vespa/searchlib/common/idocumentmetastore.h>
+#include <vespa/searchlib/common/serialnum.h>
+
+namespace proton {
+
+/**
+ * Interface used to manage the documents that are contained
+ * in a document sub database with related meta data.
+ *
+ * A document meta store will have storage of <lid, meta data> pairs
+ * (local document id, meta data (including global document id)) and
+ * mapping from lid -> meta data and gid -> lid.
+ **/
+struct IDocumentMetaStore : public search::IDocumentMetaStore,
+                            public ISimpleDocumentMetaStore
+{
+    using search::IDocumentMetaStore::DocId;
+    using search::IDocumentMetaStore::GlobalId;
+    using search::IDocumentMetaStore::BucketId;
+    using search::IDocumentMetaStore::Timestamp;
+
+    // Typedef for the tree used to map from gid -> lid
+    // Lids are stored as keys in the tree, sorted by their gid counterpart.
+    // The LidGidKeyComparator class maps from lids -> metadata by using the metadata store.
+    // TODO(geirst): move this typedef and iterator functions away from this interface.
+    typedef search::btree::BTree<DocId,
+            search::btree::BTreeNoLeafData,
+            search::btree::NoAggregated,
+            const documentmetastore::LidGidKeyComparator &> TreeType;
+    typedef TreeType::Iterator Iterator;
+    typedef std::shared_ptr<IDocumentMetaStore> SP;
+
+    virtual ~IDocumentMetaStore() {}
+
+    /**
+     * Constructs a new underlying free list for lids.
+     * This should be done after a load() and calls to put() and remove().
+     **/
+    virtual void constructFreeList() = 0;
+
+    virtual Iterator begin() const = 0;
+
+    virtual Iterator lowerBound(const BucketId &bucketId) const = 0;
+
+    virtual Iterator upperBound(const BucketId &bucketId) const = 0;
+
+    virtual Iterator lowerBound(const GlobalId &gid) const = 0;
+
+    virtual Iterator upperBound(const GlobalId &gid) const = 0;
+
+    virtual void getLids(const BucketId &bucketId, std::vector<DocId> &lids) = 0;
+
+    virtual search::AttributeGuard getActiveLidsGuard() const = 0;
+
+    /*
+     * Called by document db executor to hold unblocking of shrinking of lid
+     * space after all outstanding holdLid() operations at the time of
+     * compactLidSpace() call have been completed.
+     */
+    virtual void holdUnblockShrinkLidSpace() = 0;
+
+    // Functions that are also defined search::AttributeVector
+    virtual void commit(search::SerialNum firstSerialNum,
+                        search::SerialNum lastSerialNum) = 0;
+    virtual void removeAllOldGenerations() = 0;
+    virtual bool canShrinkLidSpace() const = 0;
+    virtual search::SerialNum getLastSerialNum() const = 0;
+
+    /*
+     * Adjust committedDocIdLimit downwards and prepare for shrinking
+     * of lid space.
+     *
+     * NOTE: Must call unblockShrinkLidSpace() before lid space can
+     * be shrunk.
+     */
+    virtual void compactLidSpace(DocId wantedLidLimit) = 0;
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h
new file mode 100644
index 00000000000..a9ed1560273
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_document_meta_store.h"
+
+namespace proton {
+
+/**
+ * API for providing write and read interface to the document meta store.
+ */
+struct IDocumentMetaStoreContext {
+
+    /**
+     * Guard for access to the read interface.
+     * This guard should be alive as long as read interface is used.
+     */
+    struct IReadGuard {
+
+        typedef std::unique_ptr<IReadGuard> UP;
+
+        virtual ~IReadGuard() {}
+
+        /**
+         * Access to read interface.
+         */
+        virtual const search::IDocumentMetaStore &get() const = 0;
+    };
+
+    typedef std::shared_ptr<IDocumentMetaStoreContext> SP;
+
+    virtual ~IDocumentMetaStoreContext() {}
+
+    /**
+     * Access to write interface.
+     * Should only be used by the writer thread.
+     */
+    virtual proton::IDocumentMetaStore & get() = 0;
+    virtual proton::IDocumentMetaStore::SP getSP() const = 0;
+
+    /**
+     * Access to read interface.
+     * Should be used by all reader threads.
+     */
+    virtual IReadGuard::UP getReadGuard() const = 0;
+
+    /**
+     * Construct free lists of underlying meta store.
+     */
+    virtual void constructFreeList() = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_simple_document_meta_store.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_simple_document_meta_store.h
new file mode 100644
index 00000000000..c13f38301a1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_simple_document_meta_store.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_bucket_handler.h"
+#include "i_store.h"
+
+namespace proton {
+
+/**
+ * Interface for a simple document meta store that combines the
+ * interfaces IStore and IBucketHandler.
+ */
+struct ISimpleDocumentMetaStore : public documentmetastore::IStore,
+                                  public documentmetastore::IBucketHandler
+{
+    virtual ~ISimpleDocumentMetaStore() {}
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h
new file mode 100644
index 00000000000..74442a25599
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h
@@ -0,0 +1,142 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "raw_document_meta_data.h"
+#include <vespa/document/base/globalid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <persistence/spi/types.h>
+
+namespace proton {
+namespace documentmetastore {
+
+
+/**
+ * Interface for storing information about mapping between global document id (gid)
+ * and local document id (lid) with additional meta data per document.
+ **/
+struct IStore
+{
+    typedef uint32_t                DocId;
+    typedef document::GlobalId      GlobalId;
+    typedef document::BucketId      BucketId;
+    typedef storage::spi::Timestamp Timestamp;
+
+    /**
+     * Result after lookup in the store.
+     */
+    struct Result
+    {
+        DocId     _lid;
+        bool      _success;
+        // Info about previous state.
+        bool      _found; // gid was known (due to earlier put or remove)
+        Timestamp _timestamp; // previous timestamp
+
+        Result()
+            : _lid(0u),
+              _success(false),
+              _found(false),
+              _timestamp()
+        {
+        }
+        void setLid(DocId lid) { _lid = lid; }
+        DocId getLid() const { return _lid; }
+        bool ok() const { return _success; }
+        void markSuccess() { _success = true; }
+        void fillPrev(Timestamp prevTimestamp) {
+            _found = true;
+            _timestamp = prevTimestamp;
+        }
+    };
+
+    virtual ~IStore() {}
+
+    /**
+     * Inspect the meta data associated with the given gid.
+     * If the gid is not found the result is not valid.
+     */
+    virtual Result inspectExisting(const GlobalId &gid) const = 0;
+
+    /**
+     * Inspect the meta data associated with the given gid.
+     * If the gid is not found the next available lid is returned in the result.
+     * This lid can be used if calling put() right afterwards.
+     */
+    virtual Result inspect(const GlobalId &gid) = 0;
+
+    /**
+     * Puts the given <lid, meta data> pair to this store.
+     * This function should assert that the given lid is the same
+     * as returned from inspect().
+     **/
+    virtual Result put(const GlobalId &gid,
+                       const BucketId &bucketId,
+                       const Timestamp &timestamp,
+                       DocId lid) = 0;
+
+    /*
+     * Update the meta data associated with the given lid.
+     * Used when handling partial updates.
+     * Returns false if there is no entry for the given lid.
+     */
+    virtual bool updateMetaData(DocId lid,
+                                const BucketId &bucketId,
+                                const Timestamp &timestamp) = 0;
+
+    /**
+     * Removes the <lid, meta data> pair with the given lid from this
+     * store. Returns false if the <lid, meta data> pair was not
+     * found or could not be removed.
+     * The caller must call removeComplete() after document removal is done.
+     **/
+    virtual bool remove(DocId lid) = 0;
+
+    /**
+     * Signal that the removal of the document associated with this lid is complete.
+     * This is typically called after the document has been removed from all
+     * other data structures. The lid is now a candidate for later reuse.
+     */
+    virtual void removeComplete(DocId lid) = 0;
+
+    /**
+     * Move meta data for fromLid to toLid. Mapping from gid to lid
+     * is updated atomically from fromLid to toLid.
+     * The caller must call removeComplete() with fromLid after document move is done.
+     */
+    virtual void move(DocId fromLid, DocId toLid) = 0;
+
+    /**
+     * Check if the lid is valid.
+     * Returns true if valid, false otherwise.
+     **/
+    virtual bool validLid(DocId lid) const = 0;
+
+    /**
+     * Removes a list of lids.
+     * The caller must call removeBatchComplete() after documents removal is done.
+     */
+    virtual void removeBatch(const std::vector<DocId> &lidsToRemove,
+                             const DocId docIdLimit) = 0;
+
+    /**
+     * Signal that the removal of the documents associated with these lids is complete.
+     */
+    virtual void removeBatchComplete(const std::vector<DocId> &lidsToRemove) = 0;
+
+    /**
+     * Returns the raw meta data stored for the given lid.
+     */
+    virtual const RawDocumentMetaData &getRawMetaData(DocId lid) const = 0;
+
+    /**
+     * Check if free list is active.
+     *
+     * Returns true if free list is active.
+     */
+    virtual bool getFreeListActive() const = 0;
+};
+
+} // namespace documentmetastore
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h
new file mode 100644
index 00000000000..3fabfbe133c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+namespace documentmetastore
+{
+
+/**
+ * Interface used to delay reuse of lids until references to the lids have
+ * been purged from the data structures in memory index and attribute vectors.
+ */
+class ILidReuseDelayer
+{
+public:
+    virtual ~ILidReuseDelayer() { }
+    /**
+     * Delay reuse of a single lid.
+     *
+     * @param lid          The lid for which to delay reuse
+     *
+     * @return bool        True if caller must handle lid reuse explicitly
+     */
+    virtual bool delayReuse(uint32_t lid) = 0;
+    /**
+     * Delay reuse of multiple lids.
+     *
+     * @param lids         The lids for which to delay reuse
+     *
+     * @return bool        True if caller must handle lid reuse explicitly
+     */
+    virtual bool delayReuse(const std::vector<uint32_t> &lids) = 0;
+    virtual void setImmediateCommit(bool immediateCommit) = 0;
+    virtual bool getImmediateCommit() const = 0;
+    virtual void setHasIndexedFields(bool hasIndexedFields) = 0;
+    virtual std::vector<uint32_t> getReuseLids() = 0;
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp
new file mode 100644
index 00000000000..2e22b14eccc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp
@@ -0,0 +1,329 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.lid_allocator");
+
+#include "lid_allocator.h"
+
+using search::fef::TermFieldMatchDataArray;
+using search::queryeval::Blueprint;
+using search::queryeval::FieldSpecBaseList;
+using search::queryeval::SimpleLeafBlueprint;
+using search::queryeval::SearchIterator;
+using search::AttributeVector;
+using vespalib::GenerationHolder;
+using search::QueryTermSimple;
+
+namespace proton {
+namespace documentmetastore {
+
+LidAllocator::LidAllocator(uint32_t size,
+                           uint32_t capacity,
+                           GenerationHolder &genHolder)
+    : _holdLids(),
+      _freeLids(size,
+                capacity,
+                genHolder,
+                true,
+                false),
+      _usedLids(size,
+                capacity,
+                genHolder,
+                false,
+                true),
+      _pendingHoldLids(size,
+                       capacity,
+                       genHolder,
+                       false,
+                       false),
+      _lidFreeListConstructed(false),
+      _activeLidsAttr(new BitAttribute("[activelids]")),
+      _activeLids(static_cast<BitAttribute &>(*_activeLidsAttr)),
+      _numActiveLids(0u)
+{
+
+}
+
+LidAllocator::DocId
+LidAllocator::getFreeLid(DocId lidLimit)
+{
+    DocId lid = _freeLids.getLowest();
+    if (lid >= lidLimit) {
+        lid = lidLimit;
+    } else {
+        _freeLids.clearBit(lid);
+    }
+    return lid;
+}
+
+LidAllocator::DocId
+LidAllocator::peekFreeLid(DocId lidLimit)
+{
+    DocId lid = _freeLids.getLowest();
+    if (lid >= lidLimit) {
+        lid = lidLimit;
+    }
+    return lid;
+}
+
+void
+LidAllocator::ensureSpace(uint32_t newSize,
+                          uint32_t newCapacity)
+{
+    _freeLids.resizeVector(newSize, newCapacity);
+    _usedLids.resizeVector(newSize, newCapacity);
+    _pendingHoldLids.resizeVector(newSize, newCapacity);
+}
+
+void
+LidAllocator::ensureSpace(DocId lid,
+                          uint32_t newSize,
+                          uint32_t newCapacity)
+{
+    ensureSpace(newSize, newCapacity);
+    while(lid >= _activeLids.getNumDocs()) {
+        DocId activeLid;
+        _activeLids.addDoc(activeLid);
+        _activeLids.commit();
+    }
+}
+
+void
+LidAllocator::markAsActive(DocId lid)
+{
+    if (_activeLids.get(lid) == 0) {
+        ++_numActiveLids;
+    }
+    _activeLids.update(lid, 1);
+    _activeLids.commit();
+}
+
+void
+LidAllocator::unregisterLid(DocId lid)
+{
+    assert(!_pendingHoldLids.testBit(lid));
+    if (isFreeListConstructed()) {
+        _pendingHoldLids.setBit(lid);
+    }
+    _usedLids.clearBit(lid);
+    if (_activeLids.get(lid) != 0) {
+        --_numActiveLids;
+    }
+    _activeLids.update(lid, 0);
+    _activeLids.commit();
+}
+
+size_t
+LidAllocator::getUsedLidsSize() const
+{
+    return _usedLids.byteSize();
+}
+
+void
+LidAllocator::trimHoldLists(generation_t firstUsed)
+{
+    _holdLids.trimHoldLists(firstUsed, _freeLids);
+}
+
+void
+LidAllocator::moveLidBegin(DocId fromLid, DocId toLid)
+{
+    assert(!_pendingHoldLids.testBit(fromLid));
+    assert(!_pendingHoldLids.testBit(toLid));
+    if (isFreeListConstructed()) {
+        assert(!_freeLids.testBit(fromLid));
+        assert(_freeLids.testBit(toLid));
+        _freeLids.clearBit(toLid);
+    }
+}
+
+void
+LidAllocator::moveLidEnd(DocId fromLid, DocId toLid)
+{
+    if (isFreeListConstructed()) {
+        // old lid must be scheduled for hold by caller
+        _pendingHoldLids.setBit(fromLid);
+    }
+    _usedLids.setBit(toLid);
+    _usedLids.clearBit(fromLid);
+    _activeLids.update(toLid, _activeLids.get(fromLid));
+    _activeLids.update(fromLid, 0);
+    _activeLids.commit();
+}
+
+void
+LidAllocator::holdLid(DocId lid,
+                      DocId lidLimit,
+                      generation_t currentGeneration)
+{
+    assert(holdLidOK(lid, lidLimit));
+    assert(isFreeListConstructed());
+    assert(lid < _usedLids.size());
+    assert(lid < _pendingHoldLids.size());
+    assert(_pendingHoldLids.testBit(lid));
+    _pendingHoldLids.clearBit(lid);
+    _holdLids.add(lid, currentGeneration);
+}
+
+void
+LidAllocator::holdLids(const std::vector<DocId> &lids,
+                       DocId lidLimit,
+                       generation_t currentGeneration)
+{
+    for (const auto &lid : lids) {
+        assert(lid > 0);
+        assert(holdLidOK(lid, lidLimit));
+        _pendingHoldLids.clearBit(lid);
+        _holdLids.add(lid, currentGeneration);
+    }
+}
+
+bool
+LidAllocator::holdLidOK(DocId lid, DocId lidLimit) const
+{
+    if (_lidFreeListConstructed &&
+        lid != 0 &&
+        lid < lidLimit &&
+        lid < _usedLids.size() &&
+        lid < _pendingHoldLids.size() &&
+        _pendingHoldLids.testBit(lid))
+    {
+        return true;
+    }
+    LOG(error,
+        "LidAllocator::holdLidOK(%u, %u): "
+        "_lidFreeListConstructed=%s, "
+        "_usedLids.size()=%d, "
+        "_pendingHoldLids.size()=%d, "
+        "_pendingHoldLids bit=%s",
+        lid, lidLimit,
+        _lidFreeListConstructed ? "true" : "false",
+        (int) _usedLids.size(),
+        (int) _pendingHoldLids.size(),
+        lid < _pendingHoldLids.size() ?
+        (_pendingHoldLids.testBit(lid) ?
+         "true" : "false" ) : "invalid"
+        );
+    return false;
+}
+
+void
+LidAllocator::constructFreeList(DocId lidLimit)
+{
+    assert(!isFreeListConstructed());
+    _holdLids.clear();
+    for (uint32_t lid = 1; lid < lidLimit; ++lid) {
+        if (!validLid(lid)) {
+            _freeLids.setBit(lid);
+        }
+    }
+}
+
+namespace {
+
+class BlackListBlueprint : public SimpleLeafBlueprint
+{
+private:
+    AttributeVector::SearchContext::UP _searchCtx;
+    vespalib::Lock _lock;
+    mutable std::vector<search::fef::TermFieldMatchData *> _matchDataVector;
+
+    virtual SearchIterator::UP
+    createLeafSearch(const TermFieldMatchDataArray &tfmda,
+                     bool strict) const
+    {
+        assert(tfmda.size() == 0);
+        (void) tfmda;
+        search::fef::TermFieldMatchData *tfmd =
+            new search::fef::TermFieldMatchData;
+        {
+            vespalib::LockGuard lock(_lock);
+            _matchDataVector.push_back(tfmd);
+        }
+        return _searchCtx->createIterator(tfmd, strict);
+    }
+
+    virtual void
+    fetchPostings(bool strict)
+    {
+        _searchCtx->fetchPostings(strict);
+    }
+
+public:
+    BlackListBlueprint(AttributeVector::SearchContext::UP searchCtx)
+        : SimpleLeafBlueprint(FieldSpecBaseList()),
+          _searchCtx(std::move(searchCtx)),
+          _matchDataVector()
+    {
+        setEstimate(HitEstimate(0, false));
+    }
+
+    ~BlackListBlueprint() {
+        for (auto matchData : _matchDataVector) {
+            delete matchData;
+        }
+    }
+};
+
+}
+
+Blueprint::UP
+LidAllocator::createBlackListBlueprint() const
+{
+    QueryTermSimple::UP term(new QueryTermSimple("0", QueryTermSimple::WORD));
+    return Blueprint::UP(
+            new BlackListBlueprint(_activeLids.getSearch(std::move(term),
+                                                         AttributeVector::SearchContext::Params())));
+}
+
+void
+LidAllocator::updateActiveLids(DocId lid, bool active)
+{
+    int8_t oldActiveFlag = _activeLids.get(lid);
+    int8_t newActiveFlag = (active ? 1 : 0);
+    if (oldActiveFlag != newActiveFlag) {
+        if (oldActiveFlag != 0) {
+            if (newActiveFlag == 0) {
+                --_numActiveLids;
+            }
+        } else {
+            ++_numActiveLids;
+        }
+    }
+    _activeLids.update(lid, newActiveFlag);
+}
+
+void
+LidAllocator::commitActiveLids()
+{
+    _activeLids.commit();
+}
+
+void
+LidAllocator::clearDocs(DocId lidLow, DocId lidLimit)
+{
+    assert(_usedLids.getNextTrueBit(lidLow) >= lidLimit);
+}
+
+void
+LidAllocator::compactLidSpace(uint32_t wantedLidLimit)
+{
+    _activeLids.compactLidSpace(wantedLidLimit);
+}
+
+void
+LidAllocator::shrinkLidSpace(DocId committedDocIdLimit)
+{
+    _activeLids.shrinkLidSpace();
+    ensureSpace(committedDocIdLimit, committedDocIdLimit);
+}
+
+uint32_t
+LidAllocator::getNumUsedLids() const
+{
+    return _usedLids.count();
+}
+
+} // namespace documentmetastore
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h
new file mode 100644
index 00000000000..0922f9d4edd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "lid_hold_list.h"
+#include "lidstatevector.h"
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/singlesmallnumericattribute.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+
+namespace proton {
+namespace documentmetastore {
+
+/**
+ * Class responsible for allocating lids and managing
+ * which lids are used, active and free.
+ */
+class LidAllocator
+{
+private:
+    typedef search::SingleValueBitNumericAttribute BitAttribute;
+    typedef uint32_t DocId;
+    typedef vespalib::GenerationHandler::generation_t generation_t;
+
+    LidHoldList                 _holdLids;
+    LidStateVector              _freeLids;
+    LidStateVector              _usedLids;
+    LidStateVector              _pendingHoldLids;
+    bool                        _lidFreeListConstructed;
+    search::AttributeVector::SP _activeLidsAttr;
+    BitAttribute               &_activeLids;
+    uint32_t                    _numActiveLids;
+
+public:
+    LidAllocator(uint32_t size,
+                 uint32_t capacity,
+                 vespalib::GenerationHolder &genHolder);
+
+    DocId getFreeLid(DocId lidLimit);
+    DocId peekFreeLid(DocId lidLimit);
+    void ensureSpace(uint32_t newSize,
+                     uint32_t newCapacity);
+    void ensureSpace(DocId lid,
+                     uint32_t newSize,
+                     uint32_t newCapacity);
+    void registerLid(DocId lid) { _usedLids.setBit(lid); }
+    void markAsActive(DocId lid);
+    void unregisterLid(DocId lid);
+    size_t getUsedLidsSize() const;
+    void trimHoldLists(generation_t firstUsed);
+    void moveLidBegin(DocId fromLid, DocId toLid);
+    void moveLidEnd(DocId fromLid, DocId toLid);
+    void holdLid(DocId lid, DocId lidLimit, generation_t currentGeneration);
+    void holdLids(const std::vector<DocId> &lids, DocId lidLimit,
+                  generation_t currentGeneration);
+    bool holdLidOK(DocId lid, DocId lidLimit) const;
+    void constructFreeList(DocId lidLimit);
+    search::queryeval::Blueprint::UP createBlackListBlueprint() const;
+    void updateActiveLids(DocId lid, bool active);
+    void commitActiveLids();
+    void clearDocs(DocId lidLow, DocId lidLimit);
+    void compactLidSpace(DocId wantedLidLimit);
+    void shrinkLidSpace(DocId committedDocIdLimit);
+    uint32_t getNumUsedLids() const;
+    uint32_t getNumActiveLids() const {
+        return _numActiveLids;
+    }
+    void setFreeListConstructed() {
+        _lidFreeListConstructed = true;
+    }
+    bool isFreeListConstructed() const {
+        return _lidFreeListConstructed;
+    }
+    bool validLid(DocId lid) const {
+        return (lid < _usedLids.size() && _usedLids.testBit(lid));
+    }
+    DocId getLowestFreeLid() const {
+        return _freeLids.getLowest();
+    }
+    DocId getHighestUsedLid() const {
+        return _usedLids.getHighest();
+    }
+    search::AttributeGuard getActiveLidsGuard() const {
+        return search::AttributeGuard(_activeLidsAttr);
+    }
+    const BitAttribute &getActiveLids() const {
+        return _activeLids;
+    }
+
+};
+
+} // namespace documentmetastore
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.cpp
new file mode 100644
index 00000000000..55f8f3f7cbe
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.cpp
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.lid_gid_key_comparator");
+
+#include "lid_gid_key_comparator.h"
+
+namespace proton {
+namespace documentmetastore {
+
+const search::IDocumentMetaStore::DocId
+LidGidKeyComparator::FIND_DOC_ID = std::numeric_limits<DocId>::max();
+
+LidGidKeyComparator::LidGidKeyComparator(const document::GlobalId &gid,
+                                         const MetaDataStore &metaDataStore,
+                                         const IGidCompare &gidCompare)
+    : _gid(gid),
+      _metaDataStore(metaDataStore),
+      _gidCompare(gidCompare)
+{
+}
+
+LidGidKeyComparator::LidGidKeyComparator(const RawDocumentMetaData &metaData,
+                                         const MetaDataStore &metaDataStore,
+                                         const IGidCompare &gidCompare)
+    : _gid(metaData.getGid()),
+      _metaDataStore(metaDataStore),
+      _gidCompare(gidCompare)
+{
+}
+
+}
+}
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.h
new file mode 100644
index 00000000000..6c6abcf2a71
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.h
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "gid_compare.h"
+#include "raw_document_meta_data.h"
+#include <vespa/document/base/globalid.h>
+#include <vespa/searchlib/common/rcuvector.h>
+#include <vespa/searchlib/common/idocumentmetastore.h>
+
+namespace proton {
+namespace documentmetastore {
+
+/**
+ * Comparator class used by the lid<->gid btree to get the lids
+ * sorted by their gid counterpart.
+ **/
+class LidGidKeyComparator
+{
+public:
+    static const search::IDocumentMetaStore::DocId FIND_DOC_ID;
+
+private:
+    typedef search::IDocumentMetaStore::DocId DocId;
+    typedef search::attribute::RcuVectorBase<RawDocumentMetaData> MetaDataStore;
+
+    const document::GlobalId &_gid;
+    const MetaDataStore      &_metaDataStore;
+    const IGidCompare        &_gidCompare;
+
+    const document::GlobalId &getGid(DocId lid) const {
+        if (lid != FIND_DOC_ID) {
+            return _metaDataStore[lid].getGid();
+        }
+        return _gid;
+    }
+
+public:
+    /**
+     * Creates a comparator that returns the given gid if
+     * FIND_DOC_ID is encountered. Otherwise the metadata store is
+     * used to map from lid -> metadata (including gid).
+     **/
+    LidGidKeyComparator(const document::GlobalId &gid,
+                        const MetaDataStore &metaDataStore,
+                        const IGidCompare &gidCompare);
+
+    LidGidKeyComparator(const RawDocumentMetaData &metaData,
+                        const MetaDataStore &metaDataStore,
+                        const IGidCompare &gidCompare);
+
+    bool operator()(const DocId &lhs, const DocId &rhs) const {
+            return _gidCompare(getGid(lhs), getGid(rhs));
+    }
+
+};
+
+} // namespace documentmetastore
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.h
new file mode 100644
index 00000000000..b2863a7269b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_hold_list.h
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "lidstatevector.h"
+#include <vespa/vespalib/util/generationhandler.h>
+#include <deque>
+
+namespace proton {
+
+/**
+ * Class used to hold <lid, generation> pairs before reuse.
+ * A lid is free for reuse if the associated generation < first used
+ * generation by readers.
+ **/
+class LidHoldList
+{
+private:
+    typedef vespalib::GenerationHandler::generation_t generation_t;
+    typedef std::pair<uint32_t, generation_t> Element;
+    typedef std::deque<Element> ElementDeque;
+
+    ElementDeque _holdList;
+
+public:
+    LidHoldList()
+        : _holdList()
+    {
+    }
+
+    /**
+     * Adds a new element with the given generation.
+     * Elements must be added with ascending generations.
+     **/
+    void add(const uint32_t data, generation_t generation) {
+        if (!_holdList.empty()) {
+            assert(generation >= _holdList.back().second);
+        }
+        _holdList.push_back(std::make_pair(data, generation));
+    }
+
+    /**
+     * Returns the total number of elements.
+     **/
+    size_t size() const { return _holdList.size(); }
+
+    /**
+     * Clears the free list.
+     **/
+    void clear() { _holdList.clear(); }
+
+    /**
+     * Frees up elements with generation < first used generation for reuse.
+     **/
+    void trimHoldLists(generation_t firstUsed, LidStateVector &freeLids)
+    {
+        while (!_holdList.empty() && _holdList.front().second < firstUsed) {
+            uint32_t lid = _holdList.front().first;
+            freeLids.setBit(lid);
+            _holdList.pop_front();
+        }
+    }
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp
new file mode 100644
index 00000000000..c38b040cb91
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "lid_reuse_delayer_config.h"
+#include <vespa/searchcore/proton/server/documentdbconfig.h>
+
+namespace proton
+{
+
+namespace documentmetastore
+{
+
+
+LidReuseDelayerConfig::LidReuseDelayerConfig()
+    : _visibilityDelay(0),
+      _hasIndexedFields(false)
+{
+}
+
+LidReuseDelayerConfig::LidReuseDelayerConfig(const DocumentDBConfig &
+                                             configSnapshot)
+    : _visibilityDelay(configSnapshot.getMaintenanceConfigSP()->
+                       getVisibilityDelay()),
+      _hasIndexedFields(configSnapshot.getSchemaSP()->getNumIndexFields() > 0)
+{
+}
+
+
+} // namespace proton::documentmetastore
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h
new file mode 100644
index 00000000000..7a5920dbf5f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+class DocumentDBConfig;
+
+namespace documentmetastore
+{
+
+/*
+ * Class representing configuration for lid reuse delayer.
+ */
+class LidReuseDelayerConfig
+{
+private:
+    fastos::TimeStamp                _visibilityDelay;
+    bool                             _hasIndexedFields;
+public:
+    LidReuseDelayerConfig();
+    explicit LidReuseDelayerConfig(const DocumentDBConfig &configSnapshot);
+    fastos::TimeStamp visibilityDelay() const { return _visibilityDelay; }
+    bool hasIndexedFields() const { return _hasIndexedFields; }
+};
+
+} // namespace proton::documentmetastore
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp
new file mode 100644
index 00000000000..ef904cadf92
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp
@@ -0,0 +1,124 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.lidreusedelayer");
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include "i_store.h"
+#include "lidreusedelayer.h"
+
+namespace proton
+{
+
+namespace documentmetastore
+{
+
+using searchcorespi::index::IThreadingService;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+
+LidReuseDelayer::LidReuseDelayer(IThreadingService &writeService,
+                                 IStore &documentMetaStore)
+    : _writeService(writeService),
+      _documentMetaStore(documentMetaStore),
+      _immediateCommit(true),
+      _hasIndexedFields(false),
+      _pendingLids()
+{
+}
+
+
+LidReuseDelayer::~LidReuseDelayer()
+{
+}
+
+
+void
+LidReuseDelayer::performDelayReuseLid(uint32_t lid)
+{
+    assert(_writeService.master().isCurrentThread());
+    _documentMetaStore.removeComplete(lid);
+}
+
+void
+LidReuseDelayer::performDelayReuseLids(const std::vector<uint32_t> lids)
+{
+    assert(_writeService.master().isCurrentThread());
+    _documentMetaStore.removeBatchComplete(lids);
+}
+
+
+bool
+LidReuseDelayer::delayReuse(uint32_t lid)
+{
+    assert(_writeService.master().isCurrentThread());
+    if (!_documentMetaStore.getFreeListActive())
+        return false;
+    if (!_immediateCommit) {
+        _pendingLids.push_back(lid);
+        return false;
+    }
+    if (!_hasIndexedFields) {
+        _documentMetaStore.removeComplete(lid);
+        return false;
+    }
+    return true;
+}
+
+
+bool
+LidReuseDelayer::delayReuse(const std::vector<uint32_t> &lids)
+{
+    assert(_writeService.master().isCurrentThread());
+    if (!_documentMetaStore.getFreeListActive() || lids.empty())
+        return false;
+    if (!_immediateCommit) {
+        _pendingLids.insert(_pendingLids.end(), lids.cbegin(), lids.cend());
+        return false;
+    }
+    if (!_hasIndexedFields) {
+        _documentMetaStore.removeBatchComplete(lids);
+        return false;
+    }
+    return true;
+}
+
+
+void
+LidReuseDelayer::setImmediateCommit(bool immediateCommit)
+{
+    assert(_pendingLids.empty());
+    _immediateCommit = immediateCommit;
+}
+
+
+bool
+LidReuseDelayer::getImmediateCommit() const
+{
+    return _immediateCommit;
+}
+
+
+void
+LidReuseDelayer::setHasIndexedFields(bool hasIndexedFields)
+{
+    assert(_pendingLids.empty());
+    _hasIndexedFields = hasIndexedFields;
+}
+
+
+std::vector<uint32_t>
+LidReuseDelayer::getReuseLids()
+{
+    std::vector<uint32_t> result;
+    assert(_writeService.master().isCurrentThread());
+    result = _pendingLids;
+    _pendingLids.clear();
+    return result;
+}
+
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h
new file mode 100644
index 00000000000..14712f2e66d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "ilidreusedelayer.h"
+
+namespace searchcorespi
+{
+
+namespace index
+{
+
+class IThreadingService;
+
+}
+
+}
+
+namespace proton
+{
+
+namespace documentmetastore
+{
+
+class IStore;
+
+/**
+ * This class delays reuse of lids until references to the lids have
+ * been purged from the data structures in memory index and attribute
+ * vectors.
+ *
+ * Note that an additional delay is added by the IStore component,
+ * where lids are put on a hold list to ensure that queries started
+ * before lid was purged also blocks reuse of lid.
+ *
+ * Currently only works correctly when visibility delay is 0.
+ */
+class LidReuseDelayer : public ILidReuseDelayer
+{
+    searchcorespi::index::IThreadingService &_writeService;
+    IStore &_documentMetaStore;
+    bool _immediateCommit;
+    bool _hasIndexedFields;
+    std::vector<uint32_t> _pendingLids; // lids waiting for commit
+
+    void
+    performDelayReuseLid(uint32_t lid);
+
+    void
+    performDelayReuseLids(const std::vector<uint32_t> lids);
+
+public:
+    LidReuseDelayer(searchcorespi::index::IThreadingService &writeService,
+                    IStore &documentMetaStore);
+    virtual ~LidReuseDelayer();
+    virtual bool delayReuse(uint32_t lid) override;
+    virtual bool delayReuse(const std::vector<uint32_t> &lids) override;
+    virtual void setImmediateCommit(bool immediateCommit) override;
+    virtual bool getImmediateCommit() const override;
+    virtual void setHasIndexedFields(bool hasIndexedFields) override;
+    virtual std::vector<uint32_t> getReuseLids() override;
+};
+
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp
new file mode 100644
index 00000000000..8d3264ba241
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp
@@ -0,0 +1,163 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.lidstatevector");
+#include "lidstatevector.h"
+
+namespace proton
+{
+
+using vespalib::GenerationHolder;
+
+LidStateVector::LidStateVector(unsigned int newSize,
+                               unsigned int newCapacity,
+                               GenerationHolder &generationHolder,
+                               bool trackLowest,
+                               bool trackHighest)
+    : _bv(newSize, newCapacity, generationHolder),
+      _lowest(trackLowest ? newSize : 0u),
+      _highest(0),
+      _count(0u),
+      _trackLowest(trackLowest),
+      _trackHighest(trackHighest)
+{
+}
+
+
+LidStateVector::~LidStateVector()
+{
+}
+
+
+void
+LidStateVector::resizeVector(uint32_t newSize, uint32_t newCapacity)
+{
+    assert(!_trackLowest || _lowest <= _bv.size());
+    assert(!_trackHighest || _bv.size() == 0 || _highest < _bv.size());
+    bool nolowest(_lowest == _bv.size());
+    if (_bv.size() > newSize) {
+        _bv.shrink(newSize);
+        assert(_count >= internalCount());
+        _count = internalCount();
+    }
+    if (_bv.capacity() < newCapacity) {
+        _bv.reserve(newCapacity);
+        assert(_count == internalCount());
+    }
+    if (_bv.size() < newSize) {
+        _bv.extend(newSize);
+        assert(_count == internalCount());
+    }
+    if (_trackLowest) {
+        if (nolowest) {
+            _lowest = _bv.size();
+        }
+        if (_lowest > _bv.size()) {
+            _lowest = _bv.size();
+        }
+    }
+    if (_trackHighest) {
+        if (_highest >= _bv.size()) {
+            _highest = _bv.size() > 0 ? _bv.getPrevTrueBit(_bv.size() - 1) : 0;
+        }
+    }
+    maybeUpdateLowest();
+    maybeUpdateHighest();
+}
+
+
+void
+LidStateVector::updateLowest(void)
+{
+    if (_lowest >= _bv.size())
+        return;
+    if (_bv.testBit(_lowest))
+        return;
+    uint32_t lowest = _bv.getNextTrueBit(_lowest);
+    assert(lowest <= _bv.size());
+    _lowest = lowest;
+}
+
+
+void
+LidStateVector::updateHighest(void)
+{
+    if (_highest == 0)
+        return;
+    if (_bv.testBit(_highest))
+        return;
+    uint32_t highest = _bv.getPrevTrueBit(_highest);
+    assert(_bv.size() == 0 || highest < _bv.size());
+    _highest = highest;
+}
+
+
+void
+LidStateVector::setBit(unsigned int idx)
+{
+    assert(idx < _bv.size());
+    if (_trackLowest && idx < _lowest) {
+        _lowest = idx;
+    }
+    if (_trackHighest && idx > _highest) {
+        _highest = idx;
+    }
+    assert(!_bv.testBit(idx));
+    _bv.slowSetBit(idx);
+    ++_count;
+    assert(_count == internalCount());
+}
+
+
+void
+LidStateVector::clearBit(unsigned int idx)
+{
+    assert(idx < _bv.size());
+    assert(_bv.testBit(idx));
+    _bv.slowClearBit(idx);
+    --_count;
+    assert(_count == internalCount());
+    maybeUpdateLowest();
+    maybeUpdateHighest();
+}
+
+
+bool
+LidStateVector::empty(void) const
+{
+    return _count == 0u;
+}
+
+
+unsigned int
+LidStateVector::getLowest(void) const
+{
+    return _lowest;
+}
+
+
+unsigned int
+LidStateVector::getHighest(void) const
+{
+    return _highest;
+}
+
+
+uint32_t
+LidStateVector::internalCount(void)
+{
+    // Called by document db executor thread.
+    return _bv.countTrueBits();
+}
+
+
+uint32_t
+LidStateVector::count(void) const
+{
+    // Called by document db executor thread or metrics related threads
+    return _count;
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h
new file mode 100644
index 00000000000..8c8c41f6184
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h
@@ -0,0 +1,106 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/growablebitvector.h>
+
+namespace proton
+{
+
+class LidStateVector
+{
+    search::GrowableBitVector _bv;
+    uint32_t _lowest;
+    uint32_t _highest;
+    uint32_t _count;
+    bool _trackLowest;
+    bool _trackHighest;
+
+    void
+    updateLowest(void);
+
+    void
+    updateHighest(void);
+
+    inline void
+    maybeUpdateLowest(void)
+    {
+        if (_trackLowest && _lowest < _bv.size() && !_bv.testBit(_lowest))
+            updateLowest();
+    }
+
+    inline void
+    maybeUpdateHighest(void)
+    {
+        if (_trackHighest && _highest != 0 && !_bv.testBit(_highest))
+            updateHighest();
+    }
+
+    /**
+     * Get number of bits set in vector.  Should only be called by
+     * write thread.
+     */
+    uint32_t
+    internalCount(void);
+public:
+    
+    LidStateVector(unsigned int newSize,
+                   unsigned int newCapacity,
+                   vespalib::GenerationHolder &generationHolder,
+                   bool trackLowest,
+                   bool trackHighest);
+
+    ~LidStateVector();
+
+    void
+    resizeVector(uint32_t newSize, uint32_t newCapacity);
+
+    void
+    setBit(unsigned int idx);
+
+    void
+    clearBit(unsigned int idx);
+
+    inline bool
+    testBit(unsigned int idx) const
+    {
+        return _bv.testBit(idx);
+    }
+
+    inline unsigned int
+    size(void) const
+    {
+        return _bv.size();
+    }
+
+    inline unsigned int
+    byteSize(void) const
+    {
+        return _bv.extraByteSize() + sizeof(LidStateVector);
+    }
+
+    bool
+    empty(void) const;
+
+    unsigned int
+    getLowest(void) const;
+    
+    unsigned int
+    getHighest(void) const;
+
+    /**
+     * Get cached number of bits set in vector.  Called by read or
+     * write thread.  Write thread must updated cached number as needed.
+     */
+    uint32_t
+    count(void) const;
+
+    unsigned int
+    getNextTrueBit(unsigned int idx) const
+    {
+        return _bv.getNextTrueBit(idx);
+    }
+}; 
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h
new file mode 100644
index 00000000000..1ccf6b5ccf4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h
@@ -0,0 +1,134 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/base/globalid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <persistence/spi/types.h>
+
+namespace proton {
+
+/**
+ * The raw data that is stored for a single document in the DocumentMetaStore.
+ */
+struct RawDocumentMetaData
+{
+    typedef document::GlobalId GlobalId;
+    typedef document::BucketId BucketId;
+    typedef storage::spi::Timestamp Timestamp;
+    GlobalId _gid;
+    uint8_t _bucketUsedBits;
+    Timestamp _timestamp;
+
+    RawDocumentMetaData(void)
+        : _gid(),
+          _bucketUsedBits(BucketId::minNumBits()),
+          _timestamp()
+    {
+    }
+
+    RawDocumentMetaData(const GlobalId &gid,
+             const BucketId &bucketId,
+             const Timestamp &timestamp)
+        : _gid(gid),
+          _bucketUsedBits(bucketId.getUsedBits()),
+          _timestamp(timestamp)
+    {
+        assert(bucketId.valid());
+        BucketId verId(gid.convertToBucketId());
+        verId.setUsedBits(_bucketUsedBits);
+        assert(bucketId.getRawId() == verId.getRawId() ||
+               bucketId.getRawId() == verId.getId());
+    }
+
+    bool
+    operator<(const GlobalId &rhs) const
+    {
+        return _gid < rhs;
+    }
+
+    bool
+    operator==(const GlobalId &rhs) const
+    {
+        return _gid == rhs;
+    }
+
+    bool
+    operator<(const RawDocumentMetaData &rhs) const
+    {
+        return _gid < rhs._gid;
+    }
+
+    bool
+    operator==(const RawDocumentMetaData &rhs) const
+    {
+        return _gid == rhs._gid;
+    }
+
+    const GlobalId &
+    getGid(void) const
+    {
+        return _gid;
+    }
+
+    GlobalId &
+    getGid(void)
+    {
+        return _gid;
+    }
+
+    void
+    setGid(const GlobalId &rhs)
+    {
+        _gid = rhs;
+    }
+
+    uint8_t
+    getBucketUsedBits(void) const
+    {
+        return _bucketUsedBits;
+    }
+
+    BucketId
+    getBucketId(void) const
+    {
+        BucketId ret(_gid.convertToBucketId());
+        ret.setUsedBits(_bucketUsedBits);
+        return ret;
+    }
+
+    void
+    setBucketUsedBits(uint8_t bucketUsedBits)
+    {
+        assert(bucketUsedBits >= BucketId::minNumBits() &&
+               bucketUsedBits <= BucketId::maxNumBits());
+        _bucketUsedBits = bucketUsedBits;
+    }
+
+    void
+    setBucketId(const BucketId &bucketId)
+    {
+        assert(bucketId.valid());
+        uint8_t bucketUsedBits = bucketId.getUsedBits();
+        BucketId verId(_gid.convertToBucketId());
+        verId.setUsedBits(bucketUsedBits);
+        assert(bucketId.getRawId() == verId.getRawId() ||
+               bucketId.getRawId() == verId.getId());
+        _bucketUsedBits = bucketUsedBits;
+    }
+
+    Timestamp
+    getTimestamp(void) const
+    {
+        return _timestamp;
+    }
+
+    void
+    setTimestamp(const Timestamp &timestamp)
+    {
+        _timestamp = timestamp;
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp
new file mode 100644
index 00000000000..7e4f3067d4e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp
@@ -0,0 +1,158 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.documentmetastore.search_context");
+
+#include "search_context.h"
+#include <vespa/searchlib/attribute/attributeiterators.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+using document::GlobalId;
+using search::AttributeIteratorBase;
+using search::AttributeVector;
+using search::QueryTermSimple;
+using search::fef::TermFieldMatchData;
+using search::queryeval::SearchIterator;
+
+namespace proton {
+namespace documentmetastore {
+
+namespace {
+
+class GidAllSearchIterator : public AttributeIteratorBase
+{
+private:
+    virtual void
+    doSeek(uint32_t docId)
+    {
+        if (_store.validLidFast(docId)) {
+            setDocId(docId);
+        }
+    }
+
+    virtual void
+    doUnpack(uint32_t docId)
+    {
+        _matchData->reset(docId);
+    }
+
+protected:
+    const DocumentMetaStore & _store;
+public:
+    GidAllSearchIterator(TermFieldMatchData *matchData,
+                         const DocumentMetaStore &store)
+        : AttributeIteratorBase(matchData),
+          _store(store)
+    {
+    }
+};
+
+class GidStrictAllSearchIterator : public GidAllSearchIterator
+{
+private:
+    uint32_t _numDocs;
+
+    virtual void
+    doSeek(uint32_t docId)
+    {
+        if (_store.validLidFast(docId)) {
+            setDocId(docId);
+        } else {
+            for (docId++; docId < _numDocs && !_store.validLidFast(docId); docId++);
+            if (docId < _numDocs) {
+                setDocId(docId);
+            } else {
+                setAtEnd();
+            }
+        }
+    }
+
+public:
+    GidStrictAllSearchIterator(TermFieldMatchData *matchData,
+                               const DocumentMetaStore &store)
+        : GidAllSearchIterator(matchData, store),
+          _numDocs(store.getNumDocs())
+    {
+    }
+};
+
+class GidSearchIterator : public GidAllSearchIterator
+{
+private:
+    const GlobalId & _gid;
+
+    virtual void
+    doSeek(uint32_t docId)
+    {
+        AttributeVector::DocId lid = 0;
+        if (_store.getLid(_gid, lid) && (lid >= docId)) {
+            setDocId(lid);
+        } else {
+            setAtEnd();
+        }
+    }
+public:
+    GidSearchIterator(TermFieldMatchData *matchData,
+                      const DocumentMetaStore &store,
+                      const GlobalId &gid)
+        : GidAllSearchIterator(matchData, store),
+          _gid(gid)
+    {
+    }
+};
+
+}
+
+bool
+SearchContext::onCmp(DocId docId, int32_t &weight) const
+{
+    (void) docId;
+    (void) weight;
+    throw vespalib::IllegalStateException(
+            "The function is not implemented for documentmetastore::SearchContext");
+    return false;
+}
+
+bool
+SearchContext::onCmp(DocId docId) const
+{
+    (void) docId;
+    throw vespalib::IllegalStateException(
+            "The function is not implemented for documentmetastore::SearchContext");
+    return false;
+}
+
+unsigned int
+SearchContext::approximateHits() const
+{
+    return _isWord ? 1 : search::AttributeVector::SearchContext::approximateHits();
+}
+
+SearchIterator::UP
+SearchContext::createIterator(TermFieldMatchData *matchData,
+                              bool strict)
+{
+    return _isWord
+        ? SearchIterator::UP(new GidSearchIterator(matchData, getStore(), _gid))
+        : strict
+            ?  SearchIterator::UP(new GidStrictAllSearchIterator(matchData,
+                                      getStore()))
+            :  SearchIterator::UP(new GidAllSearchIterator(matchData, getStore()));
+}
+
+const DocumentMetaStore &
+SearchContext::getStore() const
+{
+    return static_cast<const DocumentMetaStore &>(attribute());
+}
+
+SearchContext::SearchContext(QueryTermSimple::UP qTerm,
+                             const DocumentMetaStore &toBeSearched)
+    : search::AttributeVector::SearchContext(toBeSearched),
+      _isWord(qTerm->isWord())
+{
+}
+
+}
+}
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h
new file mode 100644
index 00000000000..6a2e6f2045e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/base/globalid.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include "documentmetastore.h"
+
+namespace proton {
+namespace documentmetastore {
+
+/**
+ * Search context used to search the document meta store for all valid documents.
+ */
+class SearchContext : public search::AttributeVector::SearchContext
+{
+private:
+    typedef search::AttributeVector::DocId DocId;
+
+    bool _isWord;
+    document::GlobalId _gid;
+
+    virtual unsigned int approximateHits() const;
+    virtual bool onCmp(DocId docId, int32_t &weight) const;
+    virtual bool onCmp(DocId docId) const;
+
+    virtual search::queryeval::SearchIterator::UP
+    createIterator(search::fef::TermFieldMatchData *matchData,
+                   bool strict);
+
+    const DocumentMetaStore &getStore() const;
+
+public:
+    SearchContext(search::QueryTermSimple::UP qTerm,
+                  const DocumentMetaStore &toBeSearched);
+};
+
+} // namespace documentmetastore
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/feedoperation/CMakeLists.txt
new file mode 100644
index 00000000000..4b0f7c1cd6e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/CMakeLists.txt
@@ -0,0 +1,23 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_feedoperation STATIC
+    SOURCES
+    compact_lid_space_operation.cpp
+    createbucketoperation.cpp
+    deletebucketoperation.cpp
+    documentoperation.cpp
+    feedoperation.cpp
+    joinbucketsoperation.cpp
+    lidvectorcontext.cpp
+    moveoperation.cpp
+    newconfigoperation.cpp
+    noopoperation.cpp
+    pruneremoveddocumentsoperation.cpp
+    putoperation.cpp
+    removedocumentsoperation.cpp
+    removeoperation.cpp
+    splitbucketoperation.cpp
+    spoolerreplayoperation.cpp
+    updateoperation.cpp
+    wipehistoryoperation.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/OWNERS b/searchcore/src/vespa/searchcore/proton/feedoperation/OWNERS
new file mode 100644
index 00000000000..7ae1acb1be9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/OWNERS
@@ -0,0 +1 @@
+geirst
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp
new file mode 100644
index 00000000000..047813d8a05
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.compact_lid_space_operation");
+
+#include "compact_lid_space_operation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+namespace proton {
+
+CompactLidSpaceOperation::CompactLidSpaceOperation()
+    : FeedOperation(FeedOperation::COMPACT_LID_SPACE),
+      _subDbId(0),
+      _lidLimit(0)
+{
+}
+
+CompactLidSpaceOperation::CompactLidSpaceOperation(uint32_t subDbId, uint32_t lidLimit)
+    : FeedOperation(FeedOperation::COMPACT_LID_SPACE),
+      _subDbId(subDbId),
+      _lidLimit(lidLimit)
+{
+}
+
+void
+CompactLidSpaceOperation::serialize(vespalib::nbostream& os) const
+{
+    os << _subDbId;
+    os << _lidLimit;
+}
+
+void
+CompactLidSpaceOperation::deserialize(vespalib::nbostream& is,
+                                      const document::DocumentTypeRepo&)
+{
+    is >> _subDbId;
+    is >> _lidLimit;
+}
+
+vespalib::string
+CompactLidSpaceOperation::toString() const
+{
+    return vespalib::make_string("CompactLidSpace(subDbId=%u, lidLimit=%u, serialNum=%" PRIu64 ")",
+            _subDbId, _lidLimit, getSerialNum());
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h
new file mode 100644
index 00000000000..4a51b5221b1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+
+namespace proton {
+
+class CompactLidSpaceOperation : public FeedOperation
+{
+private:
+    uint32_t _subDbId;
+    uint32_t _lidLimit;
+
+public:
+    CompactLidSpaceOperation();
+    CompactLidSpaceOperation(uint32_t subDbId, uint32_t lidLimit);
+    virtual ~CompactLidSpaceOperation() {}
+
+    uint32_t getSubDbId() const { return _subDbId; }
+    uint32_t getLidLimit() const { return _lidLimit; }
+
+    // Implements FeedOperation
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.cpp
new file mode 100644
index 00000000000..d761677542e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.cpp
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.createbucketoperation");
+
+#include "createbucketoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::BucketId;
+using document::DocumentTypeRepo;
+using vespalib::make_string;
+
+namespace proton {
+
+CreateBucketOperation::CreateBucketOperation()
+    : FeedOperation(FeedOperation::CREATE_BUCKET),
+      _bucketId()
+{
+}
+
+
+CreateBucketOperation::CreateBucketOperation(const BucketId &bucketId)
+    : FeedOperation(FeedOperation::CREATE_BUCKET),
+      _bucketId(bucketId)
+{
+}
+
+
+void
+CreateBucketOperation::serialize(vespalib::nbostream &os) const
+{
+    assert(_bucketId.valid());
+    os << _bucketId;
+}
+
+
+void
+CreateBucketOperation::deserialize(vespalib::nbostream &is,
+                                   const DocumentTypeRepo &)
+{
+    is >> _bucketId;
+}
+
+vespalib::string CreateBucketOperation::toString() const {
+    return make_string("CreateBucket(%s, serialNum=%" PRIu64 ")",
+                       _bucketId.toString().c_str(), getSerialNum());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.h
new file mode 100644
index 00000000000..bf03eb53eb8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+#include <vespa/document/bucket/bucketid.h>
+
+namespace proton {
+
+class CreateBucketOperation : public FeedOperation
+{
+    document::BucketId   _bucketId;
+
+public:
+    CreateBucketOperation();
+    CreateBucketOperation(const document::BucketId &bucketId);
+    virtual ~CreateBucketOperation() {}
+    const document::BucketId &getBucketId() const { return _bucketId; }
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.cpp
new file mode 100644
index 00000000000..964f2e44702
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.cpp
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.deletebucketoperation");
+
+#include "deletebucketoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::BucketId;
+using document::DocumentTypeRepo;
+using vespalib::make_string;
+
+namespace proton {
+
+DeleteBucketOperation::DeleteBucketOperation()
+    : RemoveDocumentsOperation(FeedOperation::DELETE_BUCKET),
+      _bucketId()
+{
+}
+
+
+DeleteBucketOperation::DeleteBucketOperation(const BucketId &bucketId)
+    : RemoveDocumentsOperation(FeedOperation::DELETE_BUCKET),
+      _bucketId(bucketId)
+{
+}
+
+
+void
+DeleteBucketOperation::serialize(vespalib::nbostream &os) const
+{
+    assert(_bucketId.valid());
+    os << _bucketId;
+    serializeLidsToRemove(os);
+}
+
+
+void
+DeleteBucketOperation::deserialize(vespalib::nbostream &is,
+                                   const DocumentTypeRepo &)
+{
+    is >> _bucketId;
+    deserializeLidsToRemove(is);
+}
+
+vespalib::string DeleteBucketOperation::toString() const {
+    return make_string("DeleteBucket(%s, serialNum=%" PRIu64 ")",
+                       _bucketId.toString().c_str(), getSerialNum());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.h
new file mode 100644
index 00000000000..1780634ca54
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "removedocumentsoperation.h"
+#include <vespa/document/bucket/bucketid.h>
+
+namespace proton {
+
+class DeleteBucketOperation : public RemoveDocumentsOperation
+{
+    document::BucketId   _bucketId;
+
+public:
+    DeleteBucketOperation();
+    DeleteBucketOperation(const document::BucketId &bucketId);
+    virtual ~DeleteBucketOperation() {}
+    const document::BucketId &getBucketId() const { return _bucketId; }
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
new file mode 100644
index 00000000000..4d3196e1ea1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.documentoperation");
+
+#include "documentoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::BucketId;
+using document::DocumentId;
+using document::DocumentTypeRepo;
+using document::GlobalId;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+
+namespace proton {
+
+DocumentOperation::DocumentOperation(Type type)
+    : FeedOperation(type),
+      _bucketId(),
+      _timestamp(),
+      _dbdId(),
+      _prevDbdId(),
+      _prevMarkedAsRemoved(false),
+      _prevTimestamp()
+{
+}
+
+
+DocumentOperation::DocumentOperation(Type type,
+                                     const BucketId &bucketId,
+                                     const Timestamp &timestamp)
+    : FeedOperation(type),
+      _bucketId(bucketId),
+      _timestamp(timestamp),
+      _dbdId(),
+      _prevDbdId(),
+      _prevMarkedAsRemoved(false),
+      _prevTimestamp()
+{
+}
+
+
+DocumentOperation::DocumentOperation(Type type,
+                                     const document::BucketId &bucketId,
+                                     const storage::spi::Timestamp &timestamp,
+                                     SerialNum serialNum,
+                                     DbDocumentId dbdId,
+                                     DbDocumentId prevDbdId)
+    : FeedOperation(type),
+      _bucketId(bucketId),
+      _timestamp(timestamp),
+      _dbdId(dbdId),
+      _prevDbdId(prevDbdId),
+      _prevMarkedAsRemoved(false),
+      _prevTimestamp()
+{
+    setSerialNum(serialNum);
+}
+
+
+void
+DocumentOperation::assertValidBucketId(const document::DocumentId &docId) const
+{
+    assert(_bucketId.valid());
+    uint8_t bucketUsedBits = _bucketId.getUsedBits();
+    const GlobalId &gid = docId.getGlobalId();
+    BucketId verId(gid.convertToBucketId());
+    verId.setUsedBits(bucketUsedBits);
+    assert(_bucketId.getRawId() == verId.getRawId() ||
+           _bucketId.getRawId() == verId.getId());
+}
+
+vespalib::string DocumentOperation::docArgsToString() const {
+    return make_string("%s, timestamp=%" PRIu64 ", dbdId=(%s), prevDbdId=(%s), "
+                       "prevMarkedAsRemoved=%s, prevTimestamp=%" PRIu64 ", serialNum=%" PRIu64,
+                       _bucketId.toString().c_str(), _timestamp.getValue(),
+                       _dbdId.toString().c_str(), _prevDbdId.toString().c_str(),
+                       (_prevMarkedAsRemoved ? "true" : "false"),
+                       _prevTimestamp.getValue(), getSerialNum());
+}
+
+void
+DocumentOperation::serialize(vespalib::nbostream &os) const
+{
+    os << _bucketId;
+    os << _timestamp;
+    os << _dbdId;
+    os << _prevDbdId;
+    os << _prevMarkedAsRemoved;
+    os << _prevTimestamp;
+}
+
+
+void
+DocumentOperation::deserialize(vespalib::nbostream &is,
+                               const DocumentTypeRepo &)
+{
+    is >> _bucketId;
+    is >> _timestamp;
+    is >> _dbdId;
+    is >> _prevDbdId;
+    is >> _prevMarkedAsRemoved;
+    is >> _prevTimestamp;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
new file mode 100644
index 00000000000..b9447f0ebfa
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
@@ -0,0 +1,202 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+#include <vespa/searchcore/proton/common/dbdocumentid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <persistence/spi/types.h>
+#include <vespa/searchlib/query/base.h>
+
+namespace proton {
+
+class DocumentOperation : public FeedOperation
+{
+protected:
+    document::BucketId      _bucketId;
+    storage::spi::Timestamp _timestamp;
+    DbDocumentId            _dbdId;
+    DbDocumentId            _prevDbdId;
+    bool                    _prevMarkedAsRemoved;
+    storage::spi::Timestamp _prevTimestamp;
+
+    DocumentOperation(Type type);
+
+    DocumentOperation(Type type,
+                      const document::BucketId &bucketId,
+                      const storage::spi::Timestamp &timestamp);
+
+    DocumentOperation(Type type,
+                      const document::BucketId &bucketId,
+                      const storage::spi::Timestamp &timestamp,
+                      SerialNum serialNum,
+                      DbDocumentId dbdId,
+                      DbDocumentId prevDbdId);
+
+    void
+    assertValidBucketId(const document::DocumentId &docId) const;
+    vespalib::string docArgsToString() const;
+
+public:
+    virtual
+    ~DocumentOperation(void)
+    {
+    }
+
+    const
+    document::BucketId &
+    getBucketId(void) const
+    {
+        return _bucketId;
+    }
+
+    storage::spi::Timestamp
+    getTimestamp(void) const
+    {
+        return _timestamp;
+    }
+
+    search::DocumentIdT
+    getLid() const
+    {
+        return _dbdId.getLid();
+    }
+
+    search::DocumentIdT
+    getPrevLid() const
+    {
+        return _prevDbdId.getLid();
+    }
+
+    uint32_t
+    getSubDbId(void) const
+    {
+        return _dbdId.getSubDbId();
+    }
+
+    uint32_t
+    getPrevSubDbId(void) const
+    {
+        return _prevDbdId.getSubDbId();
+    }
+
+    bool
+    getValidDbdId(void) const
+    {
+        return _dbdId.valid();
+    }
+
+    bool
+    getValidDbdId(uint32_t subDbId) const
+    {
+        return _dbdId.valid() && _dbdId.getSubDbId() == subDbId;
+    }
+
+    bool
+    getValidPrevDbdId(void) const
+    {
+        return _prevDbdId.valid();
+    }
+
+    bool
+    getValidPrevDbdId(uint32_t subDbId) const
+    {
+        return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId;
+    }
+
+    bool
+    changedDbdId(void) const
+    {
+        return _dbdId != _prevDbdId;
+    }
+    bool
+    getPrevMarkedAsRemoved(void) const
+    {
+        return _prevMarkedAsRemoved;
+    }
+
+    void
+    setPrevMarkedAsRemoved(bool prevMarkedAsRemoved)
+    {
+        _prevMarkedAsRemoved = prevMarkedAsRemoved;
+    }
+
+    DbDocumentId
+    getDbDocumentId(void) const
+    {
+        return _dbdId;
+    }
+
+    DbDocumentId
+    getPrevDbDocumentId(void) const
+    {
+        return _prevDbdId;
+    }
+
+    void
+    setDbDocumentId(DbDocumentId dbdId)
+    {
+        _dbdId = dbdId;
+    }
+
+    void
+    setPrevDbDocumentId(DbDocumentId prevDbdId)
+    {
+        _prevDbdId = prevDbdId;
+    }
+
+    search::DocumentIdT
+    getNewOrPrevLid(uint32_t subDbId) const
+    {
+        if (getValidDbdId() && getSubDbId() == subDbId)
+            return getLid();
+        if (getValidPrevDbdId() && getPrevSubDbId() == subDbId)
+            return getPrevLid();
+        return 0;
+    }
+
+    bool
+    getValidNewOrPrevDbdId(void) const
+    {
+        return getValidDbdId() || getValidPrevDbdId();
+    }
+
+    bool
+    notMovingLidInSameSubDb(void) const
+    {
+        return !getValidDbdId() ||
+            !getValidPrevDbdId() ||
+            getSubDbId() != getPrevSubDbId() ||
+            getLid() == getPrevLid();
+    }
+
+    bool
+    movingLidIfInSameSubDb(void) const
+    {
+        return !getValidDbdId() ||
+            !getValidPrevDbdId() ||
+            getSubDbId() != getPrevSubDbId() ||
+            getLid() != getPrevLid();
+    }
+
+    storage::spi::Timestamp
+    getPrevTimestamp(void) const
+    {
+        return _prevTimestamp;
+    }
+
+    void
+    setPrevTimestamp(storage::spi::Timestamp prevTimestamp)
+    {
+        _prevTimestamp = prevTimestamp;
+    }
+
+    virtual void
+    serialize(vespalib::nbostream &os) const;
+
+    virtual void
+    deserialize(vespalib::nbostream &is,
+                const document::DocumentTypeRepo &repo);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.cpp
new file mode 100644
index 00000000000..789d3ec97ee
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.cpp
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.feedoperation");
+
+#include "feedoperation.h"
+
+namespace proton {
+
+FeedOperation::FeedOperation(Type type)
+    : _type(type),
+      _serialNum(0)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
new file mode 100644
index 00000000000..c477ed0e5e1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+namespace proton {
+
+class FeedOperation
+{
+public:
+    typedef search::SerialNum SerialNum;
+    typedef std::shared_ptr<FeedOperation> SP;
+    typedef std::unique_ptr<FeedOperation> UP;
+
+    enum Type {
+        PUT                     = 1,
+        REMOVE                  = 2,
+        REMOVE_BATCH            = 3,
+        UPDATE                  = 4,
+        NOOP                    = 5,
+        NEW_CONFIG              = 6,
+        WIPE_HISTORY            = 7,
+        DELETE_BUCKET           = 9,
+        SPLIT_BUCKET            = 10,
+        JOIN_BUCKETS            = 11,
+        PRUNE_REMOVED_DOCUMENTS = 12,
+        SPOOLER_REPLAY_START    = 13,
+        SPOOLER_REPLAY_COMPLETE = 14,
+        MOVE                    = 15,
+        CREATE_BUCKET           = 16,
+        COMPACT_LID_SPACE       = 17
+    };
+
+private:
+    Type _type;
+    SerialNum _serialNum;
+
+public:
+    FeedOperation(Type type);
+    virtual ~FeedOperation() {}
+    Type getType() const { return _type; }
+    void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; }
+    SerialNum getSerialNum() const { return _serialNum; }
+    virtual void serialize(vespalib::nbostream &os) const = 0;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo) = 0;
+    virtual vespalib::string toString() const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.cpp
new file mode 100644
index 00000000000..afd96720baf
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.cpp
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.joinbucketsoperation");
+
+#include "joinbucketsoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::BucketId;
+using document::DocumentTypeRepo;
+using vespalib::make_string;
+
+namespace proton {
+
+JoinBucketsOperation::JoinBucketsOperation()
+    : FeedOperation(FeedOperation::JOIN_BUCKETS),
+      _source1(),
+      _source2(),
+      _target()
+{
+}
+
+
+JoinBucketsOperation::JoinBucketsOperation(const document::BucketId &source1,
+                                           const document::BucketId &source2,
+                                           const document::BucketId &target)
+    : FeedOperation(FeedOperation::JOIN_BUCKETS),
+      _source1(source1),
+      _source2(source2),
+      _target(target)
+{
+}
+
+
+void
+JoinBucketsOperation::serialize(vespalib::nbostream &os) const
+{
+    {
+        assert(_source1.valid() || _source2.valid());
+        assert(_target.valid());
+        if (_source1.valid()) {
+            assert(_source1.getUsedBits() > _target.getUsedBits());
+            assert(_target.contains(_source1));
+        }
+        if (_source2.valid()) {
+            assert(_source2.getUsedBits() > _target.getUsedBits());
+            assert(_target.contains(_source2));
+        }
+    }
+    os << _source1;
+    os << _source2;
+    os << _target;
+}
+
+
+void
+JoinBucketsOperation::deserialize(vespalib::nbostream &is,
+                                  const DocumentTypeRepo &)
+{
+    is >> _source1;
+    is >> _source2;
+    is >> _target;
+}
+
+vespalib::string JoinBucketsOperation::toString() const {
+    return make_string("JoinBuckets(source1=%s, source2=%s, target=%s, "
+                       "serialNum=%" PRIu64 ")",
+                       _source1.toString().c_str(),
+                       _source2.toString().c_str(),
+                       _target.toString().c_str(), getSerialNum());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.h
new file mode 100644
index 00000000000..7de28be9ecc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.h
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+#include <vespa/document/bucket/bucketid.h>
+
+namespace proton {
+
+class JoinBucketsOperation : public FeedOperation
+{
+private:
+    document::BucketId _source1;
+    document::BucketId _source2;
+    document::BucketId _target;
+public:
+    JoinBucketsOperation();
+    JoinBucketsOperation(const document::BucketId &source1,
+                         const document::BucketId &source2,
+                         const document::BucketId &target);
+    virtual ~JoinBucketsOperation() {}
+    const document::BucketId &getSource1() const { return _source1; }
+    const document::BucketId &getSource2() const { return _source2; }
+    const document::BucketId &getTarget() const { return _target; }
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp
new file mode 100644
index 00000000000..387b41bc010
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.lidvectorcontext");
+
+#include "lidvectorcontext.h"
+#include <vespa/searchlib/common/bitvector.h>
+
+using search::BitVector;
+
+namespace proton {
+
+LidVectorContext::LidVectorContext(size_t docIdLimit)
+    : _result(),
+      _docIdLimit(docIdLimit)
+{
+}
+
+
+LidVectorContext::LidVectorContext()
+    : _result(),
+      _docIdLimit(0)
+{
+}
+
+
+LidVectorContext::LidVectorContext(size_t docIdLimit,
+                                   const LidVector &lids)
+    : _result(lids),
+      _docIdLimit(docIdLimit)
+{
+}
+
+
+void
+LidVectorContext::addLid(const search::DocumentIdT lid)
+{
+    _result.push_back(lid);
+}
+
+
+void
+LidVectorContext::serialize(vespalib::nbostream &os) const
+{
+    LOG(debug, "serialize: _result.size() = %ld, _docIdLimit = %ld",
+        _result.size(), _docIdLimit);
+    os << _docIdLimit;
+    // Use of bitvector when > 1/32 of docs
+    if (_result.size() > (_docIdLimit / 32)) {
+        os << static_cast<int32_t>(BITVECTOR);
+        BitVector::UP bitVector = BitVector::create(_docIdLimit);
+        for (LidVector::const_iterator it(_result.begin()), mt(_result.end()); it != mt; it++) {
+            bitVector->setBit(*it);
+        }
+        os << *bitVector;
+    } else {
+        os << static_cast<int32_t>(ARRAY);
+        os << _result;
+    }
+}
+
+
+void
+LidVectorContext::deserialize(vespalib::nbostream &is)
+{
+    int32_t format;
+    is >> _docIdLimit;
+    is >> format;
+    LOG(debug, "deserialize: format = %d", format);
+    // Use of bitvector when > 1/32 of docs
+    if (format == BITVECTOR) {
+        BitVector::UP bitVector = BitVector::create(_docIdLimit);
+        is >> *bitVector;
+        uint32_t sz(bitVector->size());
+        assert(sz == _docIdLimit);
+        LOG(spam, "deserialize: reading bitvector of size %u", sz);
+        for (search::DocumentIdT lid(bitVector->getFirstTrueBit());
+             lid < sz;
+             lid = bitVector->getNextTrueBit(lid + 1)) {
+
+            _result.push_back(lid);
+        }
+    } else if (format == ARRAY) {
+        is >> _result;
+    }
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h
new file mode 100644
index 00000000000..6e2f9256b84
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/query/base.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/linkedptr.h>
+#include <vector>
+
+namespace proton {
+
+class LidVectorContext
+{
+public:
+    typedef std::vector<search::DocumentIdT> LidVector;
+private:
+    LidVector _result;
+    size_t    _docIdLimit;
+    enum { ARRAY = 0, BITVECTOR = 1 };
+public:
+    typedef vespalib::LinkedPtr<LidVectorContext> LP;
+    LidVectorContext();
+    LidVectorContext(size_t docIdLimit);
+    LidVectorContext(size_t docIdLimit, const LidVector &lids);
+    void addLid(const search::DocumentIdT lid);
+    void serialize(vespalib::nbostream &os) const;
+    void deserialize(vespalib::nbostream &is);
+    const LidVector &getLidVector() const { return _result; }
+    void clearLidVector() { _result.clear(); }
+    size_t getDocIdLimit() const { return _docIdLimit; }
+    size_t getNumLids() const { return _result.size(); }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp
new file mode 100644
index 00000000000..ca8e21cbf86
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.moveoperation");
+
+#include "moveoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::BucketId;
+using document::Document;
+using document::DocumentTypeRepo;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+
+namespace proton {
+
+MoveOperation::MoveOperation()
+    : DocumentOperation(FeedOperation::MOVE),
+      _doc()
+{
+}
+
+
+MoveOperation::MoveOperation(const BucketId &bucketId,
+                             const Timestamp &timestamp,
+                             const Document::SP &doc,
+                             DbDocumentId sourceDbdId,
+                             uint32_t targetSubDbId)
+    : DocumentOperation(FeedOperation::MOVE, bucketId, timestamp),
+      _doc(doc)
+{
+    setPrevDbDocumentId(sourceDbdId);
+    setDbDocumentId(DbDocumentId(targetSubDbId, 0u));
+}
+
+
+void
+MoveOperation::serialize(vespalib::nbostream &os) const
+{
+    assertValidBucketId(_doc->getId());
+    assert(movingLidIfInSameSubDb());
+    DocumentOperation::serialize(os);
+    _doc->serialize(os);
+}
+
+
+void
+MoveOperation::deserialize(vespalib::nbostream &is,
+                           const DocumentTypeRepo &repo)
+{
+    DocumentOperation::deserialize(is, repo);
+    _doc.reset(new Document(repo, is));
+}
+
+vespalib::string MoveOperation::toString() const {
+    return make_string("Move(%s, %s)",
+                       _doc.get() ?
+                       _doc->getId().getScheme().toString().c_str() : "NULL",
+                       docArgsToString().c_str());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.h
new file mode 100644
index 00000000000..0789b2afc1d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentoperation.h"
+
+namespace proton {
+
+class MoveOperation : public DocumentOperation
+{
+private:
+    document::Document::SP _doc;
+public:
+    typedef std::unique_ptr<MoveOperation> UP;
+
+    MoveOperation();
+    MoveOperation(const document::BucketId &bucketId,
+                  const storage::spi::Timestamp &timestamp,
+                  const document::Document::SP &doc,
+                  DbDocumentId sourceDbdId,
+                  uint32_t targetSubDbId);
+    virtual ~MoveOperation() {}
+    const document::Document::SP &getDocument() const { return _doc; }
+    DbDocumentId getSourceDbdId() const { return getPrevDbDocumentId(); }
+    DbDocumentId getTargetDbdId() const { return getDbDocumentId(); }
+    void setTargetLid(search::DocumentIdT lid) {
+        setDbDocumentId(DbDocumentId(getSubDbId(), lid));
+    }
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.cpp
new file mode 100644
index 00000000000..2044fdea43f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.newconfigoperation");
+
+#include "newconfigoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::DocumentTypeRepo;
+using vespalib::make_string;
+
+namespace proton {
+
+NewConfigOperation::NewConfigOperation(SerialNum serialNum,
+                                       IStreamHandler &streamHandler)
+    : FeedOperation(FeedOperation::NEW_CONFIG),
+      _streamHandler(streamHandler)
+{
+    setSerialNum(serialNum);
+}
+
+
+void
+NewConfigOperation::serialize(vespalib::nbostream &os) const
+{
+    _streamHandler.serializeConfig(getSerialNum(), os);
+}
+
+
+void
+NewConfigOperation::deserialize(vespalib::nbostream &is,
+                                const DocumentTypeRepo &)
+{
+    _streamHandler.deserializeConfig(getSerialNum(), is);
+}
+
+vespalib::string NewConfigOperation::toString() const {
+    return make_string("NewConfig(serialNum=%" PRIu64 ")", getSerialNum());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h
new file mode 100644
index 00000000000..cc6a2d18eb7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+#include <vespa/vespalib/objects/nbostream.h>
+
+namespace proton {
+
+class NewConfigOperation : public FeedOperation
+{
+public:
+    struct IStreamHandler {
+        virtual ~IStreamHandler() {}
+        virtual void serializeConfig(SerialNum serialNum,
+                                     vespalib::nbostream &os) = 0;
+        virtual void deserializeConfig(SerialNum serialNum,
+                                       vespalib::nbostream &is) = 0;
+    };
+private:
+    IStreamHandler &_streamHandler;
+public:
+    NewConfigOperation(SerialNum serialNum,
+                       IStreamHandler &streamHandler);
+    virtual ~NewConfigOperation() {}
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.cpp
new file mode 100644
index 00000000000..0ea7977f2bc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.cpp
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.noopoperation");
+
+#include "noopoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using vespalib::make_string;
+
+namespace proton {
+
+NoopOperation::NoopOperation(SerialNum serialNum)
+    : FeedOperation(FeedOperation::NOOP)
+{
+    setSerialNum(serialNum);
+}
+
+vespalib::string NoopOperation::toString() const {
+    return make_string("Noop(serialNum=%" PRIu64 ")", getSerialNum());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.h
new file mode 100644
index 00000000000..47e36b4c39d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.h
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+
+namespace proton {
+
+struct NoopOperation : FeedOperation {
+    NoopOperation() : FeedOperation(FeedOperation::NOOP) {}
+    NoopOperation(SerialNum serialNum);
+    virtual ~NoopOperation() {}
+
+    virtual void serialize(vespalib::nbostream &) const {}
+    virtual void deserialize(vespalib::nbostream &,
+                             const document::DocumentTypeRepo &) {}
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/operations.h b/searchcore/src/vespa/searchcore/proton/feedoperation/operations.h
new file mode 100644
index 00000000000..c7bd2eacd13
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/operations.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "compact_lid_space_operation.h"
+#include "createbucketoperation.h"
+#include "deletebucketoperation.h"
+#include "documentoperation.h"
+#include "feedoperation.h"
+#include "joinbucketsoperation.h"
+#include "moveoperation.h"
+#include "newconfigoperation.h"
+#include "noopoperation.h"
+#include "pruneremoveddocumentsoperation.h"
+#include "putoperation.h"
+#include "removedocumentsoperation.h"
+#include "removeoperation.h"
+#include "splitbucketoperation.h"
+#include "spoolerreplayoperation.h"
+#include "updateoperation.h"
+#include "wipehistoryoperation.h"
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp
new file mode 100644
index 00000000000..9da621b5c84
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.pruneremoveddocumentsoperation");
+
+#include "pruneremoveddocumentsoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::DocumentTypeRepo;
+using search::DocumentIdT;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+
+namespace proton {
+
+PruneRemovedDocumentsOperation::PruneRemovedDocumentsOperation()
+    : RemoveDocumentsOperation(FeedOperation::PRUNE_REMOVED_DOCUMENTS),
+      _subDbId(0)
+{
+}
+
+
+PruneRemovedDocumentsOperation::
+PruneRemovedDocumentsOperation(DocumentIdT docIdLimit,
+                               uint32_t subDbId)
+    : RemoveDocumentsOperation(FeedOperation::PRUNE_REMOVED_DOCUMENTS),
+      _subDbId(subDbId)
+{
+    LidVectorContext::LP lidsToRemove(new LidVectorContext(docIdLimit));
+    setLidsToRemove(lidsToRemove);
+}
+
+
+void
+PruneRemovedDocumentsOperation::serialize(vespalib::nbostream &os) const
+{
+    LOG(debug, "serialize(): %s", toString().c_str());
+    os << _subDbId;
+    assert(_lidsToRemoveMap.size() == 1);
+    assert(_lidsToRemoveMap.begin()->first == _subDbId);
+    serializeLidsToRemove(os);
+}
+
+
+void
+PruneRemovedDocumentsOperation::deserialize(vespalib::nbostream &is,
+                                            const DocumentTypeRepo &)
+{
+    is >> _subDbId;
+    deserializeLidsToRemove(is);
+}
+
+vespalib::string PruneRemovedDocumentsOperation::toString() const {
+    LidVectorContext::LP lids = getLidsToRemove();
+    return make_string("PruneRemovedDocuments(limitLid=%zu, subDbId=%d, "
+                       "serialNum=%" PRIu64 ")",
+                       lids.get() ? lids->getDocIdLimit() : 0,
+                       _subDbId, getSerialNum());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h
new file mode 100644
index 00000000000..35c413b2491
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "removedocumentsoperation.h"
+#include <persistence/spi/types.h>
+#include <vespa/searchlib/query/base.h>
+
+namespace proton {
+
+class PruneRemovedDocumentsOperation : public RemoveDocumentsOperation
+{
+private:
+    uint32_t _subDbId;
+public:
+    typedef std::unique_ptr<PruneRemovedDocumentsOperation> UP;
+
+    PruneRemovedDocumentsOperation();
+
+    PruneRemovedDocumentsOperation(search::DocumentIdT docIdLimit,
+                                   uint32_t subDbId);
+
+    virtual
+    ~PruneRemovedDocumentsOperation()
+    {
+    }
+
+    uint32_t getSubDbId() const { return _subDbId; }
+
+    void setLidsToRemove(const LidVectorContext::LP &lidsToRemove)
+    {
+        RemoveDocumentsOperation::setLidsToRemove(_subDbId, lidsToRemove);
+    }
+
+    const LidVectorContext::LP
+    getLidsToRemove() const
+    {
+        return RemoveDocumentsOperation::getLidsToRemove(_subDbId);
+    }
+
+    virtual void
+    serialize(vespalib::nbostream &os) const;
+
+    virtual void
+    deserialize(vespalib::nbostream &is,
+                const document::DocumentTypeRepo &repo);
+
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp
new file mode 100644
index 00000000000..7f5f52bfec4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.putoperation");
+
+#include "putoperation.h"
+
+using document::BucketId;
+using document::Document;
+using document::DocumentTypeRepo;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+
+namespace proton {
+
+PutOperation::PutOperation()
+    : DocumentOperation(FeedOperation::PUT),
+      _doc()
+{
+}
+
+
+PutOperation::PutOperation(const BucketId &bucketId,
+                           const Timestamp &timestamp,
+                           const Document::SP &doc)
+    : DocumentOperation(FeedOperation::PUT,
+                        bucketId,
+                        timestamp),
+      _doc(doc)
+{
+}
+
+
+PutOperation::PutOperation(const document::BucketId &bucketId,
+                           const storage::spi::Timestamp &timestamp,
+                           const document::Document::SP &doc,
+                           SerialNum serialNum,
+                           DbDocumentId dbdId,
+                           DbDocumentId prevDbdId)
+    : DocumentOperation(FeedOperation::PUT,
+                        bucketId,
+                        timestamp,
+                        serialNum,
+                        dbdId,
+                        prevDbdId),
+      _doc(doc)
+{
+}
+
+
+void
+PutOperation::serialize(vespalib::nbostream &os) const
+{
+    assertValidBucketId(_doc->getId());
+    DocumentOperation::serialize(os);
+    _doc->serialize(os);
+}
+
+
+void
+PutOperation::deserialize(vespalib::nbostream &is,
+                          const DocumentTypeRepo &repo)
+{
+    DocumentOperation::deserialize(is, repo);
+    _doc.reset(new Document(repo, is));
+}
+
+vespalib::string
+PutOperation::toString() const
+{
+    return make_string("Put(%s, %s)",
+                       _doc.get() ?
+                       _doc->getId().getScheme().toString().c_str() : "NULL",
+                       docArgsToString().c_str());
+}
+
+void
+PutOperation::assertValid() const
+{
+    assertValidBucketId(_doc->getId());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h
new file mode 100644
index 00000000000..61c067530d2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentoperation.h"
+
+namespace proton {
+
+class PutOperation : public DocumentOperation
+{
+    document::Document::SP _doc;
+
+public:
+    PutOperation();
+    PutOperation(const document::BucketId &bucketId,
+                 const storage::spi::Timestamp &timestamp,
+                 const document::Document::SP &doc);
+    PutOperation(const document::BucketId &bucketId,
+                 const storage::spi::Timestamp &timestamp,
+                 const document::Document::SP &doc,
+                 SerialNum serialNum,
+                 DbDocumentId dbdId,
+                 DbDocumentId prevDbdId);
+    virtual ~PutOperation() {}
+    const document::Document::SP &getDocument() const { return _doc; }
+    void assertValid() const;
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp
new file mode 100644
index 00000000000..a61819d861a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.removedocumentsoperation");
+
+#include "removedocumentsoperation.h"
+
+namespace proton {
+
+RemoveDocumentsOperation::RemoveDocumentsOperation(Type type)
+    : FeedOperation(type),
+      _lidsToRemoveMap()
+{
+}
+
+
+void
+RemoveDocumentsOperation::serializeLidsToRemove(vespalib::nbostream &os) const
+{
+    uint32_t mapSize = _lidsToRemoveMap.size();
+    os << mapSize;
+    for (LidsToRemoveMap::const_iterator
+             it = _lidsToRemoveMap.begin(), ite = _lidsToRemoveMap.end();
+         it != ite; ++it) {
+        os << it->first;
+        it->second->serialize(os);
+    }
+}
+
+
+void
+RemoveDocumentsOperation::deserializeLidsToRemove(vespalib::nbostream &is)
+{
+    uint32_t mapSize;
+    uint32_t i;
+    is >> mapSize;
+    for (i = 0; i < mapSize; ++i) {
+        uint32_t subDbId;
+        is >> subDbId;
+        LidVectorContext::LP lidsToRemove(new LidVectorContext());
+        lidsToRemove->deserialize(is);
+        setLidsToRemove(subDbId, lidsToRemove);
+    }
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.h
new file mode 100644
index 00000000000..5056a3ced5b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+#include "lidvectorcontext.h"
+
+namespace proton {
+
+class RemoveDocumentsOperation : public FeedOperation
+{
+protected:
+    typedef std::map<uint32_t, LidVectorContext::LP> LidsToRemoveMap;
+    LidsToRemoveMap _lidsToRemoveMap;
+
+    RemoveDocumentsOperation(Type type);
+
+    void serializeLidsToRemove(vespalib::nbostream &os) const;
+    void deserializeLidsToRemove(vespalib::nbostream &is);
+public:
+    virtual ~RemoveDocumentsOperation() { }
+
+    void setLidsToRemove(uint32_t subDbId, const LidVectorContext::LP &lidsToRemove) {
+        _lidsToRemoveMap[subDbId] = lidsToRemove;
+    }
+
+    const LidVectorContext::LP
+    getLidsToRemove(uint32_t subDbId) const {
+        LidsToRemoveMap::const_iterator found(_lidsToRemoveMap.find(subDbId));
+        if (found != _lidsToRemoveMap.end())
+            return found->second;
+        else
+            return LidVectorContext::LP();
+    }
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp
new file mode 100644
index 00000000000..aa3faeda77c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.removeoperation");
+
+#include "removeoperation.h"
+
+using document::BucketId;
+using document::DocumentId;
+using document::DocumentTypeRepo;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+
+namespace proton {
+
+RemoveOperation::RemoveOperation()
+    : DocumentOperation(FeedOperation::REMOVE),
+      _docId()
+{
+}
+
+
+RemoveOperation::RemoveOperation(const BucketId &bucketId,
+                                 const Timestamp &timestamp,
+                                 const DocumentId &docId)
+    : DocumentOperation(FeedOperation::REMOVE,
+                        bucketId,
+                        timestamp),
+      _docId(docId)
+{
+}
+
+
+RemoveOperation::RemoveOperation(const document::BucketId &bucketId,
+                                 const storage::spi::Timestamp &timestamp,
+                                 const document::DocumentId &docId,
+                                 SerialNum serialNum,
+                                 DbDocumentId dbdId,
+                                 DbDocumentId prevDbdId)
+    : DocumentOperation(FeedOperation::REMOVE,
+                        bucketId,
+                        timestamp,
+                        serialNum,
+                        dbdId,
+                        prevDbdId),
+      _docId(docId)
+{
+}
+
+
+void
+RemoveOperation::serialize(vespalib::nbostream &os) const
+{
+    assertValidBucketId(_docId);
+    DocumentOperation::serialize(os);
+    vespalib::string rawId = _docId.toString();
+    os.write(rawId.c_str(), rawId.size() + 1);
+}
+
+
+void
+RemoveOperation::deserialize(vespalib::nbostream &is,
+                          const DocumentTypeRepo &repo)
+{
+    DocumentOperation::deserialize(is, repo);
+    _docId = DocumentId(is);
+}
+
+vespalib::string RemoveOperation::toString() const {
+    return make_string("Remove(%s, %s)",
+                       _docId.getScheme().toString().c_str(),
+                       docArgsToString().c_str());
+}
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h
new file mode 100644
index 00000000000..54e2936150b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentoperation.h"
+
+namespace proton {
+
+class RemoveOperation : public DocumentOperation {
+    document::DocumentId _docId;
+
+public:
+    RemoveOperation();
+    RemoveOperation(const document::BucketId &bucketId,
+                    const storage::spi::Timestamp &timestamp,
+                    const document::DocumentId &docId);
+    RemoveOperation(const document::BucketId &bucketId,
+                    const storage::spi::Timestamp &timestamp,
+                    const document::DocumentId &docId,
+                    SerialNum serialNum,
+                    DbDocumentId dbdId,
+                    DbDocumentId prevDbdId);
+    virtual ~RemoveOperation() {}
+    const document::DocumentId &getDocumentId() const { return _docId; }
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+
+    bool hasDocType() const { return _docId.hasDocType(); }
+    vespalib::string getDocType() const { return _docId.getDocType(); }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.cpp
new file mode 100644
index 00000000000..3fbf0e93fdf
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.cpp
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.splitbucketoperation");
+
+#include "splitbucketoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::BucketId;
+using document::DocumentTypeRepo;
+using vespalib::make_string;
+
+namespace proton {
+
+SplitBucketOperation::SplitBucketOperation()
+    : FeedOperation(FeedOperation::SPLIT_BUCKET),
+      _source(),
+      _target1(),
+      _target2()
+{
+}
+
+
+SplitBucketOperation::SplitBucketOperation(const document::BucketId &source,
+                                           const document::BucketId &target1,
+                                           const document::BucketId &target2)
+    : FeedOperation(FeedOperation::SPLIT_BUCKET),
+      _source(source),
+      _target1(target1),
+      _target2(target2)
+{
+}
+
+
+void
+SplitBucketOperation::serialize(vespalib::nbostream &os) const
+{
+    {
+        assert(_source.valid());
+        assert(_target1.valid() || _target2.valid());
+        if (_target1.valid()) {
+            assert(_source.getUsedBits() < _target1.getUsedBits());
+            assert(_source.contains(_target1));
+        }
+        if (_target2.valid()) {
+            assert(_source.getUsedBits() < _target2.getUsedBits());
+            assert(_source.contains(_target2));
+        }
+        if (_target1.valid() && _target2.valid()) {
+            assert(_target1 != _target2);
+            assert(!_target1.contains(_target2));
+            assert(!_target2.contains(_target1));
+        }
+    }
+    os << _source;
+    os << _target1;
+    os << _target2;
+}
+
+
+void
+SplitBucketOperation::deserialize(vespalib::nbostream &is,
+                                  const DocumentTypeRepo &)
+{
+    is >> _source;
+    is >> _target1;
+    is >> _target2;
+}
+
+vespalib::string SplitBucketOperation::toString() const {
+    return make_string("SplitBucket(source=%s, target1=%s, target2=%s, "
+                       "serialNum=%" PRIu64 ")",
+                       _source.toString().c_str(),
+                       _target1.toString().c_str(),
+                       _target2.toString().c_str(), getSerialNum());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.h
new file mode 100644
index 00000000000..a9f54f262ef
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.h
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+#include <vespa/document/bucket/bucketid.h>
+
+namespace proton {
+
+class SplitBucketOperation : public FeedOperation
+{
+private:
+    document::BucketId _source;
+    document::BucketId _target1;
+    document::BucketId _target2;
+public:
+    SplitBucketOperation();
+    SplitBucketOperation(const document::BucketId &source,
+                         const document::BucketId &target1,
+                         const document::BucketId &target2);
+    virtual ~SplitBucketOperation() {}
+    const document::BucketId &getSource() const { return _source; }
+    const document::BucketId &getTarget1() const { return _target1; }
+    const document::BucketId &getTarget2() const { return _target2; }
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp
new file mode 100644
index 00000000000..9413e5c0dbc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.spoolerreplayoperation");
+
+#include "spoolerreplayoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using vespalib::make_string;
+
+namespace proton {
+
+
+SpoolerReplayOperation::SpoolerReplayOperation(Type type)
+    : FeedOperation(type),
+    _spoolerSerialNum()
+{
+}
+
+
+SpoolerReplayOperation::SpoolerReplayOperation(Type type,
+                                               SerialNum serialNum,
+                                               SerialNum spoolerSerialNum)
+    : FeedOperation(type),
+    _spoolerSerialNum(spoolerSerialNum)
+{
+    setSerialNum(serialNum);
+}
+
+
+void
+SpoolerReplayOperation::serialize(vespalib::nbostream &os) const
+{
+    LOG(debug, "serialize(): %s", toString().c_str());
+    os << _spoolerSerialNum;
+}
+
+
+void
+SpoolerReplayOperation::deserialize(vespalib::nbostream &is)
+{
+    is >> _spoolerSerialNum;
+}
+
+vespalib::string SpoolerReplayOperation::toString() const {
+    return make_string(
+            "SpoolerReplay%s(spoolerSerialNum=%" PRIu64
+            ", serialNum=%" PRIu64 ")",
+            getType() == SPOOLER_REPLAY_START ? "Start" : "Complete",
+            _spoolerSerialNum, getSerialNum());
+}
+
+
+SpoolerReplayStartOperation::SpoolerReplayStartOperation()
+    : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_START)
+{
+}
+
+
+SpoolerReplayStartOperation::SpoolerReplayStartOperation(SerialNum serialNum,
+                                                         SerialNum spoolerSerialNum)
+    : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_START,
+                             serialNum,
+                             spoolerSerialNum)
+{
+}
+
+
+SpoolerReplayCompleteOperation::SpoolerReplayCompleteOperation()
+    : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_COMPLETE)
+{
+}
+
+
+SpoolerReplayCompleteOperation::SpoolerReplayCompleteOperation(SerialNum serialNum,
+                                                               SerialNum spoolerSerialNum)
+    : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_COMPLETE,
+                             serialNum,
+                             spoolerSerialNum)
+{
+}
+
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h
new file mode 100644
index 00000000000..b1bae7ceda4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+
+namespace proton {
+
+class SpoolerReplayOperation : public FeedOperation
+{
+private:
+    SerialNum _spoolerSerialNum;
+protected:
+    SpoolerReplayOperation(Type type);
+    SpoolerReplayOperation(Type type,
+                           SerialNum serialNum,
+                           SerialNum spoolerSerialNum);
+public:
+    virtual ~SpoolerReplayOperation() {}
+    SerialNum getSpoolerSerialNum() const { return _spoolerSerialNum; }
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &) {
+        deserialize(is);
+    }
+    void deserialize(vespalib::nbostream &is);
+    virtual vespalib::string toString() const;
+};
+
+
+/**
+ * Indicate that we are starting replaying the spooler log.
+ */
+class SpoolerReplayStartOperation : public SpoolerReplayOperation
+{
+public:
+    SpoolerReplayStartOperation();
+    /**
+     * @param serialNum the current serial number of the transaction log.
+     * @param spoolerSerialNum the serial number of the first entry of the spooler log replay.
+     */
+    SpoolerReplayStartOperation(SerialNum serialNum,
+                                SerialNum spoolerSerialNum);
+};
+
+
+/**
+ * Indicate that we are complete replaying the spooler log.
+ */
+class SpoolerReplayCompleteOperation : public SpoolerReplayOperation
+{
+public:
+    SpoolerReplayCompleteOperation();
+    /**
+     * @param serialNum the current serial number of the transaction log.
+     * @param spoolerSerialNum the serial number of the last entry of the spooler log replay.
+     */
+    SpoolerReplayCompleteOperation(SerialNum serialNum,
+                                   SerialNum spoolerSerialNum);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp
new file mode 100644
index 00000000000..21c008eec99
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.updateoperation");
+
+#include "updateoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using document::BucketId;
+using document::DocumentTypeRepo;
+using document::DocumentUpdate;
+using storage::spi::Timestamp;
+using vespalib::make_string;
+
+namespace proton {
+
+UpdateOperation::UpdateOperation()
+    : DocumentOperation(FeedOperation::UPDATE),
+      _upd()
+{
+}
+
+
+UpdateOperation::UpdateOperation(const BucketId &bucketId,
+                                 const Timestamp &timestamp,
+                                 const DocumentUpdate::SP &upd)
+    : DocumentOperation(FeedOperation::UPDATE,
+                        bucketId,
+                        timestamp),
+      _upd(upd)
+{
+}
+
+
+UpdateOperation::UpdateOperation(const document::BucketId &bucketId,
+                                 const storage::spi::Timestamp &timestamp,
+                                 const document::DocumentUpdate::SP &upd,
+                                 SerialNum serialNum,
+                                 DbDocumentId dbdId,
+                                 DbDocumentId prevDbdId)
+    : DocumentOperation(FeedOperation::UPDATE,
+                        bucketId,
+                        timestamp,
+                        serialNum,
+                        dbdId,
+                        prevDbdId),
+      _upd(upd)
+{
+}
+
+
+void
+UpdateOperation::serialize(vespalib::nbostream &os) const
+{
+    assertValidBucketId(_upd->getId());
+    DocumentOperation::serialize(os);
+    _upd->serialize42(os);
+}
+
+
+void
+UpdateOperation::deserialize(vespalib::nbostream &is,
+                             const DocumentTypeRepo &repo)
+{
+    DocumentOperation::deserialize(is, repo);
+    document::ByteBuffer buf(is.peek(), is.size());
+    try {
+        DocumentUpdate::SP update(new DocumentUpdate(repo, buf,
+                                                     DocumentUpdate::
+                                                     SerializeVersion::
+                                                     SERIALIZE_42));
+        is.adjustReadPos(buf.getPos());
+        _upd = update;
+    } catch (document::DocumentTypeNotFoundException &e) {
+        LOG(warning, "Failed deserialize update operation using unknown document type '%s'",
+            e.getDocumentTypeName().c_str());
+        // Ignore this piece of data
+        is.clear();
+    }
+}
+
+vespalib::string UpdateOperation::toString() const {
+    return make_string("Update(%s, %s)",
+                       _upd.get() ?
+                       _upd->getId().getScheme().toString().c_str() : "NULL",
+                       docArgsToString().c_str());
+}
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h
new file mode 100644
index 00000000000..c5ae9960b39
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentoperation.h"
+#include <vespa/document/update/documentupdate.h>
+
+namespace proton {
+
+class UpdateOperation : public DocumentOperation
+{
+private:
+    document::DocumentUpdate::SP _upd;
+public:
+    UpdateOperation();
+    UpdateOperation(const document::BucketId &bucketId,
+                    const storage::spi::Timestamp &timestamp,
+                    const document::DocumentUpdate::SP &upd);
+    UpdateOperation(const document::BucketId &bucketId,
+                    const storage::spi::Timestamp &timestamp,
+                    const document::DocumentUpdate::SP &upd,
+                    SerialNum serialNum,
+                    DbDocumentId dbdId,
+                    DbDocumentId prevDbdId);
+    virtual ~UpdateOperation() {}
+    const document::DocumentUpdate::SP &getUpdate() const { return _upd; }
+    virtual void serialize(vespalib::nbostream &os) const;
+    virtual void deserialize(vespalib::nbostream &is,
+                             const document::DocumentTypeRepo &repo);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp
new file mode 100644
index 00000000000..5e4f2046481
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedoperation.wipehistoryoperation");
+
+#include "wipehistoryoperation.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using vespalib::make_string;
+
+namespace proton {
+
+WipeHistoryOperation::WipeHistoryOperation()
+    : FeedOperation(FeedOperation::WIPE_HISTORY),
+      _wipeTimeLimit(0) {
+}
+
+WipeHistoryOperation::WipeHistoryOperation(SerialNum serialNum,
+                                           fastos::TimeStamp wipeTimeLimit)
+    : FeedOperation(FeedOperation::WIPE_HISTORY),
+      _wipeTimeLimit(wipeTimeLimit) {
+    setSerialNum(serialNum);
+}
+
+void WipeHistoryOperation::serialize(vespalib::nbostream &str) const {
+    str << _wipeTimeLimit;
+}
+void WipeHistoryOperation::deserialize(vespalib::nbostream &str,
+                                       const document::DocumentTypeRepo &) {
+    fastos::TimeStamp::TimeT t;
+    str >> t;
+    _wipeTimeLimit = t;
+}
+
+vespalib::string WipeHistoryOperation::toString() const {
+    return make_string("WipeHistory(wipeTimeLimit=%" PRIu64
+                       ", serialNum=%" PRIu64 ")",
+                       _wipeTimeLimit.ns(), getSerialNum());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h
new file mode 100644
index 00000000000..88328953377
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "feedoperation.h"
+
+namespace proton {
+
+class WipeHistoryOperation : public FeedOperation {
+    fastos::TimeStamp _wipeTimeLimit;
+
+public:
+    WipeHistoryOperation();
+    WipeHistoryOperation(SerialNum serialNum, fastos::TimeStamp wipeTimeLimit);
+    virtual ~WipeHistoryOperation() {}
+
+    fastos::TimeStamp getWipeTimeLimit() const { return _wipeTimeLimit; }
+
+    virtual void serialize(vespalib::nbostream &str) const;
+    virtual void deserialize(vespalib::nbostream &str,
+                             const document::DocumentTypeRepo &);
+    virtual vespalib::string toString() const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/fix_log_setup.rb b/searchcore/src/vespa/searchcore/proton/fix_log_setup.rb
new file mode 100644
index 00000000000..686e129f7d4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/fix_log_setup.rb
@@ -0,0 +1,24 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+def log_setup_name(fname)
+  m = fname.match("(.*)\/(.*)\.cpp")
+  return ".proton.#{m[1]}.#{m[2]}"
+end
+
+def fix_log_setup(fname)
+  nname = "#{fname}.new"
+  puts "fix '#{fname}': #{log_setup_name(fname)}"
+  nfile = File.open(nname, "w")
+  File.open(fname, "r").each_line do |line|
+    if (line.match("LOG_SETUP"))
+      nfile.write("LOG_SETUP(\"#{log_setup_name(fname)}\");\n")
+    else
+      nfile.write(line)
+    end
+  end
+  nfile.close
+  File.rename(nname, fname)
+end
+
+Dir.glob("*/*.cpp").each do |file|
+  fix_log_setup(file)
+end
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/.gitignore b/searchcore/src/vespa/searchcore/proton/flushengine/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt
new file mode 100644
index 00000000000..6ef5d80106c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_flushengine STATIC
+    SOURCES
+    cachedflushtarget.cpp
+    flush_all_strategy.cpp
+    flushcontext.cpp
+    flushengine.cpp
+    flush_engine_explorer.cpp
+    flush_target_candidates.cpp
+    flushtargetproxy.cpp
+    flushtask.cpp
+    prepare_restart_flush_strategy.cpp
+    threadedflushtarget.cpp
+    tls_stats_factory.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/OWNERS b/searchcore/src/vespa/searchcore/proton/flushengine/OWNERS
new file mode 100644
index 00000000000..1037590124e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/OWNERS
@@ -0,0 +1 @@
+balder
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp
new file mode 100644
index 00000000000..fe10efb816e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.flushengine.cachedflushtarget");
+
+#include "cachedflushtarget.h"
+
+namespace proton {
+
+CachedFlushTarget::CachedFlushTarget(const IFlushTarget::SP &target)
+    : IFlushTarget(target->getName(), target->getType(), target->getComponent()),
+      _target(target),
+      _flushedSerialNum(target->getFlushedSerialNum()),
+      _lastFlushTime(target->getLastFlushTime()),
+      _memoryGain(target->getApproxMemoryGain()),
+      _diskGain(target->getApproxDiskGain()),
+      _needUrgentFlush(target->needUrgentFlush()),
+      _approxBytesToWriteToDisk(target->getApproxBytesToWriteToDisk())
+{
+    // empty
+}
+
+
+uint64_t
+CachedFlushTarget::getApproxBytesToWriteToDisk() const
+{
+    return _approxBytesToWriteToDisk;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h
new file mode 100644
index 00000000000..d5302ea62ca
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+
+namespace proton {
+
+using searchcorespi::FlushStats;
+using searchcorespi::IFlushTarget;
+
+/**
+ * Implements a flush target that caches the flushable memory and flush cost of
+ * a decorated target. This is used by the flush engine to avoid recalculating
+ * these during selection of flush target.
+ */
+class CachedFlushTarget : public IFlushTarget {
+private:
+    IFlushTarget::SP  _target;
+    SerialNum         _flushedSerialNum;
+    Time              _lastFlushTime;
+    MemoryGain        _memoryGain;
+    DiskGain          _diskGain;
+    bool              _needUrgentFlush;
+    uint64_t          _approxBytesToWriteToDisk;
+
+public:
+    /**
+     * Constructs a new instance of this class. This will immediately call
+     * getFlushableMemory(), getFlushCost() and getLowSerialNum() on the
+     * argument target.
+     *
+     * @param target The target to decorate.
+     */
+    CachedFlushTarget(const IFlushTarget::SP &target);
+
+    /**
+     * Returns the decorated flush target. This should not be used for anything
+     * but testing, as invoking a method on the returned target beats the
+     * purpose of decorating it.
+     *
+     * @return The decorated flush target.
+     */
+    const IFlushTarget::SP & getFlushTarget() { return _target; }
+
+    // Implements IFlushTarget.
+    virtual MemoryGain getApproxMemoryGain() const { return _memoryGain; }
+    virtual   DiskGain   getApproxDiskGain() const { return _diskGain; }
+    virtual  SerialNum getFlushedSerialNum() const { return _flushedSerialNum; }
+    virtual       Time    getLastFlushTime() const { return _lastFlushTime; }
+    virtual       bool     needUrgentFlush() const { return _needUrgentFlush; }
+
+    virtual Task::UP initFlush(SerialNum currentSerial) { return _target->initFlush(currentSerial); }
+    virtual FlushStats getLastFlushStats() const { return _target->getLastFlushStats(); }
+
+    virtual uint64_t getApproxBytesToWriteToDisk() const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.cpp
new file mode 100644
index 00000000000..57d1042df3d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.cpp
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+
+#include "flush_all_strategy.h"
+#include <algorithm>
+
+using search::SerialNum;
+
+namespace proton {
+
+namespace
+{
+
+class CompareTarget
+{
+public:
+    bool
+    operator ()(const FlushContext::SP &lfc,
+                const FlushContext::SP &rfc) const;
+};
+
+bool
+CompareTarget::operator()(const FlushContext::SP &lfc,
+                          const FlushContext::SP &rfc) const
+{
+    const IFlushTarget &lhs = *lfc->getTarget();
+    const IFlushTarget &rhs = *rfc->getTarget();
+    // Note: This assumes that last flush time is stable while doing sort
+    return lhs.getLastFlushTime() < rhs.getLastFlushTime();
+}
+
+}
+
+FlushAllStrategy::FlushAllStrategy()
+{
+}
+
+FlushContext::List
+FlushAllStrategy::getFlushTargets(const FlushContext::List &targetList,
+                                  const flushengine::TlsStatsMap &) const
+{
+    if (targetList.empty()) {
+        return FlushContext::List();
+    }
+    FlushContext::List fv(targetList);
+    std::sort(fv.begin(), fv.end(), CompareTarget());
+    return fv;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.h
new file mode 100644
index 00000000000..8a0b2fa43dc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/utility.hpp>
+#include <vespa/searchcore/proton/flushengine/iflushstrategy.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <map>
+
+namespace proton {
+
+/*
+ * Class implementing "flush" everything strategy.  Targets are just
+ * sorted on age.
+ */
+class FlushAllStrategy : public boost::noncopyable,
+                    public IFlushStrategy
+{
+public:
+    FlushAllStrategy();
+
+    // Implements IFlushStrategy
+    virtual FlushContext::List
+    getFlushTargets(const FlushContext::List &targetList,
+                    const flushengine::TlsStatsMap &) const override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp
new file mode 100644
index 00000000000..3c649dd5614
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.flushengine.flush_engine_explorer");
+#include "flush_engine_explorer.h"
+
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <vespa/vespalib/data/slime/inserter.h>
+
+using vespalib::slime::Cursor;
+using vespalib::slime::Inserter;
+using vespalib::StateExplorer;
+
+namespace proton {
+
+namespace {
+
+void
+convertToSlime(const FlushEngine::FlushMetaSet &flushingTargets,
+               const fastos::TimeStamp &now,
+               Cursor &array)
+{
+    for (const auto &target : flushingTargets) {
+        Cursor &object = array.addObject();
+        object.setString("name", target.getName());
+        object.setString("startTime", target.getStart().toString());
+        fastos::TimeStamp elapsedTime = now - target.getStart();
+        object.setDouble("elapsedTime", elapsedTime.sec());
+    }
+}
+
+void
+sortTargetList(FlushContext::List &allTargets)
+{
+    std::sort(allTargets.begin(), allTargets.end(),
+            [](const FlushContext::SP &rhs, const FlushContext::SP &lhs) {
+                return rhs->getTarget()->getFlushedSerialNum() <
+                        lhs->getTarget()->getFlushedSerialNum();
+    });
+}
+
+void
+convertToSlime(const FlushContext::List &allTargets,
+               const fastos::TimeStamp &now,
+               Cursor &array)
+{
+    for (const auto &ctx : allTargets) {
+        Cursor &object = array.addObject();
+        object.setString("name", ctx->getName());
+        const IFlushTarget::SP &target = ctx->getTarget();
+        object.setLong("flushedSerialNum", target->getFlushedSerialNum());
+        object.setLong("memoryGain", target->getApproxMemoryGain().gain());
+        object.setLong("diskGain", target->getApproxDiskGain().gain());
+        object.setString("lastFlushTime", target->getLastFlushTime().toString());
+        fastos::TimeStamp timeSinceLastFlush = now - target->getLastFlushTime();
+        object.setDouble("timeSinceLastFlush", timeSinceLastFlush.sec());
+        object.setBool("needUrgentFlush", target->needUrgentFlush());
+    }
+}
+
+}
+
+FlushEngineExplorer::FlushEngineExplorer(const FlushEngine &engine)
+    : _engine(engine)
+{
+}
+
+void
+FlushEngineExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    Cursor &object = inserter.insertObject();
+    if (full) {
+        fastos::TimeStamp now = fastos::ClockSystem::now();
+        convertToSlime(_engine.getCurrentlyFlushingSet(), now, object.setArray("flushingTargets"));
+        FlushContext::List allTargets = _engine.getTargetList(true);
+        sortTargetList(allTargets);
+        convertToSlime(allTargets, now, object.setArray("allTargets"));
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h
new file mode 100644
index 00000000000..cf11057acd7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "flushengine.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of a flush engine and its flush targets.
+ */
+class FlushEngineExplorer : public vespalib::StateExplorer
+{
+private:
+    const FlushEngine &_engine;
+
+public:
+    FlushEngineExplorer(const FlushEngine &engine);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp
new file mode 100644
index 00000000000..e83b0adc3f5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.flushengine.flush_target_candidates");
+
+#include "flush_target_candidates.h"
+#include "tls_stats.h"
+
+namespace proton {
+
+using search::SerialNum;
+
+using Config = PrepareRestartFlushStrategy::Config;
+
+namespace {
+
+SerialNum
+calculateReplayStartSerial(const FlushContext::List &sortedFlushContexts,
+                           size_t numCandidates,
+                           const flushengine::TlsStats &tlsStats)
+{
+    if (numCandidates == 0) {
+        return tlsStats.getFirstSerial();
+    }
+    if (numCandidates == sortedFlushContexts.size()) {
+        return tlsStats.getLastSerial() + 1;
+    }
+    return sortedFlushContexts[numCandidates]->getTarget()->getFlushedSerialNum() + 1;
+}
+
+double
+calculateTlsReplayCost(const flushengine::TlsStats &tlsStats,
+                       const Config &cfg,
+                       SerialNum replayStartSerial)
+{
+    SerialNum replayEndSerial = tlsStats.getLastSerial();
+    SerialNum numTotalOperations = replayEndSerial - tlsStats.getFirstSerial() + 1;
+    if (numTotalOperations == 0) {
+        return 0;
+    }
+    double numBytesPerOperation =
+        (double)tlsStats.getNumBytes() / (double)numTotalOperations;
+    SerialNum numOperationsToReplay = replayEndSerial + 1 - replayStartSerial;
+    double numBytesToReplay = numBytesPerOperation * numOperationsToReplay;
+    return numBytesToReplay * cfg.tlsReplayCost;
+}
+
+double
+calculateFlushTargetsWriteCost(const FlushContext::List &sortedFlushContexts,
+                               size_t numCandidates,
+                               const Config &cfg)
+{
+    double result = 0;
+    for (size_t i = 0; i < numCandidates; ++i) {
+        const auto &flushContext = sortedFlushContexts[i];
+        result += (flushContext->getTarget()->getApproxBytesToWriteToDisk() *
+                cfg.flushTargetWriteCost);
+    }
+    return result;
+}
+
+}
+
+FlushTargetCandidates::FlushTargetCandidates(const FlushContext::List &sortedFlushContexts,
+                                             size_t numCandidates,
+                                             const flushengine::TlsStats &tlsStats,
+                                             const Config &cfg)
+    : _sortedFlushContexts(&sortedFlushContexts),
+      _numCandidates(numCandidates),
+      _tlsReplayCost(calculateTlsReplayCost(tlsStats,
+                                            cfg,
+                                            calculateReplayStartSerial(sortedFlushContexts,
+                                                                       numCandidates,
+                                                                       tlsStats))),
+      _flushTargetsWriteCost(calculateFlushTargetsWriteCost(sortedFlushContexts,
+                                                            numCandidates,
+                                                            cfg))
+{
+}
+
+FlushContext::List
+FlushTargetCandidates::getCandidates() const
+{
+    FlushContext::List result(_sortedFlushContexts->begin(),
+            _sortedFlushContexts->begin() + _numCandidates);
+    return result;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h
new file mode 100644
index 00000000000..d1f13b61ac4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "prepare_restart_flush_strategy.h"
+
+namespace proton {
+
+namespace flushengine { class TlsStats; }
+
+/**
+ * A set of flush targets that are candidates to be flushed.
+ *
+ * The total cost of using this set of candidates is:
+ *   - the cost of replaying the TLS (after these are flushed) +
+ *   - the cost of flushing these to disk
+ */
+class FlushTargetCandidates
+{
+private:
+    const FlushContext::List *_sortedFlushContexts; // NOTE: ownership is handled outside
+    size_t _numCandidates;
+    double _tlsReplayCost;
+    double _flushTargetsWriteCost;
+
+    using Config = PrepareRestartFlushStrategy::Config;
+
+public:
+    using UP = std::unique_ptr<FlushTargetCandidates>;
+
+    FlushTargetCandidates(const FlushContext::List &sortedFlushContexts,
+                          size_t numCandidates,
+                          const flushengine::TlsStats &tlsStats,
+                          const Config &cfg);
+
+    double getTlsReplayCost() const { return _tlsReplayCost; }
+    double getFlushTargetsWriteCost() const { return _flushTargetsWriteCost; }
+    double getTotalCost() const { return getTlsReplayCost() + getFlushTargetsWriteCost(); }
+    FlushContext::List getCandidates() const;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp
new file mode 100644
index 00000000000..7f436a8d594
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.flushengine.flushcontext");
+
+#include "flushcontext.h"
+
+namespace proton {
+
+FlushContext::FlushContext(
+        const IFlushHandler::SP &handler,
+        const IFlushTarget::SP &target,
+        search::SerialNum oldestFlushable,
+        search::SerialNum lastSerial)
+    : _name(createName(*handler, *target)),
+      _handler(handler),
+      _target(target),
+      _task(),
+      _oldestFlushable(oldestFlushable),
+      _lastSerial(lastSerial)
+{
+    // empty
+}
+
+vespalib::string FlushContext::createName(const IFlushHandler & handler, const IFlushTarget & target)
+{
+    return (handler.getName() + "." + target.getName());
+}
+
+FlushContext::~FlushContext()
+{
+    if (_task.get() != NULL) {
+        LOG(warning, "Unexecuted flush task for '%s' destroyed.",
+            _name.c_str());
+    }
+}
+
+bool
+FlushContext::initFlush()
+{
+    LOG(debug, "Attempting to flush '%s'.", _name.c_str());
+    _task = _target->initFlush(std::max(_handler->getCurrentSerialNumber(), _lastSerial));
+    if (_task.get() == NULL) {
+        LOG(debug, "Target refused to init flush.");
+        return false;
+    }
+    return true;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h
new file mode 100644
index 00000000000..9f2b557b3a9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/utility.hpp>
+#include <vespa/searchcore/proton/flushengine/iflushhandler.h>
+#include <vespa/vespalib/util/executor.h>
+
+namespace proton {
+
+using searchcorespi::IFlushTarget;
+
+/**
+ * This class is used by FlushEngine to hold the necessary context for flushing
+ * a single IFlushTarget.
+ */
+class FlushContext : public boost::noncopyable {
+private:
+    vespalib::string                  _name;
+    IFlushHandler::SP                 _handler;
+    IFlushTarget::SP                  _target;
+    searchcorespi::FlushTask::UP      _task;
+    search::SerialNum _oldestFlushable;
+    search::SerialNum _lastSerial;
+
+public:
+    typedef std::shared_ptr<FlushContext> SP;
+    typedef std::vector<SP> List;
+
+    /**
+     * Create a name of the handler and the target.
+     *
+     * @param handler The flush handler that contains the given target.
+     * @param target  The target to flush.
+     * @return the name created.
+     */
+    static vespalib::string createName(const IFlushHandler & handler, const IFlushTarget & target);
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param handler The flush handler that contains the given target.
+     * @param target  The target to flush.
+     */
+    FlushContext(const IFlushHandler::SP &handler,
+                 const IFlushTarget::SP &target,
+                 search::SerialNum oldestFlushable,
+                 search::SerialNum lastSerial);
+
+    /**
+     * Destructor. Will log a warning if it contains an unexecuted task.
+     */
+    ~FlushContext();
+
+    /**
+     * This method proxies initFlush() in IFlushTarget, but simplifies the call
+     * signature. If this method returns true, the task to complete the flush is
+     * available through getTask().
+     *
+     * @param True if a flush was initiated.
+     */
+    bool initFlush();
+
+    /**
+     * Returns the unique name of this context. This is the concatenation of the
+     * handler and target names.
+     *
+     * @return The name of this.
+     */
+    const vespalib::string & getName() const { return _name; }
+
+    /**
+     * Returns the flush handler of this context.
+     *
+     * @return The handler.
+     */
+    const IFlushHandler::SP & getHandler() const { return _handler; }
+
+    /**
+     * Returns the flush target of this context.
+     *
+     * @return The target.
+     */
+    const IFlushTarget::SP & getTarget() const { return _target; }
+
+    /**
+     * Returns the oldest flushable serial number.
+     *
+     * @return The oldest flushable serial number
+     */
+    search::SerialNum getOldestFlushable() const { return _oldestFlushable; }
+
+    /**
+     * Returns the last serial number.
+     *
+     * @return The last serial number
+     */
+    search::SerialNum getLastSerial() const { return _lastSerial; }
+
+    /**
+     * Returns the task required to be run to complete an initiated flush. This
+     * is null until initFlush() has been called.
+     *
+     * @return The flush completion task.
+     */
+    searchcorespi::FlushTask::UP getTask() { return std::move(_task); }
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp
new file mode 100644
index 00000000000..b1a2e23fca0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp
@@ -0,0 +1,413 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include "cachedflushtarget.h"
+#include "flush_all_strategy.h"
+#include "flushengine.h"
+#include "flushtask.h"
+#include "tls_stats_map.h"
+#include "tls_stats_factory.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/vespalib/util/jsonwriter.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+LOG_SETUP(".proton.flushengine.flushengine");
+
+using vespalib::MonitorGuard;
+typedef vespalib::Executor::Task Task;
+
+namespace proton {
+
+namespace {
+
+search::SerialNum
+findOldestFlushedSerial(const IFlushTarget::List &lst,
+                        const IFlushHandler &handler,
+                        const IFlushTarget *self)
+{
+    search::SerialNum ret(handler.getCurrentSerialNumber());
+    for (const IFlushTarget::SP & target : lst) {
+        if (self != target.get()) {
+            ret = std::min(ret, target->getFlushedSerialNum());
+        }
+    }
+    LOG(debug, "Oldest flushed serial for '%s' will be %" PRIu64 " after flush.", handler.getName().c_str(), ret);
+    return ret;
+}
+
+}
+
+FlushEngine::FlushInfo::FlushInfo() :
+    FlushMeta("", fastos::ClockSystem::now(), 0),
+    _target()
+{
+}
+
+
+FlushEngine::FlushInfo::FlushInfo(uint32_t taskId,
+                                  const IFlushTarget::SP &target,
+                                  const vespalib::string & destination) :
+    FlushMeta(destination, fastos::ClockSystem::now(), taskId),
+    _target(target)
+{
+}
+
+FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory>
+                         tlsStatsFactory,
+                         IFlushStrategy::SP strategy, uint32_t numThreads,
+                         uint32_t idleIntervalMS, bool enableAutoPrune)
+    : _closed(false),
+      _maxConcurrent(numThreads),
+      _idleIntervalMS(idleIntervalMS),
+      _enableAutoPrune(enableAutoPrune),
+      _taskId(0),
+      _threadPool(128 * 1024),
+      _strategy(strategy),
+      _priorityStrategy(),
+      _executor(numThreads, 128 * 1024),
+      _monitor(),
+      _handlers(),
+      _flushing(),
+      _strategyLock(),
+      _strategyMonitor(),
+      _tlsStatsFactory(tlsStatsFactory)
+{
+    // empty
+}
+
+FlushEngine::~FlushEngine()
+{
+    close();
+    _executor.sync();
+}
+
+FlushEngine &
+FlushEngine::start()
+{
+    if (_threadPool.NewThread(this) == NULL) {
+        throw vespalib::IllegalStateException("Failed to start engine thread.");
+    }
+    return *this;
+}
+
+FlushEngine &
+FlushEngine::close()
+{
+    MonitorGuard strategyGuard(_strategyMonitor);
+    {
+        MonitorGuard guard(_monitor);
+        _closed = true;
+        guard.broadcast();
+    }
+    _threadPool.Close();
+    return *this;
+}
+
+void
+FlushEngine::triggerFlush()
+{
+    setStrategy(std::make_shared<FlushAllStrategy>());
+}
+
+void
+FlushEngine::kick(void)
+{
+    MonitorGuard guard(_monitor);
+    LOG(debug, "Kicking flush engine");
+    guard.broadcast();
+}
+
+bool
+FlushEngine::canFlushMore(const MonitorGuard & guard) const
+{
+    (void) guard;
+    return _maxConcurrent > _flushing.size();
+}
+
+bool
+FlushEngine::wait(size_t minimumWaitTimeIfReady)
+{
+    MonitorGuard guard(_monitor);
+    if ( (minimumWaitTimeIfReady > 0) && canFlushMore(guard)) {
+        guard.wait(minimumWaitTimeIfReady);
+    }
+    while ( ! canFlushMore(guard) ) {
+        guard.wait(1000); // broadcast when flush done
+    }
+    return !_closed;
+}
+
+void
+FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg)
+{
+    (void)thread;
+    (void)arg;
+    bool shouldIdle = false;
+    vespalib::string prevFlushName;
+    while (wait(shouldIdle ? _idleIntervalMS : 0)) {
+        shouldIdle = false;
+        prevFlushName = flushNextTarget(prevFlushName);
+        if ( ! prevFlushName.empty()) {
+            // Sleep at least 10 ms after a successful flush in order to avoid busy loop in case
+            // of strategy error or target error.
+            FastOS_Thread::Sleep(10);
+        } else {
+            shouldIdle = true;
+        }
+        if (_enableAutoPrune) {
+            prune();
+        }
+        LOG(debug, "Making another wait(idle=%s, timeMS=%d) last was '%s'", shouldIdle ? "true" : "false", shouldIdle ? _idleIntervalMS : 0, prevFlushName.c_str());
+    }
+}
+
+void FlushEngine::prune()
+{
+    if (_flushing.empty()) {
+        MonitorGuard guard(_monitor);
+        for (const auto & it : _handlers) {
+            IFlushHandler & handler(*it.second);
+            IFlushTarget::List lst = handler.getFlushTargets();
+            handler.flushDone(findOldestFlushedSerial(lst, handler, NULL));
+        }
+    }
+}
+
+bool FlushEngine::isFlushing(const MonitorGuard & guard, const vespalib::string & name) const
+{
+    (void) guard;
+    for(const auto & it : _flushing) {
+        if (name == it.second.getName()) {
+            return true;
+        }
+    }
+    return false;
+}
+
+FlushContext::List
+FlushEngine::getTargetList(bool includeFlushingTargets) const
+{
+    FlushContext::List ret;
+    {
+        MonitorGuard guard(_monitor);
+        for (const auto & it : _handlers) {
+            IFlushHandler & handler(*it.second);
+            search::SerialNum serial(handler.getCurrentSerialNumber());
+            LOG(spam, "Checking FlushHandler '%s' current serial = %ld",
+                handler.getName().c_str(), serial);
+            IFlushTarget::List lst = handler.getFlushTargets();
+            for (const IFlushTarget::SP & target : lst) {
+                LOG(spam, "Checking target '%s' with flushedSerialNum = %ld", target->getName().c_str(), target->getFlushedSerialNum());
+                if (!isFlushing(guard, FlushContext::createName(handler, *target)) || includeFlushingTargets) {
+                    ret.push_back(FlushContext::SP(new FlushContext(it.second,
+                                                                    IFlushTarget::SP(new CachedFlushTarget(target)),
+                                                                    findOldestFlushedSerial(lst, handler, target.get()),
+                                                                    serial)));
+                } else {
+                    LOG(debug, "Target '%s' with flushedSerialNum = %ld already has a flush going. Local last serial = %ld.",
+                               target->getName().c_str(), target->getFlushedSerialNum(), serial);
+                }
+            }
+        }
+    }
+    return ret;
+}
+
+std::pair<FlushContext::List,bool>
+FlushEngine::getSortedTargetList(MonitorGuard &strategyGuard) const
+{
+    (void) strategyGuard;
+    FlushContext::List unsortedTargets = getTargetList(false);
+    std::pair<FlushContext::List, bool> ret;
+    flushengine::TlsStatsMap tlsStatsMap(_tlsStatsFactory->create());
+    if (_priorityStrategy) {
+        ret = std::make_pair(_priorityStrategy->getFlushTargets(unsortedTargets, tlsStatsMap), true);
+    } else {
+        ret = std::make_pair(_strategy->getFlushTargets(unsortedTargets, tlsStatsMap), false);
+    }
+    return ret;
+}
+
+FlushContext::SP
+FlushEngine::initNextFlush(const FlushContext::List &lst)
+{
+    FlushContext::SP ctx;
+    for (const FlushContext::SP & it : lst) {
+        if (LOG_WOULD_LOG(event)) {
+            EventLogger::flushInit(it->getName());
+        }
+        if (it->initFlush()) {
+            ctx = it;
+            break;
+        }
+    }
+    if (ctx.get() != NULL) {
+        LOG(debug, "Target '%s' initiated flush of transactions %" PRIu64 " through %" PRIu64 ".",
+                   ctx->getName().c_str(),
+                   ctx->getTarget()->getFlushedSerialNum() + 1,
+                   ctx->getHandler()->getCurrentSerialNumber());
+    }
+    return ctx;
+}
+
+
+
+void
+FlushEngine::flushAll(const FlushContext::List &lst)
+{
+    LOG(debug, "%ld targets to flush.", lst.size());
+    for (const FlushContext::SP & ctx : lst) {
+        if (wait(0)) {
+            if (ctx->initFlush()) {
+                LOG(debug, "Target '%s' initiated flush of transactions %" PRIu64 " through %" PRIu64 ".",
+                           ctx->getName().c_str(),
+                           ctx->getTarget()->getFlushedSerialNum() + 1,
+                           ctx->getHandler()->getCurrentSerialNumber());
+                _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx, ctx->getOldestFlushable())));
+            } else {
+                LOG(debug, "Target '%s' failed to initiate flush of transactions %" PRIu64 " through %" PRIu64 ".",
+                           ctx->getName().c_str(),
+                           ctx->getTarget()->getFlushedSerialNum() + 1,
+                           ctx->getHandler()->getCurrentSerialNumber());
+            }
+        }
+
+    }
+}
+
+vespalib::string
+FlushEngine::flushNextTarget(const vespalib::string & name)
+{
+    MonitorGuard strategyGuard(_strategyMonitor);
+    std::pair<FlushContext::List,bool> lst = getSortedTargetList(strategyGuard);
+    if (lst.second) {
+        // Everything returned from a priority strategy should be flushed
+        flushAll(lst.first);
+        _executor.sync();
+        _priorityStrategy.reset();
+        strategyGuard.broadcast();
+        return "";
+    }
+    if (lst.first.empty()) {
+        LOG(debug, "No target to flush.");
+        return "";
+    }
+    FlushContext::SP ctx = initNextFlush(lst.first);
+    if (ctx.get() == NULL) {
+        LOG(debug, "All targets refused to flush.");
+        return "";
+    }
+    if ( name == ctx->getName()) {
+        LOG(info, "The same target %s out of %ld has been asked to flush again. "
+                  "This might indicate flush logic flaw so I will wait 1s before doing it.",
+                  name.c_str(), lst.first.size());
+        FastOS_Thread::Sleep(1000);
+    }
+    _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx, ctx->getOldestFlushable())));
+    return ctx->getName();
+}
+
+uint32_t
+FlushEngine::initFlush(const FlushContext &ctx)
+{
+    if (LOG_WOULD_LOG(event)) {
+        IFlushTarget::MemoryGain mgain(ctx.getTarget()->getApproxMemoryGain());
+        EventLogger::flushStart(ctx.getName(),
+                                mgain.getBefore(),
+                                mgain.getAfter(),
+                                mgain.gain(),
+                                ctx.getTarget()->getFlushedSerialNum() + 1,
+                                ctx.getHandler()->getCurrentSerialNumber());
+    }
+    return initFlush(ctx.getHandler(), ctx.getTarget());
+}
+
+void
+FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId)
+{
+    fastos::TimeStamp duration;
+    {
+        MonitorGuard guard(_monitor);
+        duration = fastos::TimeStamp(fastos::ClockSystem::now()) - _flushing[taskId].getStart();
+    }
+    if (LOG_WOULD_LOG(event)) {
+        FlushStats stats = ctx.getTarget()->getLastFlushStats();
+        EventLogger::flushComplete(ctx.getName(),
+                                   duration.ms(),
+                                   stats.getPath(),
+                                   stats.getPathElementsToLog());
+    }
+    LOG(debug, "FlushEngine::flushDone(taskId='%d') took '%f' secs", taskId, duration.sec());
+    MonitorGuard guard(_monitor);
+    _flushing.erase(taskId);
+    guard.broadcast();
+}
+
+IFlushHandler::SP
+FlushEngine::putFlushHandler(const DocTypeName &docTypeName,
+                             const IFlushHandler::SP &flushHandler)
+{
+    MonitorGuard guard(_monitor);
+    return _handlers.putHandler(docTypeName, flushHandler);
+}
+
+IFlushHandler::SP
+FlushEngine::getFlushHandler(const DocTypeName &docTypeName) const
+{
+    MonitorGuard guard(_monitor);
+    return _handlers.getHandler(docTypeName);
+}
+
+IFlushHandler::SP
+FlushEngine::removeFlushHandler(const DocTypeName &docTypeName)
+{
+    MonitorGuard guard(_monitor);
+    return _handlers.removeHandler(docTypeName);
+}
+
+FlushEngine::FlushMetaSet
+FlushEngine::getCurrentlyFlushingSet() const
+{
+    FlushMetaSet s;
+    vespalib::LockGuard guard(_monitor);
+    for (const auto & it : _flushing) {
+        s.insert(it.second);
+    }
+    return s;
+}
+
+uint32_t
+FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target)
+{
+    uint32_t taskId(0);
+    {
+        vespalib::LockGuard guard(_monitor);
+        taskId = _taskId++;
+        vespalib::string name(FlushContext::createName(*handler, *target));
+        FlushInfo flush(taskId, target, name);
+        _flushing[taskId] = flush;
+    }
+    LOG(debug, "FlushEngine::initFlush(handler='%s', target='%s') => taskId='%d'",
+        handler->getName().c_str(), target->getName().c_str(), taskId);
+    return taskId;
+}
+
+void
+FlushEngine::setStrategy(IFlushStrategy::SP strategy)
+{
+    MonitorGuard strategyGuard(_strategyMonitor);
+    if (_closed) {
+        return;
+    }
+    assert(!_priorityStrategy);
+    _priorityStrategy = strategy;
+    {
+        MonitorGuard guard(_monitor);
+        guard.broadcast();
+    }
+    while (_priorityStrategy) {
+        strategyGuard.wait();
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h
new file mode 100644
index 00000000000..215a8ac8de5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h
@@ -0,0 +1,185 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/common/handlermap.hpp>
+#include <vespa/searchcore/proton/flushengine/flushcontext.h>
+#include <vespa/searchcore/proton/flushengine/iflushstrategy.h>
+#include <set>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+
+namespace proton {
+
+namespace flushengine { class ITlsStatsFactory; }
+
+class FlushEngine : public boost::noncopyable,
+                    public FastOS_Runnable
+{
+public:
+    class FlushMeta {
+    public:
+        FlushMeta(const vespalib::string & name, fastos::TimeStamp start, uint32_t id) :
+            _name(name),
+            _start(start),
+            _id(id)
+        { }
+        const vespalib::string & getName() const { return _name; }
+        fastos::TimeStamp getStart() const { return _start; }
+        uint32_t getId() const { return _id; }
+        bool operator < (const FlushMeta & rhs) const { return _id < rhs._id; }
+    private:
+        vespalib::string  _name;
+        fastos::TimeStamp _start;
+        uint32_t          _id;
+    };
+    typedef std::set<FlushMeta> FlushMetaSet;
+private:
+    struct FlushInfo : public FlushMeta
+    {
+        FlushInfo();
+        FlushInfo(uint32_t taskId,
+                  const IFlushTarget::SP &target,
+                  const vespalib::string &destination);
+
+        IFlushTarget::SP  _target;
+    };
+    typedef std::map<uint32_t, FlushInfo> FlushMap;
+    typedef HandlerMap<IFlushHandler> FlushHandlerMap;
+    bool                           _closed;
+    const uint32_t                 _maxConcurrent;
+    const uint32_t                 _idleIntervalMS;
+    const bool                     _enableAutoPrune;
+    uint32_t                       _taskId;    
+    FastOS_ThreadPool              _threadPool;
+    IFlushStrategy::SP             _strategy;
+    mutable IFlushStrategy::SP     _priorityStrategy;
+    vespalib::ThreadStackExecutor  _executor;
+    vespalib::Monitor              _monitor;
+    FlushHandlerMap                _handlers;
+    FlushMap                       _flushing;
+    vespalib::Lock                 _strategyLock; // serialize setStrategy calls
+    vespalib::Monitor              _strategyMonitor;
+    std::shared_ptr<flushengine::ITlsStatsFactory> _tlsStatsFactory;
+
+    FlushContext::List getTargetList(bool includeFlushingTargets) const;
+    std::pair<FlushContext::List,bool> getSortedTargetList(vespalib::MonitorGuard &strategyGuard) const;
+    FlushContext::SP initNextFlush(const FlushContext::List &lst);
+    vespalib::string flushNextTarget(const vespalib::string & name);
+    void flushAll(const FlushContext::List &lst);
+    void prune();
+    uint32_t initFlush(const FlushContext &ctx);
+    uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target);
+    void flushDone(const FlushContext &ctx, uint32_t taskId);
+    bool canFlushMore(const vespalib::MonitorGuard & guard) const;
+    bool wait(size_t minimumWaitTimeIfReady);
+    bool isFlushing(const vespalib::MonitorGuard & guard, const vespalib::string & name) const;
+
+    friend class FlushTask;
+    friend class FlushEngineExplorer;
+
+public:
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::unique_ptr<FlushEngine> UP;
+    typedef std::shared_ptr<FlushEngine> SP;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param tlsStatsFactory A factory for creating tls statistics
+     *                        used by strategy to select best flush candiate.
+     * @param strategy   The flushing strategy to use.
+     * @param numThreads The number of worker threads to use.
+     * @param idleInterval The interval between when flushes are checked whne there are no one progressing.
+     * @param enableAutoPrune Indicate if pruning shall be done even if there
+                              are no flushing happening. Turn off for some tests.
+                              Needed for pruning  to be correct if one flush is started
+                              while another is in progress. In that case the pruning
+                              will be too conservative.
+     */
+    FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory>
+                tlsStatsFactory,
+                IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS, bool enableAutoPrune);
+
+    /**
+     * Destructor. Waits for all pending tasks to complete.
+     */
+    ~FlushEngine();
+
+    /**
+     * Observe and reset internal executor stats
+     *
+     * @return executor stats
+     **/
+    vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); }
+
+    /**
+     * Starts the scheduling thread of this manager.
+     *
+     * @return This, to allow chaining.
+     */
+    FlushEngine &start();
+
+    /**
+     * Stops the scheduling thread and. This will prevent any more flush
+     * requests being performed on the attached handlers, allowing you to flush
+     * all pending operations without having to safe-guard against this.
+     *
+     * @return This, to allow chaining.
+     */
+    FlushEngine &close();
+
+    /**
+     * Triggers an immediate flush of all flush targets.
+     * This method is synchronous and thread-safe.
+     */
+    void triggerFlush();
+
+    void
+    kick(void);
+
+    /**
+     * Registers a new flush handler for the given document type. If another
+     * handler was already registered under the same type, this method will
+     * return a pointer to that handler.
+     *
+     * @param docType      The document type to register a handler for.
+     * @param flushHandler The handler to register.
+     * @return The replaced handler, if any.
+     */
+    IFlushHandler::SP
+    putFlushHandler(const DocTypeName &docTypeName,
+                    const IFlushHandler::SP &flushHandler);
+
+    /**
+     * Returns the flush handler for the given document type. If no handler was
+     * registered, this method returns an empty shared pointer.
+     *
+     * @param docType The document type whose handler to return.
+     * @return The registered handler, if any.
+     */
+    IFlushHandler::SP
+    getFlushHandler(const DocTypeName &docTypeName) const;
+
+    /**
+     * Removes and returns the flush handler for the given document type. If no
+     * handler was registered, this method returns an empty shared pointer.
+     *
+     * @param docType The document type whose handler to remove.
+     * @return The removed handler, if any.
+     */
+    IFlushHandler::SP
+    removeFlushHandler(const DocTypeName &docTypeName);
+
+    // Implements FastOS_Runnable.
+    void Run(FastOS_ThreadInterface *thread, void *arg);
+
+    FlushMetaSet getCurrentlyFlushingSet() const;
+
+    void setStrategy(IFlushStrategy::SP strategy);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp
new file mode 100644
index 00000000000..92133c6d656
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.flushengine.flushtargetproxy");
+
+#include "flushtargetproxy.h"
+
+namespace proton {
+
+using searchcorespi::IFlushTarget;
+using searchcorespi::FlushStats;
+
+FlushTargetProxy::FlushTargetProxy(const IFlushTarget::SP &target)
+    : IFlushTarget(target->getName(), target->getType(),
+                   target->getComponent()),
+      _target(target)
+{
+}
+
+FlushTargetProxy::FlushTargetProxy(const IFlushTarget::SP &target,
+                                   const vespalib::string & prefix)
+    : IFlushTarget(prefix + "." + target->getName(), target->getType(),
+                   target->getComponent()),
+      _target(target)
+{
+}
+
+
+IFlushTarget::MemoryGain
+FlushTargetProxy::getApproxMemoryGain() const
+{
+    return _target->getApproxMemoryGain();
+}
+
+
+IFlushTarget::DiskGain
+FlushTargetProxy::getApproxDiskGain() const
+{
+    return _target->getApproxDiskGain();
+}
+
+
+IFlushTarget::SerialNum
+FlushTargetProxy::getFlushedSerialNum() const
+{
+    return _target->getFlushedSerialNum();
+}
+
+
+IFlushTarget::Time
+FlushTargetProxy::getLastFlushTime() const
+{
+    return _target->getLastFlushTime();
+}
+
+
+bool
+FlushTargetProxy::needUrgentFlush() const
+{
+    return _target->needUrgentFlush();
+}
+
+
+IFlushTarget::Task::UP
+FlushTargetProxy::initFlush(SerialNum currentSerial)
+{
+    return _target->initFlush(currentSerial);
+}
+
+
+FlushStats
+FlushTargetProxy::getLastFlushStats() const
+{
+    return _target->getLastFlushStats();
+}
+
+
+uint64_t
+FlushTargetProxy::getApproxBytesToWriteToDisk() const
+{
+    return _target->getApproxBytesToWriteToDisk();
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h
new file mode 100644
index 00000000000..04977c55b42
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+
+namespace vespalib { class Executor; }
+
+namespace proton {
+
+
+/**
+ * Implements a flush target that proxies everything to the given
+ * target.
+ */
+class FlushTargetProxy : public searchcorespi::IFlushTarget
+{
+protected:
+    IFlushTarget::SP    _target;
+
+public:
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param target   The target to decorate.
+     */
+    FlushTargetProxy(const IFlushTarget::SP &target);
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param target   The target to decorate.
+     * @param prefix   The prefix to prepend to the target
+     */
+    FlushTargetProxy(const IFlushTarget::SP &target,
+                     const vespalib::string & prefix);
+    /**
+     * Returns the decorated flush target. This should not be used for anything
+     * but testing, as invoking a method on the returned target beats the
+     * purpose of decorating it.
+     *
+     * @return The decorated flush target.
+     */
+    const IFlushTarget::SP &
+    getFlushTarget() const
+    {
+        return _target;
+    }
+    // Implements IFlushTarget.
+    virtual MemoryGain
+    getApproxMemoryGain() const;
+
+    virtual DiskGain
+    getApproxDiskGain() const;
+
+    virtual SerialNum
+    getFlushedSerialNum() const;
+
+    virtual Time
+    getLastFlushTime() const;
+
+    virtual bool
+    needUrgentFlush() const;
+
+    virtual Task::UP
+    initFlush(SerialNum currentSerial);;
+
+    virtual searchcorespi::FlushStats
+    getLastFlushStats() const;
+
+    virtual uint64_t getApproxBytesToWriteToDisk() const override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp
new file mode 100644
index 00000000000..fc692979753
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.flushengine.flushtask");
+
+#include "flushtask.h"
+
+namespace proton {
+
+FlushTask::FlushTask(uint32_t taskId,
+                     FlushEngine &engine,
+                     const FlushContext::SP &ctx,
+                     search::SerialNum serial)
+    : _taskId(taskId),
+      _engine(engine),
+      _context(ctx),
+      _serial(serial)
+{
+    LOG_ASSERT(_context.get() != NULL);
+}
+
+FlushTask::~FlushTask()
+{
+    _engine.flushDone(*_context, _taskId);
+}
+
+void
+FlushTask::run()
+{
+    searchcorespi::FlushTask::UP task(_context->getTask());
+    search::SerialNum flushSerial(task->getFlushSerial());
+    if (flushSerial != 0) {
+        _context->getHandler()->syncTls(flushSerial);
+    }
+    task->run();
+    task.reset();
+    _context->getHandler()->flushDone(_serial);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h
new file mode 100644
index 00000000000..fd27538ca4b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/flushengine/flushengine.h>
+
+namespace proton {
+
+/**
+ * This class decorates a task returned by initFlush() in IFlushTarget so that
+ * the appropriate callback is invoked on the running FlushEngine.
+ */
+class FlushTask : public boost::noncopyable,
+                  public vespalib::Executor::Task
+{
+private:
+    uint32_t                          _taskId;
+    FlushEngine                      &_engine;
+    FlushContext::SP                  _context;
+    search::SerialNum _serial;
+
+public:
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param taskId The identifier used by IFlushStrategy.
+     * @param engine The running flush engine.
+     * @param ctx    The context of the flush to perform.
+     * @param serial The oldest unflushed serial available in the handler once
+     *               this task has been run.
+     */
+    FlushTask(uint32_t taskId,
+              FlushEngine &engine,
+              const FlushContext::SP &ctx,
+              search::SerialNum serial);
+
+    /**
+     * Destructor. Notifies the engine that the flush is done to prevent the
+     * engine from locking targets because of a glitch.
+     */
+    ~FlushTask();
+
+    // Implements Executor::Task.
+    void run();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/i_tls_stats_factory.h b/searchcore/src/vespa/searchcore/proton/flushengine/i_tls_stats_factory.h
new file mode 100644
index 00000000000..ce7fdf9cf04
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/i_tls_stats_factory.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "tls_stats.h"
+
+namespace proton {
+namespace flushengine {
+
+class TlsStatsMap;
+
+/*
+ * Class used to create statistics for a transaction log server over
+ * multiple domains.
+ */
+class ITlsStatsFactory {
+public:
+    virtual ~ITlsStatsFactory() { }
+    virtual TlsStatsMap create() = 0;
+};
+
+} // namespace proton::flushengine
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h
new file mode 100644
index 00000000000..6912cbfde13
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <string>
+#include <vector>
+
+namespace proton {
+
+using searchcorespi::IFlushTarget;
+
+/**
+ * This class represents a collection of IFlushTarget objects. It is implemented
+ * by DocumentDB.
+ */
+class IFlushHandler {
+private:
+    vespalib::string _name;
+
+public:
+    typedef IFlushTarget::SerialNum SerialNum;
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::shared_ptr<IFlushHandler> SP;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param name The unique name of this handler.
+     */
+    IFlushHandler(const vespalib::string &name)
+        : _name(name)
+    {
+        // empty
+    }
+
+    /**
+     * Virtual destructor required for inheritance.
+     */
+    virtual ~IFlushHandler()
+    {
+        // empty
+    }
+
+    /**
+     * Returns the unique name of this handler.
+     *
+     * @return The name of this.
+     */
+    const vespalib::string &
+    getName() const
+    {
+        return _name;
+    }
+
+    /**
+     * Returns a list of the flush targets that belong to this handler. This
+     * method is called by the flush scheduler thread.
+     *
+     * @return The list of targets.
+     */
+    virtual std::vector<IFlushTarget::SP>
+    getFlushTargets() = 0;
+
+    /**
+     * Returns the current serial number of this handler. This is the head of
+     * the transaction log for the domain of this.
+     *
+     * @return The current serial number.
+     */
+    virtual SerialNum getCurrentSerialNumber() const = 0;
+
+    /**
+     * This method is called after a flush has been completed. All transactions
+     * up to the given serial number can be pruned from the domain of this
+     * handler. This method is called by an arbitrary worker thread.
+     *
+     * @param oldestSerial The oldest transaction that is still in use.
+     */
+    virtual void flushDone(SerialNum oldestSerial) = 0;
+
+    /*
+     * This method is called to sync tls to stable media, up to and
+     * including the given serial number.
+     *
+     * @param syncTo The last serial number that has to be persisted to stable
+     *               media.
+     */
+    virtual void
+    syncTls(SerialNum syncTo) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h
new file mode 100644
index 00000000000..7139c69a778
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/flushengine/iflushhandler.h>
+#include <vespa/searchcore/proton/flushengine/flushcontext.h>
+
+namespace proton {
+
+namespace flushengine { class TlsStatsMap; }
+
+/**
+ * This class represents a strategy used by the FlushEngine to make decisions on
+ * when and what to flush.
+ */
+class IFlushStrategy {
+public:
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::shared_ptr<IFlushStrategy> SP;
+
+    /**
+     * Virtual destructor required for inheritance.
+     */
+    virtual ~IFlushStrategy() { }
+
+    /**
+     * Takes an input of targets that are candidates for flush and returns
+     * a list of targets sorted according to priority strategy.
+     * @param targetList The list of possible flush targets.
+     * @param lastSerial is the last serialnumber known by flushengine.
+     * @return A prioritized list of targets to flush.
+     */
+    virtual FlushContext::List getFlushTargets(const FlushContext::List & targetList,
+                                               const flushengine::TlsStatsMap &
+                                               tlsStatsMap) const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp
new file mode 100644
index 00000000000..d989cb24e88
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.flushengine.prepare_restart_flush_strategy");
+
+#include "prepare_restart_flush_strategy.h"
+#include "flush_target_candidates.h"
+#include "tls_stats_map.h"
+
+namespace proton {
+
+using search::SerialNum;
+
+using Config = PrepareRestartFlushStrategy::Config;
+using FlushContextsMap = std::map<vespalib::string, FlushContext::List>;
+using FlushTargetCandidatesList = std::vector<FlushTargetCandidates::UP>;
+
+PrepareRestartFlushStrategy::Config::Config(double tlsReplayCost_,
+                                            double flushTargetWriteCost_)
+    : tlsReplayCost(tlsReplayCost_),
+      flushTargetWriteCost(flushTargetWriteCost_)
+{
+}
+
+PrepareRestartFlushStrategy::PrepareRestartFlushStrategy(const Config &cfg)
+    : _cfg(cfg)
+{
+}
+
+namespace {
+
+FlushContext::List
+removeGCFlushTargets(const FlushContext::List &flushContexts)
+{
+    FlushContext::List result;
+    for (const auto &flushContext : flushContexts) {
+        if (flushContext->getTarget()->getType() != IFlushTarget::Type::GC) {
+            result.push_back(flushContext);
+        }
+    }
+    return result;
+}
+
+FlushContextsMap
+groupByFlushHandler(const FlushContext::List &flushContexts)
+{
+    FlushContextsMap result;
+    for (const auto &flushContext : flushContexts) {
+        const vespalib::string &handlerName = flushContext->getHandler()->getName();
+        result[handlerName].push_back(flushContext);
+    }
+    return result;
+}
+
+FlushContext::List
+flatten(const FlushContextsMap &flushContextsPerHandler)
+{
+    FlushContext::List result;
+    for (const auto &entry : flushContextsPerHandler) {
+        for (const auto &flushContext : entry.second) {
+            result.push_back(flushContext);
+        }
+    }
+    return result;
+}
+
+void
+sortByOldestFlushedSerialNumber(FlushContext::List &flushContexts)
+{
+    std::sort(flushContexts.begin(), flushContexts.end(),
+            [](const auto &lhs, const auto &rhs) {
+        if (lhs->getTarget()->getFlushedSerialNum() ==
+                rhs->getTarget()->getFlushedSerialNum()) {
+            return lhs->getName() < rhs->getName();
+        }
+        return lhs->getTarget()->getFlushedSerialNum() <
+                rhs->getTarget()->getFlushedSerialNum();
+    });
+}
+
+vespalib::string
+toString(const FlushContext::List &flushContexts)
+{
+    std::ostringstream oss;
+    bool first = true;
+    for (const auto &flushContext : flushContexts) {
+        if (!first) {
+            oss << ",";
+        }
+        oss << "'" << flushContext->getName() << "'";
+        first = false;
+    }
+    return oss.str();
+}
+
+FlushContext::List
+findBestTargetsToFlush(const FlushContext::List &unsortedFlushContexts,
+                       const flushengine::TlsStats &tlsStats,
+                       const Config &cfg)
+{
+    FlushContext::List sortedFlushContexts = unsortedFlushContexts;
+    sortByOldestFlushedSerialNumber(sortedFlushContexts);
+
+    FlushTargetCandidates bestSet(sortedFlushContexts, 0, tlsStats, cfg);
+    for (size_t numCandidates = 1; numCandidates <= sortedFlushContexts.size(); ++numCandidates) {
+        FlushTargetCandidates nextSet(sortedFlushContexts, numCandidates, tlsStats, cfg);
+        LOG(debug, "findBestTargetsToFlush(): Created candidate set: "
+                "flushTargets=[%s], tlsReplayCost=%f, flushTargetsWriteCost=%f, totalCost=%f",
+                toString(nextSet.getCandidates()).c_str(),
+                nextSet.getTlsReplayCost(), nextSet.getFlushTargetsWriteCost(),
+                nextSet.getTotalCost());
+        if (nextSet.getTotalCost() < bestSet.getTotalCost()) {
+            bestSet = nextSet;
+        }
+    }
+    LOG(info, "findBestTargetsToFlush(): Best candidate set: "
+            "flushTargets=[%s], tlsReplayCost=%f, flushTargetsWriteCost=%f, totalCost=%f",
+            toString(bestSet.getCandidates()).c_str(),
+            bestSet.getTlsReplayCost(), bestSet.getFlushTargetsWriteCost(),
+            bestSet.getTotalCost());
+    return bestSet.getCandidates();
+}
+
+FlushContextsMap
+findBestTargetsToFlushPerHandler(const FlushContextsMap &flushContextsPerHandler,
+                                 const Config &cfg,
+                                 const flushengine::TlsStatsMap &tlsStatsMap)
+{
+    FlushContextsMap result;
+    for (const auto &entry : flushContextsPerHandler) {
+        const auto &handlerName = entry.first;
+        const auto &flushContexts = entry.second;
+        const auto &tlsStats = tlsStatsMap.getTlsStats(handlerName);
+        result.insert(std::make_pair(handlerName,
+                findBestTargetsToFlush(flushContexts, tlsStats, cfg)));
+    }
+    return result;
+}
+
+}
+
+FlushContext::List
+PrepareRestartFlushStrategy::getFlushTargets(const FlushContext::List &targetList,
+                                             const flushengine::TlsStatsMap &tlsStatsMap) const
+{
+    return flatten(findBestTargetsToFlushPerHandler(
+            groupByFlushHandler(removeGCFlushTargets(targetList)),
+            _cfg, tlsStatsMap));
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h
new file mode 100644
index 00000000000..f3925a802ca
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "iflushstrategy.h"
+#include <vespa/vespalib/stllike/string.h>
+#include <map>
+
+namespace proton {
+
+/**
+ * Flush strategy that is used to find flush targets to be flushed before a restart.
+ *
+ * For each flush handler, flush targets are chosen such that the cost of flushing them +
+ * the cost of replaying the transaction log after replay is as low as possible.
+ *
+ * The cost of replaying the transaction log is: the number of bytes to replay * a replay speed factor.
+ * The cost of flushing a flush target is: the number of bytes to write * a write speed factor.
+ */
+class PrepareRestartFlushStrategy : public IFlushStrategy
+{
+public:
+    struct Config
+    {
+        double tlsReplayCost;
+        double flushTargetWriteCost;
+        Config(double tlsReplayCost_, double flushTargetWriteCost_);
+    };
+
+private:
+    Config _cfg;
+
+public:
+    PrepareRestartFlushStrategy(const Config &cfg);
+
+    virtual FlushContext::List getFlushTargets(const FlushContext::List &targetList,
+                                               const flushengine::TlsStatsMap &
+                                               tlsStatsMap) const override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp
new file mode 100644
index 00000000000..15ddf3d1e08
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.flushengine.threadedflushtarget");
+
+#include "threadedflushtarget.h"
+#include <vespa/vespalib/util/executor.h>
+#include <vespa/searchcore/proton/server/igetserialnum.h>
+#include <vespa/searchlib/common/lambdatask.h>
+#include <future>
+
+using searchcorespi::IFlushTarget;
+using searchcorespi::FlushStats;
+using search::makeLambdaTask;
+
+namespace proton {
+
+ThreadedFlushTarget::ThreadedFlushTarget(vespalib::Executor &executor,
+                                         const IGetSerialNum &getSerialNum,
+                                         const IFlushTarget::SP &target)
+    : FlushTargetProxy(target),
+      _executor(executor),
+      _getSerialNum(getSerialNum)
+{
+}
+
+ThreadedFlushTarget::ThreadedFlushTarget(vespalib::Executor &executor,
+                                         const IGetSerialNum &getSerialNum,
+                                         const IFlushTarget::SP &target,
+                                         const vespalib::string & prefix)
+    : FlushTargetProxy(target, prefix),
+      _executor(executor),
+      _getSerialNum(getSerialNum)
+{
+}
+
+namespace {
+IFlushTarget::Task::UP
+callInitFlush(IFlushTarget *target, IFlushTarget::SerialNum serial,
+              const IGetSerialNum *getSerialNum) {
+    // Serial number from flush engine might have become stale, obtain
+    // a fresh serial number now.
+    search::SerialNum freshSerial = getSerialNum->getSerialNum();
+    assert(freshSerial >= serial);
+    return target->initFlush(freshSerial);
+}
+}  // namespace
+
+IFlushTarget::Task::UP
+ThreadedFlushTarget::initFlush(SerialNum currentSerial)
+{
+    std::promise<Task::UP> promise;
+    std::future<Task::UP> future = promise.get_future();
+    _executor.execute(makeLambdaTask([&]()
+                      { promise.set_value(callInitFlush(_target.get(),
+                                                        currentSerial,
+                                                        &_getSerialNum)); }));
+    return future.get();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.h b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.h
new file mode 100644
index 00000000000..f26244e1264
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.h
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "flushtargetproxy.h"
+
+namespace vespalib { class Executor; }
+
+namespace proton {
+
+using searchcorespi::FlushStats;
+using searchcorespi::IFlushTarget;
+
+class IGetSerialNum;
+
+/**
+ * Implements a flush target that runs initFlush() as a task in the given
+ * executor. This is used by the DocumentDB to ensure that initFlush() in the
+ * underlying flush targets are run in the updater thread.
+ */
+class ThreadedFlushTarget : public FlushTargetProxy
+{
+private:
+    vespalib::Executor &_executor;
+    const IGetSerialNum &_getSerialNum;
+
+public:
+    /**
+     * Constructs a new instance of this class. If the argument executor is
+     * the same as the one calling initFlush() on this object, make sure
+     * that it has more than 1 thread to avoid a deadlock.
+     *
+     * @param executor The executor to submit the task to.
+     * @param target   The target to decorate.
+     */
+    ThreadedFlushTarget(vespalib::Executor &executor,
+                        const IGetSerialNum &getSerialNum,
+                        const IFlushTarget::SP &target);
+
+    /**
+     * Constructs a new instance of this class. If the argument executor is
+     * the same as the one calling initFlush() on this object, make sure
+     * that it has more than 1 thread to avoid a deadlock.
+     *
+     * @param executor The executor to submit the task to.
+     * @param target   The target to decorate.
+     * @param prefix   The prefix to prepend to the target
+     */
+    ThreadedFlushTarget(vespalib::Executor &executor,
+                        const IGetSerialNum &getSerialNum,
+                        const IFlushTarget::SP &target,
+                        const vespalib::string & prefix);
+
+    // Implements IFlushTarget.
+    virtual Task::UP
+    initFlush(SerialNum currentSerial);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h
new file mode 100644
index 00000000000..bfa31086c42
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+namespace proton {
+namespace flushengine {
+
+/*
+ * Class representing statistics for a transaction log server domain used to
+ * adjust flush strategy.
+ */
+class TlsStats
+{
+    uint64_t _numBytes;
+    uint64_t _firstSerial;
+    uint64_t _lastSerial;
+
+public:
+    TlsStats()
+        : _numBytes(0),
+          _firstSerial(0),
+          _lastSerial(0)
+    {
+    }
+    TlsStats(uint64_t numBytes, uint64_t firstSerial, uint64_t lastSerial)
+        : _numBytes(numBytes),
+          _firstSerial(firstSerial),
+          _lastSerial(lastSerial)
+    {
+    }
+
+    uint64_t getNumBytes() const    { return _numBytes; }
+    uint64_t getFirstSerial() const { return _firstSerial; }
+    uint64_t getLastSerial() const  { return _lastSerial; }
+};
+
+} // namespace proton::flushengine
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.cpp
new file mode 100644
index 00000000000..6819849e959
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.cpp
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "tls_stats_factory.h"
+#include "tls_stats_map.h"
+#include <vespa/searchlib/transactionlog/translogserver.h>
+
+using search::transactionlog::DomainInfo;
+using search::transactionlog::DomainStats;
+
+namespace proton {
+namespace flushengine {
+
+TlsStatsFactory::TlsStatsFactory(TransLogServer::SP tls)
+    : _tls(tls)
+{
+}
+
+TlsStatsFactory::~TlsStatsFactory()
+{
+}
+
+TlsStatsMap
+TlsStatsFactory::create()
+{
+    DomainStats stats  = _tls->getDomainStats();
+    TlsStatsMap::Map map;
+    for (auto &itr : stats) {
+        const DomainInfo &di = itr.second;
+        map.insert(std::make_pair(itr.first,
+                                  TlsStats(di.byteSize,
+                                           di.range.from(),
+                                           di.range.to())));
+    }
+    return TlsStatsMap(std::move(map));
+}
+
+} // namespace proton::flushengine
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.h b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.h
new file mode 100644
index 00000000000..d580d58b964
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_tls_stats_factory.h"
+
+namespace search { namespace transactionlog { class TransLogServer; } }
+namespace proton {
+namespace flushengine {
+
+/*
+ * Class used to create statistics for a transaction log server over
+ * multiple domains.
+ */
+class TlsStatsFactory : public ITlsStatsFactory
+{
+    using TransLogServer = search::transactionlog::TransLogServer;
+    std::shared_ptr<TransLogServer> _tls;
+public:
+    TlsStatsFactory(std::shared_ptr<TransLogServer> tls);
+    virtual ~TlsStatsFactory();
+    virtual TlsStatsMap create() override;
+};
+
+} // namespace proton::flushengine
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.h b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.h
new file mode 100644
index 00000000000..b03da7407a4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/stllike/hash_map.h>
+#include "tls_stats.h"
+
+namespace proton {
+namespace flushengine {
+
+/*
+ * Class representing statistics for a transaction log server over multiple
+ * domains.
+ */
+class TlsStatsMap
+{
+public:
+    using Map = vespalib::hash_map<vespalib::string, TlsStats>;
+private:
+    Map _map;
+
+public:
+    TlsStatsMap(Map &&map)
+        : _map(std::move(map))
+    {
+    }
+
+    const TlsStats &getTlsStats(const vespalib::string &domain) const {
+        auto itr = _map.find(domain);
+        if (itr != _map.end()) {
+            return itr->second;
+        }
+        abort();
+    }
+};
+
+} // namespace proton::flushengine
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/fusion/.gitignore b/searchcore/src/vespa/searchcore/proton/fusion/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/fusion/.gitignore
diff --git a/searchcore/src/vespa/searchcore/proton/index/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/index/CMakeLists.txt
new file mode 100644
index 00000000000..25175e38388
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_index STATIC
+    SOURCES
+    diskindexwrapper.cpp
+    index_manager_initializer.cpp
+    index_writer.cpp
+    indexmanager.cpp
+    memoryindexwrapper.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/index/OWNERS b/searchcore/src/vespa/searchcore/proton/index/OWNERS
new file mode 100644
index 00000000000..64735d11d93
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/OWNERS
@@ -0,0 +1 @@
+tegge
diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp
new file mode 100644
index 00000000000..f7a16bdcc81
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.index.diskindexwrapper");
+
+#include "diskindexwrapper.h"
+
+using search::TuneFileSearch;
+
+namespace proton {
+
+DiskIndexWrapper::DiskIndexWrapper(const vespalib::string &indexDir,
+                                   const TuneFileSearch &tuneFileSearch,
+                                   size_t cacheSize)
+    : _index(indexDir, cacheSize)
+{
+    bool setupIndexOk = _index.setup(tuneFileSearch);
+    assert(setupIndexOk);
+    (void) setupIndexOk;
+}
+
+DiskIndexWrapper::DiskIndexWrapper(const DiskIndexWrapper &oldIndex,
+                                   const TuneFileSearch &tuneFileSearch,
+                                   size_t cacheSize)
+    : _index(oldIndex._index.getIndexDir(), cacheSize)
+{
+    bool setupIndexOk = _index.setup(tuneFileSearch, oldIndex._index);
+    assert(setupIndexOk);
+    (void) setupIndexOk;
+}
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h
new file mode 100644
index 00000000000..9deef043f6e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/index/idiskindex.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchlib/diskindex/diskindex.h>
+
+namespace proton {
+
+class DiskIndexWrapper : public searchcorespi::index::IDiskIndex {
+private:
+    search::diskindex::DiskIndex _index;
+
+public:
+    DiskIndexWrapper(const vespalib::string &indexDir,
+                     const search::TuneFileSearch &tuneFileSearch,
+                     size_t cacheSize);
+
+    DiskIndexWrapper(const DiskIndexWrapper &oldIndex,
+                     const search::TuneFileSearch &tuneFileSearch,
+                     size_t cacheSize);
+
+    /**
+     * Implements searchcorespi::IndexSearchable
+     */
+    virtual search::queryeval::Blueprint::UP
+    createBlueprint(const search::queryeval::IRequestContext & requestContext,
+                    const search::queryeval::FieldSpec &field,
+                    const search::query::Node &term,
+                    const search::attribute::IAttributeContext &)
+    {
+        return _index.createBlueprint(requestContext, field, term);
+    }
+    virtual search::queryeval::Blueprint::UP
+    createBlueprint(const search::queryeval::IRequestContext & requestContext,
+                    const search::queryeval::FieldSpecList &fields,
+                    const search::query::Node &term,
+                    const search::attribute::IAttributeContext &)
+    {
+        return _index.createBlueprint(requestContext, fields, term);
+    }
+    virtual search::SearchableStats getSearchableStats() const {
+        return search::SearchableStats()
+            .sizeOnDisk(_index.getSize());
+    }
+
+    /**
+     * Implements proton::IDiskIndex
+     */
+    virtual const vespalib::string &getIndexDir() const { return _index.getIndexDir(); }
+    virtual const search::index::Schema &getSchema() const { return _index.getSchema(); }
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h
new file mode 100644
index 00000000000..21dc822bb2a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/searchcorespi/index/iindexmanager.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/searchlib/common/serialnum.h>
+
+namespace proton {
+
+using searchcorespi::IIndexManager;
+
+/**
+ * Interface for an index writer that handles writes in form of put and remove
+ * to an underlying memory index.
+ **/
+class IIndexWriter {
+public:
+    typedef std::unique_ptr<IIndexWriter> UP;
+    typedef std::shared_ptr<IIndexWriter> SP;
+    using OnWriteDoneType = searchcorespi::IIndexManager::OnWriteDoneType;
+
+    virtual ~IIndexWriter() {}
+
+    virtual const IIndexManager::SP &getIndexManager() const = 0;
+
+    // feed interface
+    virtual void put(search::SerialNum serialNum,
+                     const document::Document &doc,
+                     const search::DocumentIdT lid) = 0;
+    virtual void remove(search::SerialNum serialNum,
+                        const search::DocumentIdT lid) = 0;
+    virtual void commit(search::SerialNum serialNum,
+                        OnWriteDoneType onWriteDone) = 0;
+
+    virtual void
+    heartBeat(search::SerialNum serialNum) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
new file mode 100644
index 00000000000..a183a77dafe
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.index.indexmanagerinitializer");
+#include "index_manager_initializer.h"
+#include <vespa/vespalib/io/fileutil.h>
+
+namespace proton
+{
+
+IndexManagerInitializer::
+IndexManagerInitializer(const vespalib::string &baseDir,
+                        double diskIndexWarmupTime,
+                        size_t maxFlushed,
+                        size_t cacheSize,
+                        const search::index::Schema &schema,
+                        const search::index::Schema &fusionSchema,
+                        searchcorespi::IIndexManager::Reconfigurer &
+                        reconfigurer,
+                        searchcorespi::index::IThreadingService &
+                        threadingService,
+                        vespalib::ThreadExecutor & warmupExecutor,
+                        const search::TuneFileIndexManager &
+                        tuneFileIndexManager,
+                        const search::TuneFileAttributes &tuneFileAttributes,
+                        const search::common::FileHeaderContext &
+                        fileHeaderContext,
+                        std::shared_ptr<searchcorespi::IIndexManager::SP> indexManager)
+    : _baseDir(baseDir),
+      _diskIndexWarmupTime(diskIndexWarmupTime),
+      _maxFlushed(maxFlushed),
+      _cacheSize(cacheSize),
+      _schema(schema),
+      _fusionSchema(fusionSchema),
+      _reconfigurer(reconfigurer),
+      _threadingService(threadingService),
+      _warmupExecutor(warmupExecutor),
+      _tuneFileIndexManager(tuneFileIndexManager),
+      _tuneFileAttributes(tuneFileAttributes),
+      _fileHeaderContext(fileHeaderContext),
+      _indexManager(indexManager)
+{
+}
+
+
+void
+IndexManagerInitializer::run()
+{
+    LOG(debug, "About to create proton::IndexManager with %u index field(s)",
+        _schema.getNumIndexFields());
+    vespalib::mkdir(_baseDir, false);
+    *_indexManager = std::make_shared<proton::IndexManager>
+                    (_baseDir,
+                     _diskIndexWarmupTime,
+                     _maxFlushed,
+                     _cacheSize,
+                     _schema,
+                     _fusionSchema,
+                     _reconfigurer,
+                     _threadingService,
+                     _warmupExecutor,
+                     _tuneFileIndexManager,
+                     _tuneFileAttributes,
+                     _fileHeaderContext);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h
new file mode 100644
index 00000000000..45629860229
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "indexmanager.h"
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+
+namespace proton
+{
+
+
+/*
+ * Class representing an initializer task for constructing index manager
+ * during proton startup.
+ */
+class IndexManagerInitializer :  public initializer::InitializerTask
+{
+    const vespalib::string                      _baseDir;
+    double                                      _diskIndexWarmupTime;
+    size_t                                      _maxFlushed;
+    size_t                                      _cacheSize;
+    const search::index::Schema                 _schema;
+    const search::index::Schema                 _fusionSchema;
+    searchcorespi::IIndexManager::Reconfigurer &_reconfigurer;
+    searchcorespi::index::IThreadingService    &_threadingService;
+    vespalib::ThreadExecutor                   &_warmupExecutor;
+    const search::TuneFileIndexManager          _tuneFileIndexManager;
+    const search::TuneFileAttributes            _tuneFileAttributes;
+    const search::common::FileHeaderContext    &_fileHeaderContext;
+    std::shared_ptr<searchcorespi::IIndexManager::SP> _indexManager;
+public:
+    // Note: lifetime of indexManager must be handled by caller.
+    IndexManagerInitializer(const vespalib::string &baseDir,
+                            double diskIndexWarmupTime,
+                            size_t maxFlushed,
+                            size_t cacheSize,
+                            const search::index::Schema &schema,
+                            const search::index::Schema &fusionSchema,
+                            searchcorespi::IIndexManager::Reconfigurer &
+                            reconfigurer,
+                            searchcorespi::index::IThreadingService &
+                            threadingService,
+                            vespalib::ThreadExecutor & warmupExecutor,
+                            const search::TuneFileIndexManager &
+                            tuneFileIndexManager,
+                            const search::TuneFileAttributes &
+                            tuneFileAttributes,
+                            const search::common::FileHeaderContext &
+                            fileHeaderContext,
+                            std::shared_ptr<searchcorespi::IIndexManager::SP> indexManager);
+    virtual void run() override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp
new file mode 100644
index 00000000000..b3b5e470301
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.indexadapter");
+
+#include "index_writer.h"
+#include <vespa/document/fieldvalue/document.h>
+
+using document::Document;
+using document::FieldValue;
+
+namespace proton {
+
+IndexWriter::IndexWriter(const IIndexManager::SP &mgr)
+    : _mgr(mgr)
+{
+}
+
+bool
+IndexWriter::ignoreOperation(search::SerialNum serialNum) const {
+    return (serialNum <= _mgr->getFlushedSerialNum());
+}
+
+void
+IndexWriter::put(search::SerialNum serialNum,
+                 const document::Document &doc,
+                 const search::DocumentIdT lid)
+{
+    if (ignoreOperation(serialNum)) {
+        return;
+    }
+    ns_log::Logger::LogLevel level = getDebugLevel(lid, doc.getId());
+    if (LOG_WOULD_VLOG(level)) {
+        vespalib::string s1(doc.toString(true));
+        VLOG(level, "Handle put: serial(%" PRIu64 "), docId(%s), lid(%u), document(sz=%ld)",
+            serialNum, doc.getId().toString().c_str(), lid, s1.size());
+        const size_t chunksize(30000);
+        for (size_t accum(0); accum < s1.size(); accum += chunksize) {
+            VLOG(level, "Handle put continued...: serial(%" PRIu64 "), docId(%s), lid(%u), document(sz=%ld{%ld, %ld}) {\n%.30000s\n}",
+                serialNum, doc.getId().toString().c_str()+accum, lid, s1.size(), accum, std::min(accum+chunksize, s1.size()), s1.c_str());
+        }
+    }
+    _mgr->putDocument(lid, doc, serialNum);
+}
+
+void
+IndexWriter::remove(search::SerialNum serialNum,
+                    const search::DocumentIdT lid)
+{
+    if (serialNum <= _mgr->getFlushedSerialNum()) {
+        return;
+    }
+    VLOG(getDebugLevel(lid, NULL), "Handle remove: serial(%" PRIu64 "), lid(%u)",
+        serialNum, lid);
+    _mgr->removeDocument(lid, serialNum);
+}
+
+void
+IndexWriter::commit(search::SerialNum serialNum, OnWriteDoneType onWriteDone)
+{
+    if (serialNum <= _mgr->getFlushedSerialNum()) {
+        return;
+    }
+    _mgr->commit(serialNum, onWriteDone);
+}
+
+void
+IndexWriter::heartBeat(search::SerialNum serialNum)
+{
+    _mgr->heartBeat(serialNum);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.h b/searchcore/src/vespa/searchcore/proton/index/index_writer.h
new file mode 100644
index 00000000000..f4adb6aa5d0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_index_writer.h"
+#include <vespa/searchcore/proton/common/feeddebugger.h>
+
+namespace proton {
+
+class IndexWriter : public IIndexWriter,
+                    private FeedDebugger {
+private:
+    IIndexManager::SP _mgr;
+
+    bool ignoreOperation(search::SerialNum serialNum) const;
+
+public:
+    IndexWriter(const IIndexManager::SP &mgr);
+
+    /**
+     * Implements IIndexWriter.
+     */
+    virtual const IIndexManager::SP & getIndexManager() const { return _mgr; }
+
+    virtual void put(search::SerialNum serialNum,
+                     const document::Document &doc,
+                     const search::DocumentIdT lid);
+    virtual void remove(search::SerialNum serialNum,
+                        const search::DocumentIdT lid);
+    virtual void commit(search::SerialNum serialNum,
+                        OnWriteDoneType onWriteDone) override;
+
+    virtual void
+    heartBeat(search::SerialNum serialNum);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
new file mode 100644
index 00000000000..acb5415a129
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.index.indexmanager");
+
+#include "indexmanager.h"
+#include "diskindexwrapper.h"
+#include "memoryindexwrapper.h"
+#include <vespa/searchlib/common/serialnumfileheadercontext.h>
+#include <vespa/searchlib/diskindex/fusion.h>
+
+using search::diskindex::Fusion;
+using search::common::FileHeaderContext;
+using search::common::SerialNumFileHeaderContext;
+using search::index::Schema;
+using search::index::SchemaUtil;
+using search::TuneFileIndexing;
+using search::TuneFileIndexManager;
+using search::TuneFileSearch;
+using searchcorespi::index::IDiskIndex;
+using search::diskindex::SelectorArray;
+using searchcorespi::index::IndexMaintainerConfig;
+using searchcorespi::index::IndexMaintainerContext;
+using searchcorespi::index::IMemoryIndex;
+using searchcorespi::index::IThreadingService;
+
+namespace proton {
+
+IndexManager::MaintainerOperations::MaintainerOperations(const FileHeaderContext &fileHeaderContext,
+                                                         const TuneFileIndexManager &tuneFileIndexManager,
+                                                         size_t cacheSize,
+                                                         searchcorespi::index::
+                                                         IThreadingService &
+                                                         threadingService)
+    : _cacheSize(cacheSize),
+      _fileHeaderContext(fileHeaderContext),
+      _tuneFileIndexing(tuneFileIndexManager._indexing),
+      _tuneFileSearch(tuneFileIndexManager._search),
+      _threadingService(threadingService)
+{
+}
+
+IMemoryIndex::SP
+IndexManager::MaintainerOperations::createMemoryIndex(const Schema &schema)
+{
+    return IMemoryIndex::SP(new MemoryIndexWrapper(schema,
+                                                   _fileHeaderContext,
+                                                   _tuneFileIndexing,
+                                                   _threadingService));
+}
+
+IDiskIndex::SP
+IndexManager::MaintainerOperations::loadDiskIndex(const vespalib::string &indexDir)
+{
+    return IDiskIndex::SP(new DiskIndexWrapper(indexDir, _tuneFileSearch, _cacheSize));
+}
+
+IDiskIndex::SP
+IndexManager::MaintainerOperations::reloadDiskIndex(const IDiskIndex &oldIndex)
+{
+    return IDiskIndex::SP(new DiskIndexWrapper(dynamic_cast<const DiskIndexWrapper &>(oldIndex),
+                                               _tuneFileSearch, _cacheSize));
+}
+
+bool
+IndexManager::MaintainerOperations::runFusion(const Schema &schema,
+                                              const vespalib::string &outputDir,
+                                              const std::vector<vespalib::string> &sources,
+                                              const SelectorArray &selectorArray,
+                                              SerialNum serialNum)
+{
+    SerialNumFileHeaderContext fileHeaderContext(_fileHeaderContext,
+                                                 serialNum);
+    const bool dynamic_k_doc_pos_occ_format = false;
+    return Fusion::merge(schema, outputDir, sources, selectorArray,
+                         dynamic_k_doc_pos_occ_format,
+                         _tuneFileIndexing, fileHeaderContext);
+}
+
+
+IndexManager::IndexManager(const vespalib::string &baseDir,
+                           const double warmupTime,
+                           const size_t maxFlushed,
+                           const size_t cacheSize,
+                           const Schema &schema,
+                           const Schema &fusionSchema,
+                           Reconfigurer &reconfigurer,
+                           IThreadingService &threadingService,
+                           vespalib::ThreadExecutor & warmupExecutor,
+                           const search::TuneFileIndexManager &tuneFileIndexManager,
+                           const search::TuneFileAttributes &tuneFileAttributes,
+                           const search::common::FileHeaderContext &fileHeaderContext) :
+    _operations(fileHeaderContext, tuneFileIndexManager, cacheSize,
+                threadingService),
+    _maintainer(IndexMaintainerConfig(baseDir,
+                                      warmupTime,
+                                      maxFlushed,
+                                      schema,
+                                      fusionSchema,
+                                      tuneFileAttributes),
+                IndexMaintainerContext(threadingService,
+                                       reconfigurer,
+                                       fileHeaderContext,
+                                       warmupExecutor),
+                _operations)
+{
+}
+
+IndexManager::~IndexManager()
+{
+}
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
new file mode 100644
index 00000000000..78e74bf0b80
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
@@ -0,0 +1,125 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/index/iindexmaintaineroperations.h>
+#include <vespa/searchcorespi/index/iindexmanager.h>
+#include <vespa/searchcorespi/index/indexmaintainer.h>
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+
+namespace proton {
+
+using searchcorespi::IFlushTarget;
+using searchcorespi::IIndexManager;
+
+/**
+ * The IndexManager provides a holistic view of a set of disk and
+ * memory indexes. It allows updating the active index, enables search
+ * across all indexes, and manages the set of indexes through flushing
+ * of memory indexes and fusion of disk indexes.
+ */
+class IndexManager : public boost::noncopyable, public IIndexManager
+{
+public:
+    class MaintainerOperations : public searchcorespi::index::IIndexMaintainerOperations {
+    private:
+        const size_t _cacheSize;
+        const search::common::FileHeaderContext &_fileHeaderContext;
+        const search::TuneFileIndexing _tuneFileIndexing;
+        const search::TuneFileSearch _tuneFileSearch;
+        searchcorespi::index::IThreadingService &_threadingService;
+
+    public:
+        MaintainerOperations(const search::common::FileHeaderContext &fileHeaderContext,
+                             const search::TuneFileIndexManager &tuneFileIndexManager,
+                             size_t cacheSize,
+                             searchcorespi::index::IThreadingService &
+                             threadingService);
+
+        virtual searchcorespi::index::IMemoryIndex::SP
+            createMemoryIndex(const search::index::Schema &schema);
+        virtual searchcorespi::index::IDiskIndex::SP
+            loadDiskIndex(const vespalib::string &indexDir);
+        virtual searchcorespi::index::IDiskIndex::SP
+            reloadDiskIndex(const searchcorespi::index::IDiskIndex &oldIndex);
+        virtual bool runFusion(const search::index::Schema &schema,
+                               const vespalib::string &outputDir,
+                               const std::vector<vespalib::string> &sources,
+                               const search::diskindex::SelectorArray &docIdSelector,
+                               search::SerialNum lastSerialNum);
+    };
+
+private:
+    MaintainerOperations     _operations;
+    searchcorespi::index::IndexMaintainer _maintainer;
+
+public:
+    IndexManager(const vespalib::string &baseDir,
+                 double diskIndexWarmupTime,
+                 size_t maxFlushed,
+                 size_t cacheSize,
+                 const Schema &schema,
+                 const Schema &fusionSchema,
+                 Reconfigurer &reconfigurer,
+                 searchcorespi::index::IThreadingService &threadingService,
+                 vespalib::ThreadExecutor & warmupExecutor,
+                 const search::TuneFileIndexManager &tuneFileIndexManager,
+                 const search::TuneFileAttributes &tuneFileAttributes,
+                 const search::common::FileHeaderContext &fileHeaderContext);
+    ~IndexManager();
+
+    searchcorespi::index::IndexMaintainer &getMaintainer() {
+        return _maintainer;
+    }
+
+    /**
+     * Implements searchcorespi::IIndexManager
+     **/
+    virtual void putDocument(uint32_t lid, const Document &doc,
+                             SerialNum serialNum) override {
+        _maintainer.putDocument(lid, doc, serialNum);
+    }
+
+    virtual void removeDocument(uint32_t lid, SerialNum serialNum) override {
+        _maintainer.removeDocument(lid, serialNum);
+    }
+
+    virtual void commit(SerialNum serialNum,
+                        OnWriteDoneType onWriteDone) override {
+        _maintainer.commit(serialNum, onWriteDone);
+    }
+
+    virtual void heartBeat(SerialNum serialNum) {
+        _maintainer.heartBeat(serialNum);
+    }
+
+    virtual SerialNum getCurrentSerialNum() const {
+        return _maintainer.getCurrentSerialNum();
+    }
+
+    virtual SerialNum getFlushedSerialNum() const {
+        return _maintainer.getFlushedSerialNum();
+    }
+
+    virtual searchcorespi::IndexSearchable::SP getSearchable() const {
+        return _maintainer.getSearchable();
+    }
+
+    virtual search::SearchableStats getSearchableStats() const {
+        return _maintainer.getSearchableStats();
+    }
+
+    virtual IFlushTarget::List getFlushTargets() {
+        return _maintainer.getFlushTargets();
+    }
+
+    virtual void setSchema(const Schema &schema, const Schema &fusionSchema) {
+        _maintainer.setSchema(schema, fusionSchema);
+    }
+
+    virtual void wipeHistory(SerialNum wipeSerial, const Schema &historyFields) {
+        _maintainer.wipeHistory(wipeSerial, historyFields);
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp
new file mode 100644
index 00000000000..c1535082407
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.index.memoryindexwrapper");
+
+#include "memoryindexwrapper.h"
+#include <vespa/searchlib/common/serialnumfileheadercontext.h>
+#include <vespa/searchlib/diskindex/indexbuilder.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+using search::TuneFileIndexing;
+using search::common::FileHeaderContext;
+using search::common::SerialNumFileHeaderContext;
+using search::index::Schema;
+using search::diskindex::IndexBuilder;
+using search::SerialNum;
+using vespalib::IllegalStateException;
+
+namespace proton {
+
+MemoryIndexWrapper::MemoryIndexWrapper(const search::index::Schema &schema,
+                                       const search::common::FileHeaderContext &fileHeaderContext,
+                                       const TuneFileIndexing &tuneFileIndexing,
+                                       searchcorespi::index::IThreadingService &
+                                        threadingService)
+    : _index(schema, threadingService.indexFieldInverter(),
+             threadingService.indexFieldWriter()),
+      _fileHeaderContext(fileHeaderContext),
+      _tuneFileIndexing(tuneFileIndexing)
+{
+}
+
+void
+MemoryIndexWrapper::flushToDisk(const vespalib::string &flushDir,
+                                uint32_t docIdLimit,
+                                SerialNum serialNum)
+{
+    const uint64_t numWords = _index.getNumWords();
+    _index.freeze(); // TODO(geirst): is this needed anymore?
+    IndexBuilder indexBuilder(_index.getSchema());
+    indexBuilder.setPrefix(flushDir);
+    SerialNumFileHeaderContext fileHeaderContext(_fileHeaderContext,
+                                                 serialNum);
+    indexBuilder.open(docIdLimit, numWords, _tuneFileIndexing, fileHeaderContext);
+    _index.dump(indexBuilder);
+    indexBuilder.close();
+}
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h
new file mode 100644
index 00000000000..5a058326db2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/memoryindex/memoryindex.h>
+#include <vespa/searchcorespi/index/imemoryindex.h>
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchlib/common/fileheadercontext.h>
+
+namespace proton {
+
+/**
+ * Implementation of proton::IMemoryIndex by using search::memoryindex::MemoryIndex
+ * as internal memory index.
+ */
+class MemoryIndexWrapper : public searchcorespi::index::IMemoryIndex {
+private:
+    search::memoryindex::MemoryIndex _index;
+    const search::common::FileHeaderContext &_fileHeaderContext;
+    const search::TuneFileIndexing _tuneFileIndexing;
+
+public:
+    MemoryIndexWrapper(const search::index::Schema &schema,
+                       const search::common::FileHeaderContext &fileHeaderContext,
+                       const search::TuneFileIndexing &tuneFileIndexing,
+                       searchcorespi::index::IThreadingService &
+                       threadingService);
+
+    /**
+     * Implements searchcorespi::IndexSearchable
+     */
+    virtual search::queryeval::Blueprint::UP
+    createBlueprint(const search::queryeval::IRequestContext & requestContext,
+                    const search::queryeval::FieldSpec &field,
+                    const search::query::Node &term,
+                    const search::attribute::IAttributeContext &) override
+    {
+        return _index.createBlueprint(requestContext, field, term);
+    }
+    virtual search::queryeval::Blueprint::UP
+    createBlueprint(const search::queryeval::IRequestContext & requestContext,
+                    const search::queryeval::FieldSpecList &fields,
+                    const search::query::Node &term,
+                    const search::attribute::IAttributeContext &) override
+    {
+        return _index.createBlueprint(requestContext, fields, term);
+    }
+    virtual search::SearchableStats getSearchableStats() const override {
+        return search::SearchableStats()
+            .memoryUsage(getMemoryUsage().allocatedBytes())
+            .docsInMemory(_index.getNumDocs())
+            .sizeOnDisk(0);
+    }
+
+    /**
+     * Implements proton::IMemoryIndex
+     */
+    virtual bool hasReceivedDocumentInsert() const override {
+        return _index.getDocIdLimit() > 1u;
+    }
+    virtual search::index::Schema::SP getWipeTimeSchema() const override {
+        return _index.getWipeTimeSchema();
+    }
+    virtual search::MemoryUsage getMemoryUsage() const override {
+        return _index.getMemoryUsage();
+    }
+    virtual void insertDocument(uint32_t lid, const document::Document &doc) override {
+        _index.insertDocument(lid, doc);
+    }
+    virtual void removeDocument(uint32_t lid) override {
+        _index.removeDocument(lid);
+    }
+    uint64_t getStaticMemoryFootprint() const override {
+        return _index.getStaticMemoryFootprint();
+    }
+    virtual void commit(OnWriteDoneType onWriteDone) override {
+        _index.commit(onWriteDone);
+    }
+    virtual void wipeHistory(const search::index::Schema &schema)  override{
+        _index.wipeHistory(schema);
+    }
+    virtual void flushToDisk(const vespalib::string &flushDir,
+                             uint32_t docIdLimit,
+                             search::SerialNum serialNum) override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/initializer/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/initializer/CMakeLists.txt
new file mode 100644
index 00000000000..5511819c65f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/initializer/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_initializer STATIC
+    SOURCES
+    initializer_task.cpp
+    task_runner.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp
new file mode 100644
index 00000000000..041b4546e70
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "initializer_task.h"
+
+namespace proton {
+
+namespace initializer {
+
+InitializerTask::InitializerTask()
+    : _state(State::BLOCKED),
+      _dependencies()
+{
+}
+
+
+InitializerTask::~InitializerTask()
+{
+}
+
+
+void
+InitializerTask::addDependency(SP dependency)
+{
+    _dependencies.emplace_back(std::move(dependency));
+}
+
+
+} // namespace proton::initializer
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h
new file mode 100644
index 00000000000..e72a71f7ca9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+namespace proton {
+
+namespace initializer {
+
+/*
+ * Class representign an initializer task, used to load a data
+ * structure from disk durign proton startup.
+ */
+class InitializerTask {
+public:
+    using SP = std::shared_ptr<InitializerTask>;
+    using List = std::vector<SP>;
+    enum class State {
+        BLOCKED,
+        RUNNING,
+        DONE
+    };
+private:
+    State           _state;
+    List            _dependencies;
+public:
+    InitializerTask();
+
+    virtual ~InitializerTask();
+
+    State getState() const { return _state; }
+
+    const List &getDependencies() const { return _dependencies; }
+
+    void setRunning() { _state = State::RUNNING; }
+
+    void setDone() { _state = State::DONE; }
+
+    void addDependency(SP dependency);
+
+    virtual void run() = 0;
+};
+
+} // namespace proton::initializer
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp
new file mode 100644
index 00000000000..c653dd30b9e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp
@@ -0,0 +1,133 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "task_runner.h"
+#include <vespa/searchlib/common/lambdatask.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <future>
+
+using search::makeLambdaTask;
+
+namespace proton {
+
+namespace initializer {
+
+TaskRunner::TaskRunner(vespalib::Executor &executor)
+    : _executor(executor),
+      _runningTasks(0u)
+{
+}
+
+TaskRunner::~TaskRunner()
+{
+    assert(_runningTasks == 0u);
+}
+
+void
+TaskRunner::getReadyTasks(const InitializerTask::SP task, TaskList &readyTasks,
+                          TaskSet &checked)
+{
+    if (task->getState() != State::BLOCKED) {
+        return; // task running or done, all dependencies done
+    }
+    if (!checked.insert(task.get()).second) {
+        return; // task already checked from another depender
+    }
+    const TaskList &deps = task->getDependencies();
+    bool ready = true;
+    for (const auto &dep : deps) {
+        switch (dep->getState()) {
+        case State::RUNNING:
+            ready = false;
+            break;
+        case State::DONE:
+            break;
+        case State::BLOCKED:
+            ready = false;
+            getReadyTasks(dep, readyTasks, checked);
+        }
+    }
+    if (ready) {
+        readyTasks.push_back(task);
+    }
+}
+
+void
+TaskRunner::setTaskRunning(InitializerTask &task)
+{
+    // run by context executor
+    task.setRunning();
+    ++_runningTasks;
+}
+
+void
+TaskRunner::setTaskDone(InitializerTask &task, Context::SP context)
+{
+    // run by context executor
+    task.setDone();
+    --_runningTasks;
+    pollTask(context);
+}
+
+void
+TaskRunner::internalRunTask(InitializerTask::SP task, Context::SP context)
+{
+    // run by context executor
+    assert(task->getState() == State::BLOCKED);
+    setTaskRunning(*task);
+    auto done(makeLambdaTask([=]() { setTaskDone(*task, context); }));
+    _executor.execute(makeLambdaTask([=, done(std::move(done))]() mutable
+                                     {   task->run();
+                                         context->execute(std::move(done)); }));
+}
+
+void
+TaskRunner::internalRunTasks(const TaskList &taskList, Context::SP context)
+{
+    // run by context executor
+    for (auto &task : taskList) {
+        internalRunTask(task, context);
+    }
+}
+
+void
+TaskRunner::runTask(InitializerTask::SP task)
+{
+    vespalib::ThreadStackExecutor executor(1, 128 * 1024);
+    std::promise<bool> promise;
+    std::future<bool> future = promise.get_future();
+    runTask(task, executor,
+            makeLambdaTask([&]() { promise.set_value(true);  }));
+    (void) future.get();
+}
+
+void
+TaskRunner::pollTask(Context::SP context)
+{
+    // run by context executor
+    if (context->done()) {
+        return;
+    }
+    if (context->rootTask()->getState() == State::DONE) {
+        context->setDone();
+        return;
+    }
+    TaskList readyTasks;
+    TaskSet checked;
+    getReadyTasks(context->rootTask(), readyTasks, checked);
+    internalRunTasks(readyTasks, context);
+}
+
+void
+TaskRunner::runTask(InitializerTask::SP rootTask,
+                    vespalib::Executor &contextExecutor,
+                    vespalib::Executor::Task::UP doneTask)
+{
+    Context::SP context(std::make_shared<Context>(rootTask, contextExecutor,
+                                                  std::move(doneTask)));
+    context->execute(makeLambdaTask([=]() { pollTask(context); } ));
+}
+
+} // namespace proton::initializer
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h
new file mode 100644
index 00000000000..6fc52a6c289
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "initializer_task.h"
+#include <vespa/vespalib/util/executor.h>
+#include <vespa/vespalib/stllike/hash_set.h>
+
+namespace proton {
+
+namespace initializer {
+
+/*
+ * Class to run multiple init tasks with dependent tasks.
+ */
+class TaskRunner {
+    // Executor for the tasks, not to be confused by the context executor.
+    vespalib::Executor      &_executor;     // can be multithreaded
+    uint32_t                 _runningTasks; // used by context executor
+    using State = InitializerTask::State;
+    using TaskList = InitializerTask::List;
+    using TaskSet = vespalib::hash_set<void *>;
+
+    class Context {
+        InitializerTask::SP _rootTask;
+        vespalib::Executor &_contextExecutor;      // single threaded executor
+        vespalib::Executor::Task::UP _doneTask;
+
+    public:
+        using SP = std::shared_ptr<Context>;
+        Context(InitializerTask::SP rootTask,
+                vespalib::Executor &contextExecutor,
+                vespalib::Executor::Task::UP doneTask)
+            : _rootTask(rootTask),
+              _contextExecutor(contextExecutor),
+              _doneTask(std::move(doneTask))
+        {
+        }
+        bool done() const { return !_doneTask; }
+        void execute(vespalib::Executor::Task::UP task) {
+            auto res = _contextExecutor.execute(std::move(task));
+            assert(!res);
+        }
+        void setDone() { execute(std::move(_doneTask)); }
+        const InitializerTask::SP &rootTask() { return _rootTask; }
+        void schedulePoll();
+    };
+    void getReadyTasks(const InitializerTask::SP task, TaskList &readyTasks,
+                       TaskSet &checked);
+
+    void setTaskRunning(InitializerTask &task);
+
+    void setTaskDone(InitializerTask &task, Context::SP context);
+
+    void internalRunTask(InitializerTask::SP task, Context::SP context);
+
+    void internalRunTasks(const TaskList &taskList, Context::SP context);
+
+    void pollTask(Context::SP context);
+public:
+    TaskRunner(vespalib::Executor &executor);
+
+    virtual ~TaskRunner();
+
+    // Depecreated blocking API
+    void runTask(InitializerTask::SP task);
+
+    // Event based API, executor must be single threaded
+    void runTask(InitializerTask::SP rootTask,
+                 vespalib::Executor &contextExecutor,
+                 vespalib::Executor::Task::UP doneTask);
+};
+
+} // namespace proton::initializer
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/.gitignore b/searchcore/src/vespa/searchcore/proton/matchengine/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matchengine/CMakeLists.txt
new file mode 100644
index 00000000000..81532230876
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/CMakeLists.txt
@@ -0,0 +1,6 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_matchengine STATIC
+    SOURCES
+    matchengine.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/OWNERS b/searchcore/src/vespa/searchcore/proton/matchengine/OWNERS
new file mode 100644
index 00000000000..12b533ec610
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/OWNERS
@@ -0,0 +1 @@
+havardpe
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h b/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h
new file mode 100644
index 00000000000..17bfc2fa958
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/utility.hpp>
+#include <vespa/searchcore/proton/matching/isearchcontext.h>
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
+#include <vespa/searchlib/engine/searchreply.h>
+#include <vespa/searchlib/engine/searchrequest.h>
+#include <vespa/vespalib/util/thread_bundle.h>
+
+namespace proton {
+
+/**
+ * This interface describes a sync match operation handler. It is implemented by
+ * the DocumentDB class, and used by the MatchEngine class to delegate
+ * operations to the appropriate db.
+ */
+class IMatchHandler {
+public:
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::unique_ptr<IMatchHandler> UP;
+    typedef std::shared_ptr<IMatchHandler> SP;
+
+    /**
+     * Virtual destructor to allow inheritance.
+     */
+    virtual ~IMatchHandler() { }
+
+    /**
+     * @return Use the request and produce the matching result.
+     */
+    virtual search::engine::SearchReply::UP match(
+            const ISearchHandler::SP &searchHandler,
+            const search::engine::SearchRequest &req,
+            vespalib::ThreadBundle &threadBundle) const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp
new file mode 100644
index 00000000000..bdaa534a5be
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp
@@ -0,0 +1,204 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "matchengine.h"
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matchengine.matchengine");
+#include <vespa/searchcore/proton/common/state_reporter_utils.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <algorithm>
+
+namespace {
+
+class SearchTask : public vespalib::Executor::Task {
+private:
+    proton::MatchEngine                   &_engine;
+    search::engine::SearchRequest::Source  _request;
+    search::engine::SearchClient          &_client;
+
+public:
+    SearchTask(proton::MatchEngine &engine,
+               search::engine::SearchRequest::Source request,
+               search::engine::SearchClient &client)
+        : _engine(engine),
+          _request(std::move(request)),
+          _client(client)
+    {
+        // empty
+    }
+
+    void run() {
+        _engine.performSearch(std::move(_request), _client);
+    }
+};
+
+
+} // namespace anon
+
+namespace proton {
+
+using namespace matching;
+using namespace vespalib::slime;
+
+MatchEngine::MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t distributionKey)
+    : _lock(),
+      _distributionKey(distributionKey),
+      _closed(false),
+      _handlers(),
+      _executor(std::max(size_t(1), numThreads / threadsPerSearch), 256 * 1024),
+      _threadBundlePool(std::max(size_t(1), threadsPerSearch)),
+      _online(false),
+      _nodeUp(false),
+      _inService(false)
+{
+    // empty
+}
+
+MatchEngine::~MatchEngine()
+{
+    _executor.shutdown().sync();
+}
+
+void
+MatchEngine::close()
+{
+    LOG(debug, "Closing search interface.");
+    {
+        vespalib::LockGuard guard(_lock);
+        _closed = true;
+    }
+
+    LOG(debug, "Handshaking with task manager.");
+    _executor.sync();
+}
+
+ISearchHandler::SP
+MatchEngine::putSearchHandler(const DocTypeName &docTypeName,
+                              const ISearchHandler::SP &searchHandler)
+{
+    vespalib::LockGuard guard(_lock);
+    return _handlers.putHandler(docTypeName, searchHandler);
+}
+
+ISearchHandler::SP
+MatchEngine::getSearchHandler(const DocTypeName &docTypeName)
+{
+    vespalib::LockGuard guard(_lock);
+    return _handlers.getHandler(docTypeName);
+}
+
+ISearchHandler::SP
+MatchEngine::removeSearchHandler(const DocTypeName &docTypeName)
+{
+    vespalib::LockGuard guard(_lock);
+    return _handlers.removeHandler(docTypeName);
+}
+
+search::engine::SearchReply::UP
+MatchEngine::search(search::engine::SearchRequest::Source request,
+                    search::engine::SearchClient &client)
+{
+    if (_closed || !_nodeUp) {
+        search::engine::SearchReply::UP ret;
+        ret.reset(new search::engine::SearchReply);
+        ret->setDistributionKey(_distributionKey);
+
+        // TODO: Notify closed.
+
+        return ret;
+    }
+    vespalib::Executor::Task::UP task;
+    task.reset(new SearchTask(*this, std::move(request), client));
+    _executor.execute(std::move(task));
+    return search::engine::SearchReply::UP();
+}
+
+void
+MatchEngine::performSearch(search::engine::SearchRequest::Source req,
+                           search::engine::SearchClient &client)
+{
+    search::engine::SearchReply::UP ret(new search::engine::SearchReply);
+
+    if (req.get() != NULL) {
+        ISearchHandler::SP searchHandler;
+        vespalib::SimpleThreadBundle::UP threadBundle = _threadBundlePool.obtain();
+        { // try to find the match handler corresponding to the specified search doc type
+            vespalib::LockGuard guard(_lock);
+            DocTypeName docTypeName(*req.get());
+            searchHandler = _handlers.getHandler(docTypeName);
+        }
+        if (searchHandler.get() != NULL) {
+            ret = searchHandler->match(searchHandler, *req.get(), *threadBundle);
+        } else {
+            HandlerMap<ISearchHandler>::Snapshot::UP snapshot;
+            {
+                vespalib::LockGuard guard(_lock);
+                snapshot = _handlers.snapshot();
+            }
+            if (snapshot->valid()) {
+                ISearchHandler::SP handler = snapshot->getSP();
+                ret = handler->match(handler, *req.get(), *threadBundle); // use the first handler
+            }
+        }
+        _threadBundlePool.release(std::move(threadBundle));
+    }
+    ret->request = req.release();
+    ret->setDistributionKey(_distributionKey);
+    client.searchDone(std::move(ret));
+}
+
+void MatchEngine::setOnline()
+{
+    _online = true;
+}
+
+void MatchEngine::setOffline()
+{
+    _online = false;
+}
+
+void MatchEngine::setInService()
+{
+    _inService = true;
+}
+
+void MatchEngine::setOutOfService()
+{
+    _inService = false;
+}
+
+bool MatchEngine::isOnline() const {
+    return _online && _nodeUp && _inService;
+}
+
+
+void
+MatchEngine::setNodeUp(bool nodeUp)
+{
+    _nodeUp = nodeUp;
+}
+
+
+StatusReport::UP
+MatchEngine::reportStatus() const
+{
+    if (isOnline()) {
+        return StatusReport::create(StatusReport::Params("matchengine").
+                state(StatusReport::UPOK).
+                internalState("ONLINE"));
+    } else {
+        return StatusReport::create(StatusReport::Params("matchengine").
+                state(StatusReport::DOWN).
+                internalState("OFFLINE").
+                message("Search interface is offline"));
+    }
+}
+
+void
+MatchEngine::get_state(const Inserter &inserter, bool full) const
+{
+    (void) full;
+    Cursor &object = inserter.insertObject();
+    StateReporterUtils::convertToSlime(*reportStatus(), ObjectInserter(object, "status"));
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h
new file mode 100644
index 00000000000..33f35ace94d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h
@@ -0,0 +1,156 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/common/handlermap.hpp>
+#include <vespa/searchcore/proton/common/statusreport.h>
+#include <vespa/searchcore/proton/matchengine/imatchhandler.h>
+#include <vespa/searchlib/engine/searchapi.h>
+#include <vespa/vespalib/net/state_explorer.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/simple_thread_bundle.h>
+
+namespace proton {
+
+class MatchEngine : public boost::noncopyable,
+                    public search::engine::SearchServer,
+                    public vespalib::StateExplorer
+{
+private:
+    vespalib::Lock                     _lock;
+    const uint32_t                     _distributionKey;
+    bool                               _closed;
+    HandlerMap<ISearchHandler>         _handlers;
+    vespalib::ThreadStackExecutor      _executor;
+    vespalib::SimpleThreadBundle::Pool _threadBundlePool;
+    bool                               _online;
+    bool                               _nodeUp;
+    bool                               _inService;
+
+public:
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::unique_ptr<MatchEngine> UP;
+    typedef std::shared_ptr<MatchEngine> SP;
+
+    /**
+     * Constructs a new match engine. This does the necessary setup of the
+     * internal structures, without actually starting any threads. Before
+     * searching, you should register handlers for all known document types
+     * using the putSearchHandler() method.
+     *
+     * @param numThreads Number of threads allocated for handling search requests.
+     * @param threadsPerSearch number of threads used for each search
+     * @param distributionKey distributionkey of this node.
+     */
+    MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t distributionKey);
+
+    /**
+     * Frees any allocated resources. this will also stop all internal threads
+     * and wait for them to finish. All pending search requests are deleted.
+     */
+    ~MatchEngine();
+
+    /**
+     * Observe and reset internal executor stats
+     *
+     * @return executor stats
+     **/
+    vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); }
+
+    /**
+     * Closes the request handler interface. This will prevent any more data
+     * from entering this object, allowing you to flush all pending operations
+     * without having to safe-guard against input.
+     */
+    void close();
+
+    /**
+     * Registers a new search handler for the given document type. If another
+     * handler was already registered under the same type, this method will
+     * return a pointer to that handler.
+     *
+     * @param docType      The document type to register a handler for.
+     * @param matchHandler The handler to register.
+     * @return The replaced handler, if any.
+     */
+    ISearchHandler::SP
+    putSearchHandler(const DocTypeName &docTypeName,
+                     const ISearchHandler::SP &searchHandler);
+
+    /**
+     * Returns the search handler for the given document type. If no
+     * handler was registered, this method returns an empty shared
+     * pointer.
+     *
+     * @param docType The document type whose handler to return.
+     * @return The registered handler, if any.
+     */
+    ISearchHandler::SP
+    getSearchHandler(const DocTypeName &docTypeName);
+
+    /**
+     * Removes and returns the search handler for the given document
+     * type. If no handler was registered, this method returns an
+     * empty shared pointer.
+     *
+     * @param docType The document type whose handler to remove.
+     * @return The removed handler, if any.
+     */
+    ISearchHandler::SP
+    removeSearchHandler(const DocTypeName &docTypeName);
+
+    /**
+     * Performs the given search request in the current thread and passes the
+     * result to the given client. This method is used by the interal worker
+     * threads.
+     *
+     * @param req    The search request to perform.
+     * @param client The client to pass the results to.
+     */
+    void performSearch(search::engine::SearchRequest::Source req,
+                       search::engine::SearchClient &client);
+
+    /**
+     * Will enable the search interface.
+     */
+    void setOnline();
+
+    /**
+     * Will disable the search interface.
+     */
+    void setOffline();
+
+    /**
+     * Allow search interface to be online.
+     */
+    void setOutOfService();
+
+    /**
+     * Disallow search interface.
+     */
+    void setInService();
+
+    /** obtain current online status */
+    bool isOnline() const;
+
+    /** 
+     * Set node up/down, based on info from cluster controller.
+     */
+    void
+    setNodeUp(bool nodeUp);
+
+    StatusReport::UP reportStatus() const;
+
+    // Implements SearchServer.
+    search::engine::SearchReply::UP search(
+            search::engine::SearchRequest::Source request,
+            search::engine::SearchClient &client);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/.create-overrides.sh b/searchcore/src/vespa/searchcore/proton/matching/.create-overrides.sh
new file mode 100644
index 00000000000..82811afd790
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/.create-overrides.sh
@@ -0,0 +1,5 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ns_open="namespace proton {
+namespace matching {"
+ns_close="} // namespace proton::matching
+} // namespace proton"
diff --git a/searchcore/src/vespa/searchcore/proton/matching/.gitignore b/searchcore/src/vespa/searchcore/proton/matching/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt
new file mode 100644
index 00000000000..270a138d294
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt
@@ -0,0 +1,35 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_matching STATIC
+    SOURCES
+    attribute_limiter.cpp
+    blueprintbuilder.cpp
+    docid_range_scheduler.cpp
+    document_scorer.cpp
+    fakesearchcontext.cpp
+    handlerecorder.cpp
+    i_match_loop_communicator.cpp
+    indexenvironment.cpp
+    match_loop_communicator.cpp
+    match_master.cpp
+    match_params.cpp
+    match_phase_limit_calculator.cpp
+    match_phase_limiter.cpp
+    match_thread.cpp
+    match_tools.cpp
+    matcher.cpp
+    matching_stats.cpp
+    partial_result.cpp
+    query.cpp
+    queryenvironment.cpp
+    querylimiter.cpp
+    querynodes.cpp
+    requestcontext.cpp
+    result_processor.cpp
+    session_manager_explorer.cpp
+    sessionmanager.cpp
+    termdataextractor.cpp
+    termdatafromnode.cpp
+    viewresolver.cpp
+    DEPENDS
+    searchcore_grouping
+)
diff --git a/searchcore/src/vespa/searchcore/proton/matching/OWNERS b/searchcore/src/vespa/searchcore/proton/matching/OWNERS
new file mode 100644
index 00000000000..12b533ec610
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/OWNERS
@@ -0,0 +1 @@
+havardpe
diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp
new file mode 100644
index 00000000000..43008db813d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "attribute_limiter.h"
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/searchlib/query/tree/range.h>
+#include <vespa/searchlib/query/tree/simplequery.h>
+
+using namespace search::queryeval;
+using namespace search::query;
+using vespalib::make_string;
+using vespalib::string;
+
+namespace proton {
+namespace matching {
+
+AttributeLimiter::AttributeLimiter(Searchable &searchable_attributes,
+                                   const IRequestContext & requestContext,
+                                   const string &attribute_name,
+                                   bool descending,
+                                   const string &diversity_attribute,
+                                   double diversityCutoffFactor,
+                                   DiversityCutoffStrategy diversityCutoffStrategy)
+    : _searchable_attributes(searchable_attributes),
+      _requestContext(requestContext),
+      _attribute_name(attribute_name),
+      _descending(descending),
+      _diversity_attribute(diversity_attribute),
+      _lock(),
+      _match_datas(),
+      _blueprint(),
+      _estimatedHits(-1),
+      _diversityCutoffFactor(diversityCutoffFactor),
+      _diversityCutoffStrategy(diversityCutoffStrategy)
+{
+}
+
+namespace {
+
+vespalib::string STRICT_STR("strict");
+vespalib::string LOOSE_STR("loose");
+
+}
+
+AttributeLimiter::DiversityCutoffStrategy
+AttributeLimiter::toDiversityCutoffStrategy(const vespalib::stringref & strategy)
+{
+    return (strategy == STRICT_STR) ? DiversityCutoffStrategy::STRICT : DiversityCutoffStrategy::LOOSE;
+}
+
+const vespalib::string &
+AttributeLimiter::toString(DiversityCutoffStrategy strategy)
+{
+    return (strategy == DiversityCutoffStrategy::STRICT) ? STRICT_STR : LOOSE_STR;
+}
+
+SearchIterator::UP
+AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, bool strictSearch)
+{
+    vespalib::LockGuard guard(_lock);
+    const uint32_t my_field_id = 0;
+    search::fef::MatchDataLayout layout;
+    auto my_handle = layout.allocTermField(my_field_id);
+    if ( ! _blueprint ) {
+        const uint32_t no_unique_id = 0;
+        string range_spec = make_string("[;;%s%zu", (_descending)? "-" : "", want_hits);
+        if (max_group_size < want_hits) {
+            size_t cutoffGroups = (_diversityCutoffFactor*want_hits)/max_group_size;
+            range_spec.append(make_string(";%s;%zu;%zu;%s]", _diversity_attribute.c_str(), max_group_size,
+                                          cutoffGroups, toString(_diversityCutoffStrategy).c_str()));
+        } else {
+            range_spec.push_back(']');
+        }
+        Range range(range_spec);
+        SimpleRangeTerm node(range, _attribute_name, no_unique_id, Weight(0));
+        FieldSpecList field; // single field API is protected
+        field.add(FieldSpec(_attribute_name, my_field_id, my_handle));
+        _blueprint = _searchable_attributes.createBlueprint(_requestContext, field, node);
+        _blueprint->fetchPostings(strictSearch);
+        _estimatedHits = _blueprint->getState().estimate().estHits;
+    }
+    _match_datas.push_back(layout.createMatchData());
+    return _blueprint->createSearch(*_match_datas.back(), strictSearch);
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h
new file mode 100644
index 00000000000..10e8de9ebbc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/queryeval/searchable.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+
+#include <vespa/searchlib/fef/matchdata.h>
+
+namespace proton {
+namespace matching {
+
+/**
+ * This class is responsible for creating attribute-based search
+ * iterators that are used to limit the search space. Each search
+ * thread wants a separate search iterator, but the blueprint is
+ * shared between threads. All threads should request the same number
+ * of hits, so this class just lets the first thread requesting a
+ * search decide the number of hits in the underlying blueprint.
+ **/
+class AttributeLimiter
+{
+public:
+    enum DiversityCutoffStrategy { LOOSE, STRICT};
+    AttributeLimiter(search::queryeval::Searchable &searchable_attributes,
+                     const search::queryeval::IRequestContext & requestContext,
+                     const vespalib::string &attribute_name, bool descending,
+                     const vespalib::string &diversity_attribute,
+                     double diversityCutoffFactor,
+                     DiversityCutoffStrategy diversityCutoffStrategy);
+    search::queryeval::SearchIterator::UP create_search(size_t want_hits, size_t max_group_size, bool strictSearch);
+    bool was_used() const { return ((!_match_datas.empty()) || (_blueprint.get() != nullptr)); }
+    ssize_t getEstimatedHits() const { return _estimatedHits; }
+    static DiversityCutoffStrategy toDiversityCutoffStrategy(const vespalib::stringref & strategy);
+private:
+    const vespalib::string & toString(DiversityCutoffStrategy strategy);
+    search::queryeval::Searchable            & _searchable_attributes;
+    const search::queryeval::IRequestContext & _requestContext;
+    vespalib::string                           _attribute_name;
+    bool                                       _descending;
+    vespalib::string                           _diversity_attribute;
+    vespalib::Lock                             _lock;
+    std::vector<search::fef::MatchData::UP>    _match_datas;
+    search::queryeval::Blueprint::UP           _blueprint;
+    ssize_t                                    _estimatedHits;
+    double                                     _diversityCutoffFactor;
+    DiversityCutoffStrategy                    _diversityCutoffStrategy;
+};
+
+} // namespace proton::matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp
new file mode 100644
index 00000000000..4392a5d112e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp
@@ -0,0 +1,187 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.blueprintbuilder");
+
+#include "querynodes.h"
+#include "blueprintbuilder.h"
+#include <vespa/searchlib/query/tree/customtypevisitor.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/queryeval/leaf_blueprints.h>
+#include <vespa/searchlib/queryeval/intermediate_blueprints.h>
+#include <vespa/searchlib/queryeval/equiv_blueprint.h>
+#include <vespa/searchlib/queryeval/searchable.h>
+#include <vespa/searchlib/queryeval/get_weight_from_node.h>
+#include <vespa/searchlib/fef/handle.h>
+#include <vespa/searchlib/fef/fieldinfo.h>
+#include <vector>
+
+using namespace search::queryeval;
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+struct Mixer {
+    std::unique_ptr<OrBlueprint> attributes;
+
+    Mixer() : attributes() {}
+
+    void addAttribute(Blueprint::UP attr) {
+        if (attributes.get() == 0) {
+            attributes.reset(new OrBlueprint());
+        }
+        attributes->addChild(std::move(attr));
+    }
+
+    Blueprint::UP mix(Blueprint::UP indexes) {
+        if (attributes.get() == 0) {
+            if (indexes.get() == 0) {
+                return Blueprint::UP(new EmptyBlueprint());
+            }
+            return Blueprint::UP(std::move(indexes));
+        }
+        if (indexes.get() == 0) {
+            if (attributes->childCnt() == 1) {
+                return attributes->removeChild(0);
+            } else {
+                return Blueprint::UP(std::move(attributes));
+            }
+        }
+        attributes->addChild(Blueprint::UP(std::move(indexes)));
+        return Blueprint::UP(std::move(attributes));
+    }
+};
+
+/**
+ * requires that match data space has been reserved
+ */
+class BlueprintBuilderVisitor :
+        public search::query::CustomTypeVisitor<ProtonNodeTypes>
+{
+private:
+    const IRequestContext & _requestContext;
+    ISearchContext &_context;
+    Blueprint::UP   _result;
+
+    void buildChildren(IntermediateBlueprint &parent,
+                       const std::vector<search::query::Node *> &children)
+    {
+        for (size_t i = 0; i < children.size(); ++i) {
+            parent.addChild(BlueprintBuilder::build(_requestContext, *children[i], _context));
+        }
+    }
+
+    template <typename NodeType>
+    void buildIntermediate(IntermediateBlueprint *b, NodeType &n) {
+        std::unique_ptr<IntermediateBlueprint> blueprint(b);
+        buildChildren(*blueprint, n.getChildren());
+        _result.reset(blueprint.release());
+    }
+
+    void buildWeakAnd(ProtonWeakAnd &n) {
+        WeakAndBlueprint *wand = new WeakAndBlueprint(n.getMinHits());
+        Blueprint::UP result(wand);
+        for (size_t i = 0; i < n.getChildren().size(); ++i) {
+            search::query::Node &node = *n.getChildren()[i];
+            uint32_t weight = getWeightFromNode(node).percent();
+            wand->addTerm(BlueprintBuilder::build(_requestContext, node, _context), weight);
+        }
+        _result = std::move(result);
+    }
+
+    void buildEquiv(ProtonEquiv &n) {
+        double eqw = n.getWeight().percent();
+        FieldSpecBaseList specs;
+        for (size_t i = 0; i < n.numFields(); ++i) {
+            specs.add(n.field(i).fieldSpec());
+        }
+        EquivBlueprint *eq = new EquivBlueprint(specs, n.children_mdl);
+        _result.reset(eq);
+        for (size_t i = 0; i < n.getChildren().size(); ++i) {
+            search::query::Node &node = *n.getChildren()[i];
+            double w = getWeightFromNode(node).percent();
+            eq->addTerm(BlueprintBuilder::build(_requestContext, node, _context), w / eqw);
+        }
+        n.setDocumentFrequency(_result->getState().estimate().estHits, _context.getDocIdLimit());
+    }
+
+    template <typename NodeType>
+    void buildTerm(NodeType &n) {
+        FieldSpecList indexFields;
+        Mixer mixer;
+        for (size_t i = 0; i < n.numFields(); ++i) {
+            const ProtonTermData::FieldEntry &field = n.field(i);
+            assert(field.getFieldId() != search::fef::IllegalFieldId);
+            assert(field.getHandle() != search::fef::IllegalHandle);
+            if (field.attribute_field) {
+                FieldSpecList attrField;
+                attrField.add(field.fieldSpec());
+                mixer.addAttribute(_context.getAttributes().createBlueprint(_requestContext, attrField, n));
+            } else {
+                indexFields.add(field.fieldSpec());
+            }
+        }
+        Blueprint::UP indexBlueprint;
+        if (!indexFields.empty()) {
+            indexBlueprint = _context.getIndexes().createBlueprint(_requestContext, indexFields, n);
+        }
+        _result = mixer.mix(std::move(indexBlueprint));
+        n.setDocumentFrequency(_result->getState().estimate().estHits, _context.getDocIdLimit());
+    }
+
+protected:
+    virtual void visit(ProtonAnd &n)     { buildIntermediate(new AndBlueprint(), n); }
+    virtual void visit(ProtonAndNot &n)  { buildIntermediate(new AndNotBlueprint(), n); }
+    virtual void visit(ProtonOr &n)      { buildIntermediate(new OrBlueprint(), n); }
+    virtual void visit(ProtonWeakAnd &n) { buildWeakAnd(n); }
+    virtual void visit(ProtonEquiv &n)   { buildEquiv(n); }
+    virtual void visit(ProtonRank &n)    { buildIntermediate(new RankBlueprint(), n); }
+    virtual void visit(ProtonNear &n)    { buildIntermediate(new NearBlueprint(n.getDistance()), n); }
+    virtual void visit(ProtonONear &n)   { buildIntermediate(new ONearBlueprint(n.getDistance()), n); }
+
+    virtual void visit(ProtonWeightedSetTerm &n) { buildTerm(n); }
+    virtual void visit(ProtonDotProduct &n) { buildTerm(n); }
+    virtual void visit(ProtonWandTerm &n) { buildTerm(n); }
+
+    virtual void visit(ProtonPhrase &n)        { buildTerm(n); }
+    virtual void visit(ProtonNumberTerm &n)    { buildTerm(n); }
+    virtual void visit(ProtonLocationTerm &n)  { buildTerm(n); }
+    virtual void visit(ProtonPrefixTerm &n)    { buildTerm(n); }
+    virtual void visit(ProtonRangeTerm &n)     { buildTerm(n); }
+    virtual void visit(ProtonStringTerm &n)    { buildTerm(n); }
+    virtual void visit(ProtonSubstringTerm &n) { buildTerm(n); }
+    virtual void visit(ProtonSuffixTerm &n)    { buildTerm(n); }
+    virtual void visit(ProtonPredicateQuery &n) { buildTerm(n); }
+    virtual void visit(ProtonRegExpTerm &n)    { buildTerm(n); }
+
+public:
+    BlueprintBuilderVisitor(const IRequestContext & requestContext, ISearchContext &context) :
+        _requestContext(requestContext),
+        _context(context),
+        _result()
+    { }
+    Blueprint::UP build() {
+        assert(_result);
+        return std::move(_result);
+    }
+};
+
+} // namespace proton::matching::<unnamed>
+
+search::queryeval::Blueprint::UP
+BlueprintBuilder::build(const IRequestContext & requestContext,
+                        search::query::Node &node,
+                        ISearchContext &context)
+{
+    BlueprintBuilderVisitor visitor(requestContext, context);
+    node.accept(visitor);
+    Blueprint::UP result = visitor.build();
+    result->setDocIdLimit(context.getDocIdLimit());
+    return result;
+}
+
+}  // namespace matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h
new file mode 100644
index 00000000000..0d740f3a139
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "isearchcontext.h"
+#include <vespa/searchlib/query/tree/node.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+
+namespace proton {
+namespace matching {
+
+struct BlueprintBuilder {
+    /**
+     * Build a tree of blueprints from the query tree and inject
+     * blueprint meta-data back into corresponding query tree nodes.
+     */
+    static search::queryeval::Blueprint::UP
+    build(const search::queryeval::IRequestContext & requestContext,
+          search::query::Node &node,
+          ISearchContext &context);
+};
+
+}  // namespace matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.cpp b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.cpp
new file mode 100644
index 00000000000..012b34e94cb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.cpp
@@ -0,0 +1,152 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "docid_range_scheduler.h"
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+size_t clamped_sub(size_t a, size_t b) { return (b > a) ? 0 : (a - b); }
+
+} // namespace proton::matching::<unnamed>
+
+const std::atomic<size_t> IdleObserver::_always_zero(0);
+
+//-----------------------------------------------------------------------------
+
+PartitionDocidRangeScheduler::PartitionDocidRangeScheduler(size_t num_threads, uint32_t docid_limit)
+    : _ranges()
+{
+    DocidRangeSplitter splitter(DocidRange(1, docid_limit), num_threads);
+    for (size_t i = 0; i < num_threads; ++i) {
+        _ranges.push_back(splitter.get(i));
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+DocidRange
+TaskDocidRangeScheduler::next_task(size_t thread_id)
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    DocidRange work = _splitter.get(_next_task);
+    if (_next_task < _num_tasks) {
+        ++_next_task;
+    }
+    _assigned[thread_id] += work.size();
+    size_t todo = _unassigned.load(std::memory_order::memory_order_relaxed);
+    _unassigned.store(clamped_sub(todo, work.size()), std::memory_order::memory_order_relaxed);
+    return work;
+}
+
+TaskDocidRangeScheduler::TaskDocidRangeScheduler(size_t num_threads, size_t num_tasks, uint32_t docid_limit)
+    : _lock(),
+      _splitter(DocidRange(1, docid_limit), num_tasks),
+      _next_task(0),
+      _num_tasks(num_tasks),
+      _assigned(num_threads, 0),
+      _unassigned(_splitter.full_range().size())
+{
+}
+
+//-----------------------------------------------------------------------------
+
+size_t
+AdaptiveDocidRangeScheduler::take_idle(const Guard &)
+{
+    size_t thread_id = _idle.back();
+    _idle.pop_back();
+    _num_idle.store(_idle.size(), std::memory_order::memory_order_relaxed);
+    assert(_workers[thread_id].is_idle);
+    return thread_id;
+}
+
+void
+AdaptiveDocidRangeScheduler::make_idle(const Guard &, size_t thread_id)
+{
+    assert(!_workers[thread_id].is_idle);
+    _workers[thread_id].is_idle = true;
+    _idle.push_back(thread_id);
+    _num_idle.store(_idle.size(), std::memory_order::memory_order_relaxed);    
+}
+
+void
+AdaptiveDocidRangeScheduler::donate(const Guard &guard, size_t src_thread, DocidRange range)
+{
+    size_t dst_thread = take_idle(guard);
+    _workers[dst_thread].next_range = range;
+    _workers[dst_thread].is_idle = false;
+    _workers[dst_thread].condition.notify_one();
+    _assigned[src_thread] -= range.size();
+    _assigned[dst_thread] += range.size();
+}
+
+bool
+AdaptiveDocidRangeScheduler::all_work_done(const Guard &) const
+{
+    // when all threads are idle at the same time there is no more work
+    return ((_idle.size() + 1) == _workers.size());
+}
+
+DocidRange
+AdaptiveDocidRangeScheduler::finalize(const Guard &guard, size_t thread_id)
+{
+    while (!_idle.empty()) {
+        donate(guard, thread_id, DocidRange());
+    }
+    return DocidRange();
+}
+
+AdaptiveDocidRangeScheduler::AdaptiveDocidRangeScheduler(size_t num_threads, uint32_t min_task, uint32_t docid_limit)
+    : _splitter(DocidRange(1, docid_limit), num_threads),
+      _min_task(std::max(1u, min_task)),
+      _lock(),
+      _assigned(num_threads, 0),
+      _workers(num_threads),
+      _idle(),
+      _num_idle(_idle.size())
+{
+    _idle.reserve(num_threads);
+    for (size_t i = 0; i < num_threads; ++i) {
+        _assigned[i] = _splitter.get(i).size();
+    }
+}
+
+DocidRange
+AdaptiveDocidRangeScheduler::next_range(size_t thread_id)
+{
+    Guard guard(_lock);
+    if (all_work_done(guard)) {
+        return finalize(guard, thread_id);
+    }
+    make_idle(guard, thread_id);
+    while (_workers[thread_id].is_idle) {
+        _workers[thread_id].condition.wait(guard);
+    }
+    return _workers[thread_id].next_range;
+}
+
+DocidRange
+AdaptiveDocidRangeScheduler::share_range(size_t thread_id, DocidRange todo)
+{
+    size_t max_parts = (todo.size() / _min_task);
+    if (max_parts > 1) {
+        Guard guard(_lock);
+        size_t parts = std::min(_idle.size() + 1, max_parts);
+        if (parts > 1) {
+            DocidRangeSplitter splitter(todo, parts);
+            for (size_t i = 1; i < parts; ++i) {
+                donate(guard, thread_id, splitter.get(i));
+            }
+            return splitter.get(0);
+        }
+    }
+    return todo;
+}
+
+//-----------------------------------------------------------------------------
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h
new file mode 100644
index 00000000000..63567bd97f5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h
@@ -0,0 +1,206 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/queryeval/begin_and_end_id.h>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include <algorithm>
+
+namespace proton {
+namespace matching {
+
+/**
+ * A range of document ids representing a subset of the search space.
+ **/
+struct DocidRange {
+    uint32_t begin;
+    uint32_t end;
+    DocidRange() : begin(search::endDocId), end(search::endDocId) {}
+    DocidRange(uint32_t begin_in, uint32_t end_in)
+        : begin(begin_in), end(std::max(begin_in, end_in)) {}
+    bool empty() const { return (end <= begin); }
+    size_t size() const { return (end - begin); }
+};
+
+/**
+ * Utility used to split a docid range into multiple consecutive
+ * pieces of equal size.
+ **/
+class DocidRangeSplitter {
+private:
+    DocidRange _range;
+    uint32_t   _step;
+    uint32_t   _skew;
+
+    uint32_t offset(uint32_t i) const {
+        return std::min(_range.end, _range.begin + (_step * i) + std::min(i, _skew));
+    }
+
+public:
+    DocidRangeSplitter(DocidRange total_range, size_t count)
+        : _range(total_range),
+          _step(_range.size() / count),
+          _skew(_range.size() % count) {}
+    DocidRange get(size_t i) const { return DocidRange(offset(i), offset(i + 1)); }
+    DocidRange full_range() const { return _range; }
+};
+
+/**
+ * Utility class used to poll the current number of idle worker
+ * threads as cheaply as possible.
+ **/
+class IdleObserver {
+private:
+    static const std::atomic<size_t> _always_zero;
+    const std::atomic<size_t> &_num_idle;
+public:
+    IdleObserver() : _num_idle(_always_zero) {}
+    IdleObserver(const std::atomic<size_t> &num_idle) : _num_idle(num_idle) {}
+    bool is_always_zero() const { return (&_num_idle == &_always_zero); }
+    size_t get() const { return _num_idle.load(std::memory_order::memory_order_relaxed); }
+};
+
+/**
+ * Interface for the component responsible for assigning docid ranges
+ * to search threads during multi-threaded query execution. Each
+ * worker starts by calling the 'first_range' function to get
+ * something to do. When a worker is ready for more work, it calls the
+ * 'next_range' function. When a worker is assigned an empty range,
+ * its work is done.
+ * 
+ * The 'total_span' function returns a range that is guaranteed to
+ * contain all ranges assigned to the given worker. The 'total_size'
+ * function returns the accumulated size of all ranges assigned to the
+ * given worker. The 'unassigned_size' function returns the
+ * accumulated size of all currently unassigned ranges.
+ *
+ * Note that the return values from 'total_span', 'total_size' and
+ * 'unassigned_size' may or may not account for the range returned
+ * from 'first_range' since the scheduler is allowed to pre-assign
+ * ranges to workers. Calling 'first_range' first ensures that all
+ * other return values make sense.
+ *
+ * The 'idle_observer' and 'share_range' functions are used for
+ * work-sharing, where a worker thread potentially can offload some of
+ * its remaining work to another idle worker thread. The
+ * 'idle_observer' function is used to obtain an object that makes it
+ * cheap to check whether other threads may be idle. The IdleObserver
+ * object will also indicate whether the scheduler implementation
+ * supports work-sharing at all. This enables the inner loop to be
+ * further specialized by not trying to share work if the scheduler
+ * does not support it. The 'share_range' function may be called by
+ * any worker thread to try to share some of its remaining work with
+ * other idle worker threads. Note that workers that employ
+ * work-sharing must support processing docid ranges in non-increasing
+ * order. The 'share_range' function should be called when the
+ * IdleObserver indicates that other threads may be idle. All
+ * remaining work currently assigned to the worker thread is passed
+ * into the 'share_range' function. If no other threads are available,
+ * the entire range will be returned back out again. If some work
+ * could be re-assigned to other threads, the 'share_range' function
+ * will return the remaining work to be done by the thread calling
+ * it. The returned range is guaranteed to be a prefix of the range
+ * passed as input to the 'share_range' function.
+ **/
+struct DocidRangeScheduler {
+    typedef std::unique_ptr<DocidRangeScheduler> UP;
+    virtual DocidRange first_range(size_t thread_id) = 0;
+    virtual DocidRange next_range(size_t thread_id) = 0;
+    virtual DocidRange total_span(size_t thread_id) const = 0;
+    virtual size_t total_size(size_t thread_id) const = 0;
+    virtual size_t unassigned_size() const = 0;
+    virtual IdleObserver make_idle_observer() const = 0;
+    virtual DocidRange share_range(size_t thread_id, DocidRange todo) = 0;
+    virtual ~DocidRangeScheduler() {}
+};
+
+/**
+ * A scheduler dividing the total docid space into a single docid
+ * range (partition) for each thread. The first thread gets the first
+ * part and so on.
+ **/
+class PartitionDocidRangeScheduler : public DocidRangeScheduler
+{
+private:
+    std::vector<DocidRange> _ranges;
+public:
+    PartitionDocidRangeScheduler(size_t num_threads, uint32_t docid_limit);
+    DocidRange first_range(size_t thread_id) override { return _ranges[thread_id]; }
+    DocidRange next_range(size_t) override { return DocidRange(); }
+    DocidRange total_span(size_t thread_id) const override { return _ranges[thread_id]; }
+    size_t total_size(size_t thread_id) const override { return _ranges[thread_id].size(); }
+    size_t unassigned_size() const override { return 0; }
+    IdleObserver make_idle_observer() const override { return IdleObserver(); }
+    DocidRange share_range(size_t, DocidRange todo) override { return todo; }
+};
+
+/**
+ * A scheduler dividing the total docid space into tasks of equal
+ * size. Tasks are assigned according to increasing docid to the first
+ * worker thread that wants more to do.
+ **/
+class TaskDocidRangeScheduler : public DocidRangeScheduler
+{
+private:
+    std::mutex          _lock;
+    DocidRangeSplitter  _splitter;
+    size_t              _next_task;
+    size_t              _num_tasks;
+    std::vector<size_t> _assigned;
+    std::atomic<size_t> _unassigned;
+
+    DocidRange next_task(size_t thread_id);
+public:
+    TaskDocidRangeScheduler(size_t num_threads, size_t num_tasks, uint32_t docid_limit);
+    DocidRange first_range(size_t thread_id) override { return next_task(thread_id); }
+    DocidRange next_range(size_t thread_id) override { return next_task(thread_id); }
+    DocidRange total_span(size_t) const override { return _splitter.full_range(); }
+    size_t total_size(size_t thread_id) const override { return _assigned[thread_id]; }
+    size_t unassigned_size() const override { return _unassigned.load(std::memory_order::memory_order_relaxed); }
+    IdleObserver make_idle_observer() const override { return IdleObserver(); }
+    DocidRange share_range(size_t, DocidRange todo) override { return todo; }
+};
+
+/**
+ * An adaptive scheduler that begins by giving each thread an equal
+ * part of the docid space and then uses cooperative work-sharing to
+ * re-distribute work between threads as needed.
+ **/
+class AdaptiveDocidRangeScheduler : public DocidRangeScheduler
+{
+private:
+    using Guard = std::unique_lock<std::mutex>;
+    struct Worker {
+        std::condition_variable condition;
+        bool                    is_idle;
+        DocidRange              next_range;
+        Worker() : condition(), is_idle(false), next_range() {}
+    };
+    DocidRangeSplitter  _splitter;
+    uint32_t            _min_task;
+    std::mutex          _lock;
+    std::vector<size_t> _assigned;
+    std::vector<Worker> _workers;
+    std::vector<size_t> _idle;
+    std::atomic<size_t> _num_idle;
+
+    VESPA_DLL_LOCAL size_t take_idle(const Guard &guard);
+    VESPA_DLL_LOCAL void make_idle(const Guard &guard, size_t thread_id);
+    VESPA_DLL_LOCAL void donate(const Guard &guard, size_t src_thread, DocidRange range);
+    VESPA_DLL_LOCAL bool all_work_done(const Guard &guard) const;
+    VESPA_DLL_LOCAL DocidRange finalize(const Guard &guard, size_t thread_id);
+public:
+    AdaptiveDocidRangeScheduler(size_t num_threads, uint32_t min_task, uint32_t docid_limit);
+    DocidRange first_range(size_t thread_id) override { return _splitter.get(thread_id); }
+    DocidRange next_range(size_t thread_id) override;
+    DocidRange total_span(size_t) const override { return _splitter.full_range(); }
+    size_t total_size(size_t thread_id) const override { return _assigned[thread_id]; }
+    size_t unassigned_size() const override { return 0; }
+    IdleObserver make_idle_observer() const override { return IdleObserver(_num_idle); }
+    DocidRange share_range(size_t, DocidRange todo) override;
+};
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp b/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp
new file mode 100644
index 00000000000..09e65ab3a78
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "document_scorer.h"
+
+using search::feature_t;
+using search::fef::FeatureHandle;
+using search::fef::RankProgram;
+using search::queryeval::SearchIterator;
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+const feature_t *
+extractScoreFeature(const RankProgram &rankProgram)
+{
+    std::vector<vespalib::string> featureNames;
+    std::vector<FeatureHandle> featureHandles;
+    rankProgram.get_seed_handles(featureNames, featureHandles);
+    assert(featureNames.size() == 1);
+    assert(featureHandles.size() == 1);
+    return rankProgram.match_data().resolveFeature(featureHandles.front());
+}
+
+}
+
+DocumentScorer::DocumentScorer(RankProgram &rankProgram,
+                               SearchIterator &searchItr)
+    : _rankProgram(rankProgram),
+      _searchItr(searchItr),
+      _scoreFeature(extractScoreFeature(rankProgram))
+{
+}
+
+feature_t
+DocumentScorer::score(uint32_t docId)
+{
+    return doScore(docId);
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/document_scorer.h b/searchcore/src/vespa/searchcore/proton/matching/document_scorer.h
new file mode 100644
index 00000000000..d235da859a2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/document_scorer.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/fef/rank_program.h>
+#include <vespa/searchlib/queryeval/hitcollector.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+
+namespace proton {
+namespace matching {
+
+/**
+ * Class used to calculate the rank score for a set of documents using
+ * a rank program for calculation and a search iterator for unpacking match data.
+ * The calculateScore() function is always called in increasing docId order.
+ */
+class DocumentScorer : public search::queryeval::HitCollector::DocumentScorer
+{
+private:
+    search::fef::RankProgram &_rankProgram;
+    search::queryeval::SearchIterator &_searchItr;
+    const search::feature_t *_scoreFeature;
+
+public:
+    DocumentScorer(search::fef::RankProgram &rankProgram,
+                   search::queryeval::SearchIterator &searchItr);
+
+    search::feature_t doScore(uint32_t docId) {
+        _searchItr.unpack(docId);
+        _rankProgram.run(docId);
+        return *_scoreFeature;
+    }
+
+    virtual search::feature_t score(uint32_t docId) override;
+};
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp
new file mode 100644
index 00000000000..6c6e8bb869d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.fakesearchcontext");
+
+#include "fakesearchcontext.h"
+
+namespace proton {
+namespace matching {
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h
new file mode 100644
index 00000000000..3db4a74b7f7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "isearchcontext.h"
+#include <vespa/searchcore/proton/common/indexsearchabletosearchableadapter.h>
+#include <vespa/searchcorespi/index/fakeindexsearchable.h>
+#include <vespa/searchcorespi/index/indexcollection.h>
+#include <vespa/searchlib/queryeval/fake_searchable.h>
+#include <vespa/searchlib/attribute/fixedsourceselector.h>
+#include <algorithm>
+#include <map>
+#include <vector>
+
+namespace proton {
+namespace matching {
+
+using searchcorespi::FakeIndexSearchable;
+using searchcorespi::IndexSearchable;
+using searchcorespi::IndexCollection;
+
+class FakeSearchContext : public ISearchContext
+{
+public:
+    typedef search::queryeval::FakeSearchable FakeSearchable;
+
+private:
+    vespalib::Clock                        _clock;
+    vespalib::Doom                         _doom;
+    search::queryeval::ISourceSelector::SP _selector;
+    search::attribute::IAttributeContext  *_attrCtx;
+    IndexCollection::SP                    _indexes;
+    IndexSearchableToSearchableAdapter     _searchableAdapter;
+    FakeSearchable                         _attrSearchable;
+    uint32_t                               _docIdLimit;
+
+public:
+    FakeSearchContext(size_t initialNumDocs=0)
+        : _clock(),
+          _doom(_clock, -1),
+          _selector(new search::FixedSourceSelector(0, "fs", initialNumDocs)),
+          _attrCtx(NULL),
+          _indexes(new IndexCollection(_selector)),
+          _searchableAdapter(_indexes, *_attrCtx),
+          _attrSearchable(),
+          _docIdLimit(initialNumDocs) {}
+
+    FakeSearchContext &addIdx(uint32_t id) {
+        _indexes->append(id, IndexSearchable::SP(new FakeIndexSearchable()));
+        return *this;
+    }
+
+    FakeSearchContext &setLimit(uint32_t limit) {
+        _docIdLimit = limit;
+        return *this;
+    }
+
+    FakeSearchable &attr() { return _attrSearchable; }
+
+    FakeIndexSearchable &idx(uint32_t i) {
+        return static_cast<FakeIndexSearchable &>(_indexes->getSearchable(i));
+    }
+
+    search::queryeval::ISourceSelector &selector() { return *_selector; }
+
+    // Implements ISearchContext
+    virtual search::queryeval::Searchable &getIndexes() {
+        return _searchableAdapter;
+    }
+
+    virtual search::queryeval::Searchable &getAttributes() {
+        return _attrSearchable;
+    }
+
+    virtual uint32_t getDocIdLimit() {
+        return _docIdLimit;
+    }
+    virtual const vespalib::Doom & getDoom() const { return _doom; }
+};
+
+}  // namespace matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.cpp b/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.cpp
new file mode 100644
index 00000000000..b7d30a6b051
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.cpp
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "handlerecorder.h"
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/backtrace.h>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".proton.matching.handlerecorder");
+
+using search::fef::TermFieldHandle;
+
+namespace proton {
+namespace matching {
+
+#ifdef __PIC__
+    #define TLS_LINKAGE __attribute__((visibility("hidden"), tls_model("initial-exec")))
+#else
+    #define TLS_LINKAGE __attribute__((visibility("hidden"), tls_model("local-exec")))
+#endif
+
+namespace {
+     __thread HandleRecorder * _T_recorder TLS_LINKAGE = NULL;
+     __thread bool _T_assert_all_handles_are_registered = false;
+}
+
+HandleRecorder::HandleRecorder() :
+    _handles()
+{
+}
+
+vespalib::string
+HandleRecorder::toString() const
+{
+    vespalib::asciistream os;
+    std::vector<TermFieldHandle> sorted;
+    for (TermFieldHandle handle : _handles) {
+        sorted.push_back(handle);
+    }
+    std::sort(sorted.begin(), sorted.end());
+    if ( !sorted.empty() ) {
+        os << sorted[0];
+        for (size_t i(1); i < sorted.size(); i++) {
+            os << ' ' << sorted[i];
+        }
+    }
+    return os.str();
+}
+
+HandleRecorder::Binder::Binder(HandleRecorder & recorder)
+{
+    _T_recorder = & recorder;
+}
+
+HandleRecorder::Asserter::Asserter()
+{
+    _T_assert_all_handles_are_registered = true;
+}
+
+HandleRecorder::Asserter::~Asserter()
+{
+    _T_assert_all_handles_are_registered = false;
+}
+
+HandleRecorder::~HandleRecorder()
+{
+}
+
+HandleRecorder::Binder::~Binder()
+{
+    _T_recorder = NULL;
+}
+
+void HandleRecorder::registerHandle(TermFieldHandle handle)
+{
+    // There should be no registration of handles that is not recorded.
+    // That will lead to issues later on.
+    if (_T_recorder != NULL) {
+        LOG(spam, "Handle %d - StackTrace : %s", handle, vespalib::getStackTrace(0).c_str());
+        _T_recorder->add(handle);
+    } else if (_T_assert_all_handles_are_registered) {
+        assert(_T_recorder != NULL);
+    }
+}
+
+}  // namespace matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.h b/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.h
new file mode 100644
index 00000000000..1cb20275c9e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.h
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/fef/handle.h>
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/util/noncopyable.hpp>
+
+namespace proton {
+namespace matching {
+
+/**
+ * This is a recorder that will register all handles used by any features for a given query.
+ * It is activated using thread locals by using the Binder.
+ * In order to ensure that no handles goes by unnoticed and asserter is added. It should typically have the
+ * same lifespan as the recorder itself.
+ * After the Binders has gone out of scope this recorder has a list of all feature handles that might be
+ * by this query. This can then be used to avoid a lot of unpacking of data.
+ */
+class HandleRecorder
+{
+public:
+    typedef vespalib::hash_set<search::fef::TermFieldHandle> HandleSet;
+    class Binder : public vespalib::noncopyable {
+    public:
+        Binder(HandleRecorder & recorder);
+        ~Binder();
+    };
+    class Asserter : public vespalib::noncopyable {
+    public:
+        Asserter();
+        ~Asserter();
+    };
+    HandleRecorder();
+    ~HandleRecorder();
+    const HandleSet & getHandles() const { return _handles; }
+    static void registerHandle(search::fef::TermFieldHandle handle);
+    vespalib::string toString() const;
+private:
+    void add(search::fef::TermFieldHandle handle) {
+        _handles.insert(handle);
+    }
+    HandleSet _handles;
+};
+
+}  // namespace matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.cpp b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.cpp
new file mode 100644
index 00000000000..7e040019de2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.cpp
@@ -0,0 +1,4 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "i_match_loop_communicator.h"
diff --git a/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h
new file mode 100644
index 00000000000..bc10db2bd65
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/feature.h>
+#include <vespa/searchlib/queryeval/scores.h>
+#include <utility>
+
+namespace proton {
+namespace matching {
+
+struct IMatchLoopCommunicator {
+    typedef search::feature_t feature_t;
+    typedef search::queryeval::Scores Range;
+    typedef std::pair<Range, Range> RangePair;
+    struct Matches {
+        size_t hits;
+        size_t docs;
+        Matches() : hits(0), docs(0) {}
+        Matches(size_t hits_in, size_t docs_in) : hits(hits_in), docs(docs_in) {}
+        void add(const Matches &rhs) {
+            hits += rhs.hits;
+            docs += rhs.docs;
+        }
+    };
+    virtual double estimate_match_frequency(const Matches &matches) = 0;
+    virtual size_t selectBest(const std::vector<feature_t> &sortedScores) = 0;
+    virtual RangePair rangeCover(const RangePair &ranges) = 0;
+    virtual ~IMatchLoopCommunicator() {}
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp
new file mode 100644
index 00000000000..be964aa9473
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp
@@ -0,0 +1,164 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.indexenvironment");
+#include "indexenvironment.h"
+
+#include <vespa/searchlib/fef/functiontablefactory.h>
+#include <vespa/searchlib/fef/indexproperties.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+
+using namespace search::fef;
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+search::fef::CollectionType convertCollectionType(search::index::Schema::CollectionType type) {
+    switch (type) {
+    case search::index::Schema::SINGLE:      return search::fef::CollectionType::SINGLE;
+    case search::index::Schema::ARRAY:       return search::fef::CollectionType::ARRAY;
+    case search::index::Schema::WEIGHTEDSET: return search::fef::CollectionType::WEIGHTEDSET;
+    default:
+        abort();
+    }
+}
+
+}
+
+void
+IndexEnvironment::extractFields(const search::index::Schema &schema)
+{
+    typedef search::index::Schema::Field SchemaField;
+    for (uint32_t i = 0; i < schema.getNumAttributeFields(); ++i) {
+        const SchemaField &field = schema.getAttributeField(i);
+        search::fef::FieldInfo fieldInfo(search::fef::FieldType::ATTRIBUTE,
+                                         convertCollectionType(field.getCollectionType()),
+                                         field.getName(), _fields.size());
+        fieldInfo.set_data_type(field.getDataType());
+        _fieldNames[field.getName()] = _fields.size();
+        _fields.push_back(fieldInfo);
+    }
+    for (uint32_t i = 0; i < schema.getNumIndexFields(); ++i) {
+        const SchemaField &field = schema.getIndexField(i);
+        search::fef::FieldInfo fieldInfo(search::fef::FieldType::INDEX,
+                                         convertCollectionType(field.getCollectionType()),
+                                         field.getName(), _fields.size());
+        fieldInfo.set_data_type(field.getDataType());
+        if (indexproperties::IsFilterField::check(
+                    _properties, field.getName()))
+        {
+            fieldInfo.setFilter(true);
+        }
+        FieldNameMap::const_iterator itr = _fieldNames.find(field.getName());
+        if (itr != _fieldNames.end()) { // override the attribute field
+            FieldInfo shadow_field(fieldInfo.type(),
+                                   fieldInfo.collection(),
+                                   fieldInfo.name(),
+                                   itr->second);
+            shadow_field.set_data_type(fieldInfo.get_data_type());
+            shadow_field.addAttribute(); // tell ranking about the shadowed attribute
+            _fields[itr->second] = shadow_field;
+        } else {
+            _fieldNames[field.getName()] = _fields.size();
+            _fields.push_back(fieldInfo);
+        }
+    }
+
+    //TODO: This is a kludge to get [documentmetastore] searchable
+    {
+        search::fef::FieldInfo fieldInfo(search::fef::FieldType::HIDDEN_ATTRIBUTE,
+                                         search::fef::CollectionType::SINGLE,
+                                         DocumentMetaStore::getFixedName(),
+                                         _fields.size());
+        fieldInfo.set_data_type(FieldInfo::DataType::RAW);
+        fieldInfo.setFilter(true);
+        _fieldNames[DocumentMetaStore::getFixedName()] = _fields.size();
+        _fields.push_back(fieldInfo);
+    }
+}
+
+IndexEnvironment::IndexEnvironment(const search::index::Schema &schema,
+                                   const search::fef::Properties &props)
+    : _tableManager(),
+      _properties(props),
+      _fieldNames(),
+      _fields(),
+      _motivation(UNKNOWN)
+{
+    _tableManager.addFactory(
+            search::fef::ITableFactory::SP(
+                    new search::fef::FunctionTableFactory(256)));
+    extractFields(schema);
+}
+
+const search::fef::Properties &
+IndexEnvironment::getProperties() const
+{
+    return _properties;
+}
+
+uint32_t
+IndexEnvironment::getNumFields() const
+{
+    return _fields.size();
+}
+
+const search::fef::FieldInfo *
+IndexEnvironment::getField(uint32_t id) const
+{
+    if (id < _fields.size()) {
+        return &_fields[id];
+    }
+    return 0;
+}
+
+const search::fef::FieldInfo *
+IndexEnvironment::getFieldByName(const string &name) const
+{
+    typedef std::map<string, uint32_t>::const_iterator ITR;
+    ITR pos = _fieldNames.find(name);
+    if (pos == _fieldNames.end()) {
+        return 0;
+    }
+    return getField(pos->second);
+}
+
+const search::fef::ITableManager &
+IndexEnvironment::getTableManager() const
+{
+    return _tableManager;
+}
+
+IIndexEnvironment::FeatureMotivation
+IndexEnvironment::getFeatureMotivation() const
+{
+    return _motivation;
+}
+
+void
+IndexEnvironment::hintFeatureMotivation(FeatureMotivation motivation) const
+{
+    _motivation = motivation;
+}
+
+void
+IndexEnvironment::hintFieldAccess(uint32_t fieldId) const
+{
+    (void) fieldId;
+}
+
+void
+IndexEnvironment::hintAttributeAccess(const string &name) const
+{
+    (void) name;
+}
+
+IndexEnvironment::~IndexEnvironment()
+{
+}
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h
new file mode 100644
index 00000000000..114f1ea7127
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/fef/fieldinfo.h>
+#include <vespa/searchlib/fef/iindexenvironment.h>
+#include <vespa/searchlib/fef/properties.h>
+#include <vespa/searchlib/fef/tablemanager.h>
+#include <vespa/searchcommon/common/schema.h>
+
+namespace proton {
+namespace matching {
+
+/**
+ * Index environment implementation for the proton matching pipeline.
+ **/
+class IndexEnvironment : public search::fef::IIndexEnvironment
+{
+private:
+    typedef std::map<string, uint32_t> FieldNameMap;
+    search::fef::TableManager           _tableManager;
+    search::fef::Properties             _properties;
+    FieldNameMap                        _fieldNames;
+    std::vector<search::fef::FieldInfo> _fields;
+    mutable FeatureMotivation           _motivation;
+
+    /**
+     * Extract field information from the given schema and populate
+     * this index environment.
+     **/
+    void extractFields(const search::index::Schema &schema);
+
+public:
+    /**
+     * Sets up this index environment based on the given schema and
+     * properties.
+     *
+     * @param schema the index schema
+     * @param props config
+     **/
+    IndexEnvironment(const search::index::Schema &schema,
+                     const search::fef::Properties &props);
+
+    // inherited from search::fef::IIndexEnvironment
+    virtual const search::fef::Properties &getProperties() const;
+
+    // inherited from search::fef::IIndexEnvironment
+    virtual uint32_t getNumFields() const;
+
+    // inherited from search::fef::IIndexEnvironment
+    virtual const search::fef::FieldInfo *getField(uint32_t id) const;
+
+    // inherited from search::fef::IIndexEnvironment
+    virtual const search::fef::FieldInfo *
+    getFieldByName(const string &name) const;
+
+    // inherited from search::fef::IIndexEnvironment
+    virtual const search::fef::ITableManager &getTableManager() const;
+
+    virtual FeatureMotivation getFeatureMotivation() const override;
+
+    // inherited from search::fef::IIndexEnvironment
+    virtual void hintFeatureMotivation(FeatureMotivation motivation) const;
+
+    // inherited from search::fef::IIndexEnvironment
+    virtual void hintFieldAccess(uint32_t fieldId) const;
+
+    // inherited from search::fef::IIndexEnvironment
+    virtual void hintAttributeAccess(const string &name) const;
+
+    virtual ~IndexEnvironment();
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h
new file mode 100644
index 00000000000..5d018cd5f41
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/queryeval/searchable.h>
+
+#include <memory>
+
+namespace proton {
+namespace matching {
+
+/**
+ * Interface used to expose searchable data to the matching
+ * pipeline. Ownership of the objects exposed through this interface
+ * is handled by the implementation. Cleanup is triggered by deleting
+ * the context interface. All searchable attributes are exposed
+ * through a single instance of the Searchable interface. Indexed
+ * fields are exposed as multiple Searchable instances that are
+ * assigned separate source ids. A source selector is used to
+ * determine which source should be used for each document.
+ **/
+class ISearchContext
+{
+public:
+    /**
+     * Convenience typedef for an auto pointer to this interface.
+     **/
+    typedef std::unique_ptr<ISearchContext> UP;
+
+    typedef search::queryeval::Searchable      Searchable;
+
+    /**
+     * Obtain the index fields searchable.
+     *
+     * @return index fields searchable.
+     **/
+    virtual Searchable &getIndexes() = 0;
+
+    /**
+     * Obtain the attribute fields searchable.
+     *
+     * @return attribute fields searchable.
+     **/
+    virtual Searchable &getAttributes() = 0;
+
+    /**
+     * Obtain the limit value for local document ids. This value is
+     * larger than all local docids that are currently in use. It will
+     * be used both to terminate matching and as an estimate on the
+     * total number of documents.
+     *
+     * @return local document id limit value
+     **/
+    virtual uint32_t getDocIdLimit() = 0;
+
+    /**
+     * Deleting the context will trigger cleanup in the
+     * implementation.
+     **/
+    virtual ~ISearchContext() {}
+};
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/isessioncachepruner.h b/searchcore/src/vespa/searchcore/proton/matching/isessioncachepruner.h
new file mode 100644
index 00000000000..710329680bf
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/isessioncachepruner.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fastos/timestamp.h>
+
+namespace proton {
+namespace matching {
+
+struct ISessionCachePruner {
+    virtual ~ISessionCachePruner() {}
+
+    virtual void pruneTimedOutSessions(fastos::TimeStamp currentTime) = 0;
+};
+
+}  // namespace proton::matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_context.h b/searchcore/src/vespa/searchcore/proton/matching/match_context.h
new file mode 100644
index 00000000000..a37e08a9bc8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_context.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "isearchcontext.h"
+#include <vespa/searchcommon/attribute/iattributecontext.h>
+#include <memory>
+
+namespace proton {
+namespace matching {
+
+class MatchContext {
+    search::attribute::IAttributeContext::UP _attrCtx;
+    ISearchContext::UP _searchCtx;
+
+public:
+    typedef std::unique_ptr<MatchContext> UP;
+
+    MatchContext(search::attribute::IAttributeContext::UP attrCtx,
+                 ISearchContext::UP searchCtx)
+        : _attrCtx(std::move(attrCtx)),
+          _searchCtx(std::move(searchCtx)) {
+        assert(_attrCtx.get());
+        assert(_searchCtx.get());
+    }
+
+    search::attribute::IAttributeContext &getAttributeContext() const
+    { return *_attrCtx; }
+    ISearchContext &getSearchContext() const { return *_searchCtx; }
+    void releaseEnumGuards() { _attrCtx->releaseEnumGuards(); }
+};
+
+}  // namespace proton::matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp
new file mode 100644
index 00000000000..5ca7915ab3a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "match_loop_communicator.h"
+#include <vespa/vespalib/util/priority_queue.h>
+#include <algorithm>
+
+namespace proton {
+namespace matching {
+
+void
+MatchLoopCommunicator::EstimateMatchFrequency::mingle()
+{
+    double freqSum = 0.0;
+    for (size_t i = 0; i < size(); ++i) {
+        if (in(i).docs > 0) {
+            double h = in(i).hits;
+            double d = in(i).docs;
+            freqSum += h/d;
+        }
+    }
+    double freq = freqSum / size();
+    for (size_t i = 0; i < size(); ++i) {
+        out(i) = freq;
+    }
+}
+
+void
+MatchLoopCommunicator::SelectBest::mingle()
+{
+    vespalib::PriorityQueue<uint32_t, SelectCmp> queue(SelectCmp(*this));
+    for (size_t i = 0; i < size(); ++i) {
+        if (!in(i).empty()) {
+            queue.push(i);
+        }
+    }
+    for (size_t picked = 0; picked < topN && !queue.empty(); ++picked) {
+        uint32_t i = queue.front();
+        if (in(i).size() > ++out(i)) {
+            queue.adjust();
+        } else {
+            queue.pop_front();
+        }
+    }
+}
+
+void
+MatchLoopCommunicator::RangeCover::mingle()
+{
+    size_t i = 0;
+    while (i < size() && (!in(i).first.isValid() || !in(i).second.isValid())) {
+        ++i;
+    }
+    if (i < size()) {
+        RangePair result = in(i++);
+        for (; i < size(); ++i) {
+            if (in(i).first.isValid() && in(i).second.isValid()) {
+                result.first.low = std::min(result.first.low, in(i).first.low);
+                result.first.high = std::max(result.first.high, in(i).first.high);
+                result.second.low = std::min(result.second.low, in(i).second.low);
+                result.second.high = std::max(result.second.high, in(i).second.high);
+            }
+        }
+        for (size_t j = 0; j < size(); ++j) {
+            out(j) = result;
+        }
+    }
+}
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h
new file mode 100644
index 00000000000..61ec70c2a96
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_match_loop_communicator.h"
+#include <vespa/vespalib/util/rendezvous.h>
+
+namespace proton {
+namespace matching {
+
+class MatchLoopCommunicator : public IMatchLoopCommunicator
+{
+private:
+    struct EstimateMatchFrequency : vespalib::Rendezvous<Matches, double> {
+        EstimateMatchFrequency(size_t n)
+            : vespalib::Rendezvous<Matches, double>(n) {}
+        virtual void mingle();
+    };
+    struct SelectBest : vespalib::Rendezvous<std::vector<feature_t>, size_t> {
+        size_t topN;
+        SelectBest(size_t n, size_t topN_in)
+            : vespalib::Rendezvous<std::vector<feature_t>, size_t>(n), topN(topN_in) {}
+        virtual void mingle();
+        bool cmp(const uint32_t &a, const uint32_t &b) {
+            return (in(a)[out(a)] > in(b)[out(b)]);
+        }
+    };
+    struct SelectCmp {
+        SelectBest &sb;
+        SelectCmp(SelectBest &sb_in) : sb(sb_in) {}
+        bool operator()(const uint32_t &a, const uint32_t &b) const {
+            return (sb.cmp(a, b));
+        }
+    };
+    struct RangeCover : vespalib::Rendezvous<RangePair, RangePair> {
+        RangeCover(size_t n)
+            : vespalib::Rendezvous<RangePair, RangePair>(n) {}
+        virtual void mingle();
+    };
+    EstimateMatchFrequency _estimate_match_frequency;
+    SelectBest             _selectBest;
+    RangeCover             _rangeCover;
+
+public:
+    MatchLoopCommunicator(size_t threads, size_t topN)
+        : _estimate_match_frequency(threads), _selectBest(threads, topN), _rangeCover(threads) {}
+
+    virtual double estimate_match_frequency(const Matches &matches) {
+        return _estimate_match_frequency.rendezvous(matches);
+    }
+    virtual size_t selectBest(const std::vector<feature_t> &sortedScores) {
+        return _selectBest.rendezvous(sortedScores);
+    }
+    virtual RangePair rangeCover(const RangePair &ranges) {
+        return _rangeCover.rendezvous(ranges);
+    }
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp
new file mode 100644
index 00000000000..a3069177157
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp
@@ -0,0 +1,142 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.match_master");
+#include "match_master.h"
+
+#include "docid_range_scheduler.h"
+#include "match_loop_communicator.h"
+#include "match_thread.h"
+#include "matching_stats.h"
+#include <memory>
+#include <vespa/searchlib/common/featureset.h>
+#include <vespa/searchlib/common/resultset.h>
+
+namespace proton {
+namespace matching {
+
+using namespace search::fef;
+using search::queryeval::SearchIterator;
+using search::FeatureSet;
+
+namespace {
+
+struct TimedMatchLoopCommunicator : IMatchLoopCommunicator {
+    IMatchLoopCommunicator &communicator;
+    fastos::StopWatch rerank_time;
+    TimedMatchLoopCommunicator(IMatchLoopCommunicator &com) : communicator(com) {}
+    virtual double estimate_match_frequency(const Matches &matches) {
+        return communicator.estimate_match_frequency(matches);
+    }
+    virtual size_t selectBest(const std::vector<feature_t> &sortedScores) {
+        size_t result = communicator.selectBest(sortedScores);
+        rerank_time.start();
+        return result;
+    }
+    virtual RangePair rangeCover(const RangePair &ranges) {
+        RangePair result = communicator.rangeCover(ranges);
+        rerank_time.stop();
+        return result;
+    }
+};
+
+DocidRangeScheduler::UP
+createScheduler(uint32_t numThreads, uint32_t numSearchPartitions, uint32_t numDocs)
+{
+    if (numSearchPartitions == 0) {
+        return std::make_unique<AdaptiveDocidRangeScheduler>(numThreads, 1, numDocs);
+    }
+    if (numSearchPartitions <= numThreads) {
+        return std::make_unique<PartitionDocidRangeScheduler>(numThreads, numDocs);
+    }
+    return std::make_unique<TaskDocidRangeScheduler>(numThreads, numSearchPartitions, numDocs);
+}
+
+} // namespace proton::matching::<unnamed>
+
+ResultProcessor::Result::UP
+MatchMaster::match(const MatchParams &params,
+                   vespalib::ThreadBundle &threadBundle,
+                   const MatchToolsFactory &matchToolsFactory,
+                   ResultProcessor &resultProcessor,
+                   uint32_t distributionKey,
+                   uint32_t numSearchPartitions)
+{
+    fastos::StopWatch query_latency_time;
+    query_latency_time.start();
+    vespalib::DualMergeDirector mergeDirector(threadBundle.size());
+    MatchLoopCommunicator communicator(threadBundle.size(), params.heapSize);
+    TimedMatchLoopCommunicator timedCommunicator(communicator);
+    DocidRangeScheduler::UP scheduler = createScheduler(threadBundle.size(), numSearchPartitions, params.numDocs);
+
+    std::vector<MatchThread::UP> threadState;
+    std::vector<vespalib::Runnable*> targets;
+    for (size_t i = 0; i < threadBundle.size(); ++i) {
+        IMatchLoopCommunicator &com =
+            (i == 0)?
+            static_cast<IMatchLoopCommunicator&>(timedCommunicator) :
+            static_cast<IMatchLoopCommunicator&>(communicator);
+        threadState.emplace_back(std::make_unique<MatchThread>(i, threadBundle.size(),
+                        params, matchToolsFactory, com, *scheduler,
+                        resultProcessor, mergeDirector, distributionKey));
+        targets.push_back(threadState.back().get());
+    }
+    resultProcessor.prepareThreadContextCreation(threadBundle.size());
+    threadBundle.run(targets);
+    ResultProcessor::Result::UP reply = resultProcessor.makeReply();
+    query_latency_time.stop();
+    double query_time_s = query_latency_time.elapsed().sec();
+    double rerank_time_s = timedCommunicator.rerank_time.elapsed().sec();
+    double match_time_s = 0.0;
+    for (size_t i = 0; i < threadState.size(); ++i) {
+        match_time_s = std::max(match_time_s, threadState[i]->get_match_time());
+        _stats.merge_partition(threadState[i]->get_thread_stats(), i);
+    }
+    _stats.queryLatency(query_time_s);
+    _stats.matchTime(match_time_s - rerank_time_s);
+    _stats.rerankTime(rerank_time_s);
+    _stats.groupingTime(query_time_s - match_time_s);
+    _stats.queries(1);
+    if (matchToolsFactory.match_limiter().was_limited()) {
+        _stats.limited_queries(1);        
+    }
+    return reply;
+}
+
+FeatureSet::SP
+MatchMaster::getFeatureSet(const MatchToolsFactory &matchToolsFactory,
+                           const std::vector<uint32_t> &docs, bool summaryFeatures)
+{
+    MatchTools::UP matchTools = matchToolsFactory.createMatchTools();
+    RankProgram::UP rankProgram = summaryFeatures ? matchTools->summary_program() :
+            matchTools->dump_program();
+
+    std::vector<vespalib::string> featureNames;
+    std::vector<search::fef::FeatureHandle> handles;
+    rankProgram->get_seed_handles(featureNames, handles);
+    FeatureSet::SP retval(new FeatureSet(featureNames, docs.size()));
+    FeatureSet &fs = *retval.get();
+
+    SearchIterator::UP search = matchTools->createSearch(rankProgram->match_data());
+    search->initFullRange();
+    for (uint32_t i = 0; i < docs.size(); ++i) {
+        if (search->seek(docs[i])) {
+            uint32_t docId = search->getDocId();
+            search->unpack(docId);
+            rankProgram->run(docId);
+            search::feature_t * f = fs.getFeaturesByIndex(
+                    fs.addDocId(docId));
+            for (uint32_t j = 0; j < featureNames.size(); ++j) {
+                f[j] = *rankProgram->match_data().resolveFeature(handles[j]);
+            }
+        } else {
+            LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. "
+                "Skipping hit", docs[i]);
+        }
+    }
+    return retval;
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.h b/searchcore/src/vespa/searchcore/proton/matching/match_master.h
new file mode 100644
index 00000000000..286937f79c7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/clock.h>
+#include <vespa/vespalib/util/thread_bundle.h>
+#include <vespa/searchlib/engine/searchreply.h>
+#include <vespa/searchlib/common/featureset.h>
+#include "match_tools.h"
+#include "result_processor.h"
+#include "match_params.h"
+#include "matching_stats.h"
+
+namespace proton {
+namespace matching {
+
+/**
+ * Handles overall matching and keeps track of match threads.
+ **/
+class MatchMaster
+{
+private:
+    MatchingStats _stats;
+
+public:
+    const MatchingStats &getStats() const { return _stats; }
+    ResultProcessor::Result::UP match(const MatchParams &params,
+                                          vespalib::ThreadBundle &threadBundle,
+                                          const MatchToolsFactory &matchToolsFactory,
+                                          ResultProcessor &resultProcessor,
+                                          uint32_t distributionKey,
+                                          uint32_t numSearchPartitions);
+
+    static search::FeatureSet::SP
+    getFeatureSet(const MatchToolsFactory &matchToolsFactory,
+                  const std::vector<uint32_t> &docs, bool summaryFeatures);
+};
+
+} // namespace proton::matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp
new file mode 100644
index 00000000000..a52bdac59a1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "match_params.h"
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+uint32_t computeArraySize(uint32_t hitsPlussOffset, uint32_t heapSize, uint32_t arraySize)
+{
+    return std::max(hitsPlussOffset, std::max(heapSize, arraySize));
+}
+
+}
+
+MatchParams::MatchParams(uint32_t          numDocs_in,
+                         uint32_t          heapSize_in,
+                         uint32_t          arraySize_in,
+                         search::feature_t rankDropLimit_in,
+                         uint32_t          offset_in,
+                         uint32_t          hits_in,
+                         bool              hasFinalRank,
+                         bool              needRanking)
+    : numDocs(numDocs_in),
+      heapSize((hasFinalRank && needRanking) ? heapSize_in : 0),
+      arraySize((needRanking && ((heapSize_in + arraySize_in) > 0))
+                ? computeArraySize(hits_in + offset_in, heapSize, arraySize_in)
+                : 0),
+      offset(offset_in),
+      hits(hits_in),
+      rankDropLimit(rankDropLimit_in)
+{
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_params.h b/searchcore/src/vespa/searchcore/proton/matching/match_params.h
new file mode 100644
index 00000000000..bdcb650d306
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_params.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/fef/fef.h>
+
+namespace proton {
+namespace matching {
+
+/**
+ * Numeric matching parameters. Some of these comes from the config,
+ * others from the request.
+ **/
+struct MatchParams {
+    const uint32_t          numDocs;
+    const uint32_t          heapSize;
+    const uint32_t          arraySize;
+    const uint32_t          offset;
+    const uint32_t          hits;
+    const search::feature_t rankDropLimit;
+
+    MatchParams(uint32_t          numDocs_in,
+                uint32_t          heapSize_in,
+                uint32_t          arraySize_in,
+                search::feature_t rankDropLimit_in,
+                uint32_t          offset_in,
+                uint32_t          hits_in,
+                bool              hasFinalRank,
+                bool              needRanking=true);
+};
+
+} // namespace proton::matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_phase_limit_calculator.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limit_calculator.cpp
new file mode 100644
index 00000000000..b15358ee5e1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limit_calculator.cpp
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "match_phase_limit_calculator.h"
+
+namespace proton {
+namespace matching {
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_phase_limit_calculator.h b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limit_calculator.h
new file mode 100644
index 00000000000..b824eae7bac
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limit_calculator.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "isearchcontext.h"
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+
+namespace proton {
+namespace matching {
+
+/**
+ * This class is used for all calculations related to limiting the
+ * number of results produced during matching based on the 'max-hits'
+ * configuration in the 'match-phase' part of the rank profile in the
+ * search definition.
+ **/
+class MatchPhaseLimitCalculator
+{
+private:
+    const size_t _max_hits;
+    const size_t _min_groups;
+    const size_t _sample_hits;
+
+public:
+    /**
+     * @param max_hits the number of hits you want
+     * @param min_groups the minimum number of diversity groups you want
+     * @param sample fraction of max_hits to be used as sample size before performing match phase limiting
+     */
+    MatchPhaseLimitCalculator(size_t max_hits, size_t min_groups, double sample) :
+        _max_hits(max_hits),
+        _min_groups(std::max(size_t(1), min_groups)),
+        _sample_hits(max_hits * sample)
+    {}
+    size_t sample_hits_per_thread(size_t num_threads) const {
+        return std::max(size_t(1), std::max(128 / num_threads, _sample_hits / num_threads));
+    }
+    size_t wanted_num_docs(double hit_rate) const {
+        return std::min((double)0x7fffFFFF, std::max(128.0, _max_hits / hit_rate));
+    }
+    size_t estimated_hits(double hit_rate, size_t num_docs) const {
+        return (size_t) (hit_rate * num_docs);
+    }
+    size_t max_group_size(size_t wanted_num_docs_in) const {
+        return (wanted_num_docs_in / _min_groups);
+    }
+};
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.cpp
new file mode 100644
index 00000000000..bbec59e521b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.cpp
@@ -0,0 +1,150 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "match_phase_limiter.h"
+#include <vespa/searchlib/queryeval/andsearchstrict.h>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".proton.matching.match_phase_limiter");
+
+using search::queryeval::SearchIterator;
+using search::queryeval::Searchable;
+using search::queryeval::IRequestContext;
+using search::queryeval::AndSearchStrict;
+using search::queryeval::NoUnpack;
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+template<bool PRE_FILTER>
+class LimitedSearchT : public LimitedSearch {
+public:
+    LimitedSearchT(SearchIterator::UP limiter, SearchIterator::UP search) :
+        LimitedSearch(std::move(PRE_FILTER ? limiter : search),
+                      std::move(PRE_FILTER ? search : limiter))
+    {
+    }
+    void doUnpack(uint32_t docId) override { 
+        if (PRE_FILTER) {
+            getSecond().doUnpack(docId);
+        } else {
+            getFirst().doUnpack(docId);
+        }
+    }
+};
+
+} // namespace proton::matching::<unnamed>
+
+void
+LimitedSearch::doSeek(uint32_t docId)
+{
+    
+    uint32_t currentId(docId);
+    for (; !isAtEnd(currentId); currentId++) {
+        _first->seek(currentId);
+        currentId = _first->getDocId();
+        if (isAtEnd(currentId)) {
+            break;
+        }
+        if (_second->seek(currentId)) {
+            break;
+        }
+    }
+    setDocId(currentId);
+}
+
+void
+LimitedSearch::initRange(uint32_t begin, uint32_t end) {
+    SearchIterator::initRange(begin, end);
+    getFirst().initRange(begin, end);
+    getSecond().initRange(begin, end);
+}
+
+void
+LimitedSearch::visitMembers(vespalib::ObjectVisitor &visitor) const
+{
+    visit(visitor, "first", getFirst());
+    visit(visitor, "second", getSecond());
+}
+
+MatchPhaseLimiter::MatchPhaseLimiter(uint32_t docIdLimit,
+                                     Searchable &searchable_attributes,
+                                     IRequestContext & requestContext,
+                                     const vespalib::string &attribute_name,
+                                     size_t max_hits, bool descending,
+                                     double max_filter_coverage,
+                                     double samplePercentage, double postFilterMultiplier,
+                                     const vespalib::string &diversity_attribute,
+                                     uint32_t diversity_min_groups,
+                                     double diversify_cutoff_factor,
+                                     AttributeLimiter::DiversityCutoffStrategy diversity_cutoff_strategy)
+    : _postFilterMultiplier(postFilterMultiplier),
+      _maxFilterCoverage(max_filter_coverage),
+      _calculator(max_hits, diversity_min_groups, samplePercentage),
+      _limiter_factory(searchable_attributes, requestContext, attribute_name, descending,
+                       diversity_attribute, diversify_cutoff_factor, diversity_cutoff_strategy),
+      _coverage(docIdLimit)
+{
+}
+
+namespace {
+
+template <bool PRE_FILTER>
+SearchIterator::UP
+do_limit(AttributeLimiter &limiter_factory, SearchIterator::UP search,
+         size_t wanted_num_docs, size_t max_group_size,
+         uint32_t current_id, uint32_t end_id)
+{
+    SearchIterator::UP limiter = limiter_factory.create_search(wanted_num_docs, max_group_size, PRE_FILTER);
+    limiter = search->andWith(std::move(limiter), wanted_num_docs);
+    if (limiter) {
+        search.reset(new LimitedSearchT<PRE_FILTER>(std::move(limiter), std::move(search)));
+    }
+    search->initRange(current_id + 1, end_id);
+    return search;
+}
+
+} // namespace proton::matching::<unnamed>
+
+SearchIterator::UP
+MatchPhaseLimiter::maybe_limit(SearchIterator::UP search,
+                               double match_freq, size_t num_docs)
+{
+    size_t wanted_num_docs = _calculator.wanted_num_docs(match_freq);
+    size_t max_filter_docs = static_cast<size_t>(num_docs * _maxFilterCoverage);
+    size_t upper_limited_corpus_size = std::min(num_docs, max_filter_docs);
+    if (upper_limited_corpus_size <= wanted_num_docs) {
+        LOG(debug, "Will not limit ! maybe_limit(hit_rate=%g, num_docs=%ld, max_filter_docs=%ld) = wanted_num_docs=%ld",
+            match_freq, num_docs, max_filter_docs, wanted_num_docs);
+        return search;
+    }
+    uint32_t current_id = search->getDocId();
+    uint32_t end_id = search->getEndId();
+    size_t total_query_hits = _calculator.estimated_hits(match_freq, num_docs);
+    size_t max_group_size = _calculator.max_group_size(wanted_num_docs);
+    bool use_pre_filter = (wanted_num_docs < (total_query_hits * _postFilterMultiplier));
+    LOG(debug, "Will do %s filter :  maybe_limit(hit_rate=%g, num_docs=%zu, max_filter_docs=%ld) = wanted_num_docs=%zu,"
+        " max_group_size=%zu, current_docid=%u, end_docid=%u, total_query_hits=%ld",
+        use_pre_filter ? "pre" : "post", match_freq, num_docs, max_filter_docs, wanted_num_docs,
+        max_group_size, current_id, end_id, total_query_hits);
+    return (use_pre_filter)
+        ? do_limit<true>(_limiter_factory, std::move(search), wanted_num_docs, max_group_size, current_id, end_id)
+        : do_limit<false>(_limiter_factory, std::move(search), wanted_num_docs, max_group_size, current_id, end_id);
+}
+
+void
+MatchPhaseLimiter::updateDocIdSpaceEstimate(size_t searchedDocIdSpace, size_t remainingDocIdSpace)
+{
+    _coverage.update(searchedDocIdSpace, remainingDocIdSpace, _limiter_factory.getEstimatedHits());
+}
+
+size_t
+MatchPhaseLimiter::getDocIdSpaceEstimate() const
+{
+    return _coverage.getEstimate();
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.h b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.h
new file mode 100644
index 00000000000..7763aeb3723
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.h
@@ -0,0 +1,127 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "match_phase_limit_calculator.h"
+#include "attribute_limiter.h"
+
+#include <vespa/searchlib/queryeval/searchable.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <atomic>
+
+namespace proton {
+namespace matching {
+
+class LimitedSearch : public search::queryeval::SearchIterator {
+public:
+    LimitedSearch(SearchIterator::UP first, SearchIterator::UP second) :
+        _first(std::move(first)),
+        _second(std::move(second))
+    {
+    }
+    void doSeek(uint32_t docId) override;
+    void initRange(uint32_t begin, uint32_t end) override;
+    void visitMembers(vespalib::ObjectVisitor &visitor) const override;
+    const SearchIterator &  getFirst() const { return *_first; }
+    const SearchIterator & getSecond() const { return *_second; }
+    SearchIterator &  getFirst() { return *_first; }
+    SearchIterator & getSecond() { return *_second; }
+private:
+    SearchIterator::UP _first;
+    SearchIterator::UP _second;
+};
+
+/**
+ * Interface defining how we intend to use the match phase limiter
+ * functionality. The first step is to check whether we should enable
+ * this functionality at all. If enabled; we need to match some hits
+ * in each match thread for estimation purposes. The total number of
+ * matches (hits) and the total document space searched (docs) are
+ * aggregated across all match threads and each match thread will use
+ * the maybe_limit function to possibly augment its iterator tree to
+ * limit the number of matches.
+ **/
+struct MaybeMatchPhaseLimiter {
+    typedef search::queryeval::SearchIterator SearchIterator;
+    typedef std::unique_ptr<MaybeMatchPhaseLimiter> UP;
+    virtual bool is_enabled() const = 0;
+    virtual bool was_limited() const = 0;
+    virtual size_t sample_hits_per_thread(size_t num_threads) const = 0;
+    virtual SearchIterator::UP maybe_limit(SearchIterator::UP search, double match_freq, size_t num_docs) = 0;
+    virtual void updateDocIdSpaceEstimate(size_t searchedDocIdSpace, size_t remainingDocIdSpace) = 0;
+    virtual size_t getDocIdSpaceEstimate() const = 0;
+    virtual ~MaybeMatchPhaseLimiter() {}
+};
+
+/**
+ * This class is used when match phase limiting is not configured.
+ **/
+struct NoMatchPhaseLimiter : MaybeMatchPhaseLimiter {
+    bool is_enabled() const override { return false; }
+    bool was_limited() const override { return false; }
+    size_t sample_hits_per_thread(size_t) const override { return 0; }
+    SearchIterator::UP maybe_limit(SearchIterator::UP search, double, size_t) override {
+        return search;
+    }
+    void updateDocIdSpaceEstimate(size_t, size_t) override { }
+    size_t getDocIdSpaceEstimate() const override { return std::numeric_limits<size_t>::max(); }
+};
+
+/**
+ * This class is is used when rank phase limiting is configured.
+ **/
+class MatchPhaseLimiter : public MaybeMatchPhaseLimiter
+{
+private:
+    class Coverage {
+    public:
+        Coverage(uint32_t docIdLimit) :
+            _docIdLimit(docIdLimit),
+            _searched(0)
+        { }
+        void update(size_t searched, size_t remaining, ssize_t hits) {
+            if (hits >= 0) {
+                _searched += (searched + (hits*remaining)/_docIdLimit);
+            } else {
+                _searched += (searched + remaining);
+            }
+        }
+        uint32_t getEstimate() const { return _searched; }
+    private:
+        const uint32_t        _docIdLimit;
+        std::atomic<uint32_t> _searched;
+    };
+    const double              _postFilterMultiplier;
+    const double              _maxFilterCoverage;
+    MatchPhaseLimitCalculator _calculator;
+    AttributeLimiter          _limiter_factory;
+    Coverage                  _coverage;
+
+public:
+    MatchPhaseLimiter(uint32_t docIdLimit,
+                      search::queryeval::Searchable &searchable_attributes,
+                      search::queryeval::IRequestContext & requestContext,
+                      const vespalib::string &attribute_name,
+                      size_t max_hits, bool descending,
+                      double max_filter_coverage,
+                      double samplePercentage, double postFilterMultiplier,
+                      const vespalib::string &diversity_attribute,
+                      uint32_t diversity_min_groups,
+                      double diversify_cutoff_factor,
+                      AttributeLimiter::DiversityCutoffStrategy diversity_cutoff_strategy);
+    bool is_enabled() const override { return true; }
+    bool was_limited() const override { return _limiter_factory.was_used(); }
+    size_t sample_hits_per_thread(size_t num_threads) const override {
+        return _calculator.sample_hits_per_thread(num_threads);
+    }
+    SearchIterator::UP maybe_limit(SearchIterator::UP search, double match_freq, size_t num_docs) override;
+    void updateDocIdSpaceEstimate(size_t searchedDocIdSpace, size_t remainingDocIdSpace) override;
+    size_t getDocIdSpaceEstimate() const override;
+};
+
+} // namespace proton::matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
new file mode 100644
index 00000000000..e6bf8f616b7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
@@ -0,0 +1,350 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "match_thread.h"
+#include "match_tools.h"
+#include "handlerecorder.h"
+#include "i_match_loop_communicator.h"
+#include "matching_stats.h"
+#include "document_scorer.h"
+#include <memory>
+#include <vespa/searchlib/common/featureset.h>
+#include <vespa/searchlib/common/resultset.h>
+#include <vespa/searchlib/fef/fef.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/searchlib/queryeval/hitcollector.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/multibitvectoriterator.h>
+#include <vespa/searchlib/queryeval/andnotsearch.h>
+#include <vespa/vespalib/util/clock.h>
+#include <vespa/vespalib/util/closure.h>
+#include <vespa/vespalib/util/thread_bundle.h>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".proton.matching.match_thread");
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+using search::fef::FeatureHandle;
+using search::fef::IllegalHandle;
+using search::fef::RankProgram;
+
+const double *get_score_feature(const RankProgram &rankProgram) {
+    std::vector<vespalib::string> featureNames;
+    std::vector<FeatureHandle> featureHandles;
+    rankProgram.get_seed_handles(featureNames, featureHandles);
+    assert(featureNames.size() == 1);
+    assert(featureHandles.size() == 1);
+    return rankProgram.match_data().resolveFeature(featureHandles.front());
+}
+
+}
+
+using vespalib::Doom;
+using search::queryeval::OptimizedAndNotForBlackListing;
+
+template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work>
+void
+MatchThread::match_loop(MatchTools &matchTools, IteratorT search,
+                        RankProgram &ranking, HitCollector &hits)
+{
+    const Doom &doom = matchTools.doom();
+    const double *score_feature = do_rank ? get_score_feature(ranking) : nullptr;
+    uint32_t matches = 0;
+    uint32_t matches_limit = (do_limit) ? matchTools.match_limiter().sample_hits_per_thread(num_threads) : 0;
+    IdleObserver idle_observer = scheduler.make_idle_observer();
+    for (DocidRange docid_range = scheduler.first_range(thread_id);
+         !docid_range.empty();
+         docid_range = scheduler.next_range(thread_id))
+    {
+        search->initRange(docid_range.begin, docid_range.end);
+        uint32_t docId = search->seekFirst(docid_range.begin);
+        while ((docId < docid_range.end) && !doom.doom()) {
+            if (do_rank) {
+                search->unpack(docId);
+                ranking.run(docId);
+                double score = *score_feature;
+                // convert NaN and Inf scores to -Inf
+                if (__builtin_expect(isnan(score) || isinf(score), false)) {
+                    score = -HUGE_VAL;
+                }
+                // invert test since default drop limit is -NaN (keep all hits)
+                if (!(score <= matchParams.rankDropLimit)) {
+                    hits.addHit(docId, score);
+                }
+            } else {
+                hits.addHit(docId, 0.0);
+            }
+            ++matches;
+            if (do_limit && matches == matches_limit) {
+                const size_t local_todo = (docid_range.end - docId - 1);
+                const size_t searchedSoFar = (scheduler.total_size(thread_id) - local_todo);
+                IMatchLoopCommunicator::Matches my_matches(matches, searchedSoFar);
+                WaitTimer count_matches_timer(wait_time_s);
+                double match_freq = communicator.estimate_match_frequency(my_matches);
+                const size_t global_todo = scheduler.unassigned_size();
+                count_matches_timer.done();
+                search.reset(matchTools.match_limiter().maybe_limit(SearchIterator::UP(search.release()),
+                                match_freq,
+                                matchParams.numDocs).release());
+                matchTools.match_limiter().updateDocIdSpaceEstimate(searchedSoFar, local_todo + (global_todo / num_threads));
+                LOG(debug, "Limit=%d has been reached at docid=%d which is after %zu docs.",
+                    matches, docId, searchedSoFar);
+                LOG(debug, "SearchIterator after limiter: %s", search->asString().c_str());
+                docId = search->seekFirst(docId + 1);
+            } else if (do_share_work && (idle_observer.get() > 0)) {
+                DocidRange todo(docId + 1, docid_range.end);
+                DocidRange my_work = scheduler.share_range(thread_id, todo);
+                if (my_work.end < todo.end) {
+                    docid_range = my_work;
+                    search->initRange(docid_range.begin, docid_range.end);
+                    docId = search->seekFirst(docid_range.begin);
+                } else {
+                    docId = search->seekNext(docId + 1);
+                }
+            } else {
+                docId = search->seekNext(docId + 1);
+            }
+        }
+    }
+    if (do_limit && matches < matches_limit) {
+        IMatchLoopCommunicator::Matches my_matches(matches, scheduler.total_size(thread_id));
+        WaitTimer count_matches_timer(wait_time_s);
+        LOG(debug, "Limit not reached (had %d) at docid=%d which is after %zu docs.",
+            matches, scheduler.total_span(thread_id).end, scheduler.total_size(thread_id));
+        communicator.estimate_match_frequency(my_matches); // for other threads
+        matchTools.match_limiter().updateDocIdSpaceEstimate(scheduler.total_size(thread_id), 0);
+        count_matches_timer.done();
+    }
+    thread_stats.docsMatched(matches);
+    if (do_rank) {
+        thread_stats.docsRanked(matches);
+    }
+}
+
+template <typename IteratorT, bool do_rank, bool do_limit>
+void
+MatchThread::match_loop_helper_2(MatchTools &matchTools, IteratorT search,
+                                 RankProgram &ranking, HitCollector &hits)
+{
+    if (scheduler.make_idle_observer().is_always_zero()) {
+        match_loop<IteratorT, do_rank, do_limit, false>(matchTools, std::move(search), ranking, hits);
+    } else {
+        match_loop<IteratorT, do_rank, do_limit, true>(matchTools, std::move(search), ranking, hits);
+    }
+}
+
+template <typename IteratorT, bool do_rank>
+void
+MatchThread::match_loop_helper(MatchTools &matchTools, IteratorT search,
+                               RankProgram &ranking, HitCollector &hits)
+{
+    if (matchTools.match_limiter().is_enabled()) {
+        match_loop_helper_2<IteratorT, do_rank, true>(matchTools, std::move(search), ranking, hits);
+    } else {
+        match_loop_helper_2<IteratorT, do_rank, false>(matchTools, std::move(search), ranking, hits);
+    }
+}
+
+using search::fef::TermFieldHandle;
+using search::fef::MatchData;
+
+class FastSeekWrapper
+{
+private:
+    typedef search::queryeval::SearchIterator SearchIterator;
+public:
+    FastSeekWrapper(SearchIterator::UP iterator)
+    {
+        reset(iterator.release());
+    }
+    void initRange(uint32_t begin_id, uint32_t end_id) {
+        _search->initRange(begin_id, end_id);
+    }
+    uint32_t seekFirst(uint32_t docId) {
+        return _search->seekFirst(docId);
+    }
+    uint32_t seekNext(uint32_t docId) {
+        return _search->seekFast(docId);
+    }
+    vespalib::string asString() const {
+        return _search->asString();
+    }
+    void unpack(uint32_t docId) {
+        _search->unpack(docId);
+    }
+    void reset(SearchIterator * search) {
+        _search.reset(&dynamic_cast<OptimizedAndNotForBlackListing &>(*search));
+    }
+    OptimizedAndNotForBlackListing * release() {
+        return _search.release();
+    }
+    FastSeekWrapper * operator ->() { return this; }
+private:
+    std::unique_ptr<OptimizedAndNotForBlackListing> _search;
+};
+
+MatchThread::MatchThread(size_t thread_id_in,
+                         size_t num_threads_in,
+                         const MatchParams &mp,
+                         const MatchToolsFactory &mtf,
+                         IMatchLoopCommunicator &com,
+                         DocidRangeScheduler &sched,
+                         ResultProcessor &rp,
+                         vespalib::DualMergeDirector &md,
+                         uint32_t distributionKey) :
+    thread_id(thread_id_in),
+    num_threads(num_threads_in),
+    matchParams(mp),
+    matchToolsFactory(mtf),
+    communicator(com),
+    scheduler(sched),
+    _distributionKey(distributionKey),
+    resultProcessor(rp),
+    mergeDirector(md),
+    resultContext(),
+    thread_stats(),
+    total_time_s(0.0),
+    match_time_s(0.0),
+    wait_time_s(0.0)
+{
+}
+
+search::ResultSet::UP
+MatchThread::findMatches(MatchTools &matchTools)
+{
+    const Doom &doom = matchTools.doom();
+    RankProgram::UP ranking = matchTools.first_phase_program();
+    SearchIterator::UP search = matchTools.createSearch(ranking->match_data());
+    LOG(debug, "SearchIterator: %s", search->asString().c_str());
+    search = search::queryeval::MultiBitVectorIteratorBase::optimize(std::move(search));
+    LOG(debug, "SearchIterator after MultiBitVectorIteratorBase::optimize(): %s", search->asString().c_str());
+    HitCollector hits(matchParams.numDocs, matchParams.arraySize, matchParams.heapSize);
+    if (matchTools.has_first_phase_rank() && ((matchParams.arraySize + matchParams.heapSize) != 0)) {
+        match_loop_helper<SearchIterator::UP, true>(matchTools, std::move(search), *ranking, hits);
+    } else {
+        if ((dynamic_cast<const OptimizedAndNotForBlackListing *>(search.get()) != 0) &&
+            ! matchTools.match_limiter().is_enabled())  // We cannot replace the iterator if we inline the loop.
+        {
+            match_loop_helper<FastSeekWrapper, false>(matchTools, FastSeekWrapper(std::move(search)), *ranking, hits);
+        } else {
+            match_loop_helper<SearchIterator::UP, false>(matchTools, std::move(search), *ranking, hits);
+        }
+    }
+    if (matchTools.has_second_phase_rank()) {
+        { // 2nd phase ranking
+            ranking = matchTools.second_phase_program();
+            search = matchTools.createSearch(ranking->match_data());
+            DocidRange docid_range = scheduler.total_span(thread_id);
+            search->initRange(docid_range.begin, docid_range.end);
+            auto sorted_scores = hits.getSortedHeapScores();
+            WaitTimer select_best_timer(wait_time_s);
+            size_t useHits = communicator.selectBest(sorted_scores);
+            select_best_timer.done();
+            DocumentScorer scorer(*ranking, *search);
+            uint32_t reRanked = hits.reRank(scorer, doom.doom() ? 0 : useHits);
+            thread_stats.docsReRanked(reRanked);
+        }
+        { // rank scaling
+            auto my_ranges = hits.getRanges();
+            WaitTimer range_cover_timer(wait_time_s);
+            auto ranges = communicator.rangeCover(my_ranges);
+            range_cover_timer.done();
+            hits.setRanges(ranges);
+        }
+    }
+    return hits.getResultSet();
+}
+
+void
+MatchThread::processResult(const Doom & doom, 
+                           search::ResultSet::UP result,
+                           ResultProcessor::Context &context)
+{
+    if (doom.doom()) return;
+    bool hasGrouping = (context.grouping.get() != 0);
+    if (context.sort->hasSortData() || hasGrouping) {
+        result->mergeWithBitOverflow();
+    }
+    if (doom.doom()) return;
+    size_t             totalHits = result->getNumHits();
+    search::RankedHit *hits      = result->getArray();
+    size_t             numHits   = result->getArrayUsed();
+    search::BitVector *bits  = result->getBitOverflow();
+    if (bits != nullptr && hits != nullptr) {
+        bits->andNotWithT(search::RankedHitIterator(hits, numHits));
+    }
+    if (doom.doom()) return;
+    if (hasGrouping) {
+        search::grouping::GroupingManager man(*context.grouping);
+        man.groupUnordered(hits, numHits, bits);
+    }
+    if (doom.doom()) return;
+    size_t sortLimit = hasGrouping ? numHits : context.result->maxSize();
+    context.sort->sorter->sortResults(hits, numHits, sortLimit);
+    if (doom.doom()) return;
+    if (hasGrouping) {
+        search::grouping::GroupingManager man(*context.grouping);
+        man.groupInRelevanceOrder(hits, numHits);
+    }
+    if (doom.doom()) return;
+    PartialResult &pr = *context.result;
+    pr.totalHits(totalHits);
+    size_t maxHits = std::min(numHits, pr.maxSize());
+    if (pr.hasSortData()) {
+        FastS_SortSpec &spec = context.sort->sortSpec;
+        for (size_t i = 0; i < maxHits; ++i) {
+            pr.add(hits[i], spec.getSortRef(i));
+        }
+    } else {
+        for (size_t i = 0; i < maxHits; ++i) {
+            pr.add(hits[i]);
+        }
+        if ((bits != nullptr) && (pr.size() < pr.maxSize())) {
+            for (unsigned int bitId = bits->getFirstTrueBit();
+                 (bitId < bits->size()) && (pr.size() < pr.maxSize());
+                 bitId = bits->getNextTrueBit(bitId + 1))
+            {
+                pr.add(search::RankedHit(bitId));
+            }
+        }
+    }
+    if (hasGrouping) {
+        context.grouping->setDistributionKey(_distributionKey);
+    }
+}
+
+void
+MatchThread::run()
+{
+    fastos::StopWatch total_time;
+    fastos::StopWatch match_time;
+    total_time.start();
+    match_time.start();
+    MatchTools::UP matchTools = matchToolsFactory.createMatchTools();
+    search::ResultSet::UP result = findMatches(*matchTools);
+    match_time.stop();
+    match_time_s = match_time.elapsed().sec();
+    resultContext = resultProcessor.createThreadContext(matchTools->doom(), thread_id);
+    {
+        WaitTimer get_token_timer(wait_time_s);
+        QueryLimiter::Token::UP processToken(
+                matchTools->getQueryLimiter().getToken(matchTools->doom(),
+                        scheduler.total_size(thread_id),
+                        result->getNumHits(),
+                        resultContext->sort->hasSortData(),
+                        resultContext->grouping.get() != 0));
+        get_token_timer.done();
+        processResult(matchTools->doom(), std::move(result), *resultContext);
+    }
+    total_time.stop();
+    total_time_s = total_time.elapsed().sec();
+    thread_stats.active_time(total_time_s - wait_time_s).wait_time(wait_time_s);
+    mergeDirector.dualMerge(thread_id, *resultContext->result, resultContext->groupingSource);
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
new file mode 100644
index 00000000000..5b298695253
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/runnable.h>
+#include <vespa/vespalib/util/dual_merge_director.h>
+#include <vespa/searchlib/common/resultset.h>
+#include <vespa/searchlib/common/sortresults.h>
+#include <vespa/searchlib/queryeval/hitcollector.h>
+#include "match_tools.h"
+#include "i_match_loop_communicator.h"
+#include "match_params.h"
+#include "matching_stats.h"
+#include "partial_result.h"
+#include "result_processor.h"
+#include "docid_range_scheduler.h"
+
+namespace proton {
+namespace matching {
+
+/**
+ * Runs a single match thread and keeps track of local state.
+ **/
+class MatchThread : public vespalib::Runnable
+{
+public:
+    typedef std::unique_ptr<MatchThread> UP;
+    typedef search::queryeval::SearchIterator SearchIterator;
+    typedef search::fef::MatchData MatchData;
+    typedef search::queryeval::HitCollector HitCollector;
+    typedef search::fef::RankProgram RankProgram;
+
+private:
+    size_t                        thread_id;
+    size_t                        num_threads;
+    MatchParams                   matchParams;
+    const MatchToolsFactory      &matchToolsFactory;
+    IMatchLoopCommunicator       &communicator;
+    DocidRangeScheduler          &scheduler;
+    uint32_t                      _distributionKey;
+    ResultProcessor              &resultProcessor;
+    vespalib::DualMergeDirector  &mergeDirector;
+    ResultProcessor::Context::UP  resultContext;
+    MatchingStats::Partition      thread_stats;
+    double                        total_time_s;
+    double                        match_time_s;
+    double                        wait_time_s;
+
+    struct WaitTimer {
+        double &wait_time_s;
+        fastos::StopWatch wait_time;
+        WaitTimer(double &wait_time_s_in)
+            : wait_time_s(wait_time_s_in), wait_time()
+        {
+            wait_time.start();
+        }
+        void done() {
+            wait_time.stop();
+            wait_time_s += wait_time.elapsed().sec();
+        }
+    };
+
+    template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work>
+    void match_loop(MatchTools &matchTools, IteratorT search,
+                    RankProgram &ranking, HitCollector &hits) __attribute__((noinline));
+
+    template <typename IteratorT, bool do_rank, bool do_limit>
+    void match_loop_helper_2(MatchTools &matchTools, IteratorT search,
+                             RankProgram &ranking, HitCollector &hits);
+
+    template <typename IteratorT, bool do_rank>
+    void match_loop_helper(MatchTools &matchTools, IteratorT search,
+                           RankProgram &ranking, HitCollector &hits);
+
+    search::ResultSet::UP findMatches(MatchTools &matchTools);
+
+    void processResult(const vespalib::Doom & doom,
+                       search::ResultSet::UP result,
+                       ResultProcessor::Context &context);
+
+public:
+    MatchThread(size_t thread_id_in,
+                size_t num_threads_in,
+                const MatchParams &mp,
+                const MatchToolsFactory &mtf,
+                IMatchLoopCommunicator &com,
+                DocidRangeScheduler &sched,
+                ResultProcessor &rp,
+                vespalib::DualMergeDirector &md,
+                uint32_t distributionKey);
+    virtual void run();
+    const MatchingStats::Partition &get_thread_stats() const { return thread_stats; }
+    double get_match_time() const { return match_time_s; }
+};
+
+} // namespace proton::matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp
new file mode 100644
index 00000000000..341bf2dcc77
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp
@@ -0,0 +1,172 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.match_context");
+#include "match_tools.h"
+#include "querynodes.h"
+#include <vespa/searchlib/parsequery/stackdumpiterator.h>
+#include <vespa/searchlib/query/tree/querytreecreator.h>
+
+using search::attribute::IAttributeContext;
+using search::queryeval::IRequestContext;
+using namespace search::fef;
+using namespace search::fef::indexproperties::matchphase;
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+size_t
+tagMatchData(const HandleRecorder::HandleSet & handles, MatchData & md)
+{
+    size_t ignored(0);
+    for (TermFieldHandle handle(0); handle < md.getNumTermFields(); handle++) {
+        if (handles.find(handle) == handles.end()) {
+            md.resolveTermField(handle)->tagAsNotNeeded();
+            ignored++;
+        }
+    }
+    return ignored;
+}
+
+search::fef::RankProgram::UP setup_program(search::fef::RankProgram::UP program,
+                                           const MatchDataLayout &mdl,
+                                           const QueryEnvironment &queryEnv,
+                                           const Properties &featureOverrides)
+{
+    HandleRecorder recorder;
+    {
+        HandleRecorder::Binder bind(recorder);
+        program->setup(mdl, queryEnv, featureOverrides);
+    }
+    tagMatchData(recorder.getHandles(), program->match_data());
+    return program;
+}
+
+}
+
+MatchTools::MatchTools(QueryLimiter & queryLimiter,
+                       const vespalib::Doom & doom_in,
+                       const Query &query,
+                       MaybeMatchPhaseLimiter & match_limiter_in,
+                       const QueryEnvironment & queryEnv,
+                       const MatchDataLayout & mdl,
+                       const RankSetup & rankSetup,
+                       const Properties & featureOverrides)
+    : _queryLimiter(queryLimiter),
+      _doom(doom_in),
+      _query(query),
+      _match_limiter(match_limiter_in),
+      _queryEnv(queryEnv),
+      _rankSetup(rankSetup),
+      _featureOverrides(featureOverrides),
+      _mdl(mdl),
+      _handleRecorder()
+{
+    HandleRecorder::Binder bind(_handleRecorder);
+}
+
+search::fef::RankProgram::UP
+MatchTools::first_phase_program() const {
+    auto program = setup_program(_rankSetup.create_first_phase_program(),
+                                 _mdl, _queryEnv, _featureOverrides);
+    program->match_data().set_termwise_limit(_rankSetup.get_termwise_limit());
+    return program;
+}
+
+search::fef::RankProgram::UP
+MatchTools::second_phase_program() const {
+    return setup_program(_rankSetup.create_second_phase_program(),
+                         _mdl, _queryEnv, _featureOverrides);
+}
+
+search::fef::RankProgram::UP
+MatchTools::summary_program() const {
+    return setup_program(_rankSetup.create_summary_program(),
+                         _mdl, _queryEnv, _featureOverrides);
+}
+
+search::fef::RankProgram::UP
+MatchTools::dump_program() const {
+    return setup_program(_rankSetup.create_dump_program(),
+                         _mdl, _queryEnv, _featureOverrides);
+}
+
+//-----------------------------------------------------------------------------
+
+MatchToolsFactory::
+MatchToolsFactory(QueryLimiter                    & queryLimiter,
+             const vespalib::Doom                 & doom_in,
+             ISearchContext                       & searchContext,
+             IAttributeContext                    & attributeContext,
+             const vespalib::stringref            & queryStack,
+             const vespalib::string               & location,
+             const ViewResolver                   & viewResolver,
+             const search::IDocumentMetaStore     & metaStore,
+             const IIndexEnvironment & indexEnv,
+             const RankSetup         & rankSetup,
+             const Properties        & rankProperties,
+             const Properties        & featureOverrides)
+    : _queryLimiter(queryLimiter),
+      _requestContext(doom_in, attributeContext),
+      _query(),
+      _match_limiter(),
+      _queryEnv(indexEnv, attributeContext, rankProperties),
+      _mdl(),
+      _rankSetup(rankSetup),
+      _featureOverrides(featureOverrides)
+{
+    _valid = _query.buildTree(queryStack, location, viewResolver, indexEnv);
+    if (_valid) {
+        _query.extractTerms(_queryEnv.terms());
+        _query.extractLocations(_queryEnv.locations());
+        _query.setBlackListBlueprint(metaStore.createBlackListBlueprint());
+        _query.reserveHandles(_requestContext, searchContext, _mdl);
+        _query.optimize();
+        _query.fetchPostings();
+        _rankSetup.prepareSharedState(_queryEnv, _queryEnv.getObjectStore());
+        vespalib::string limit_attribute = DegradationAttribute::lookup(rankProperties);
+        size_t limit_maxhits = DegradationMaxHits::lookup(rankProperties);
+        bool limit_ascending = DegradationAscendingOrder::lookup(rankProperties);
+        double limit_max_filter_coverage = DegradationMaxFilterCoverage::lookup(rankProperties);
+        double samplePercentage = DegradationSamplePercentage::lookup(rankProperties);
+        double postFilterMultiplier = DegradationPostFilterMultiplier::lookup(rankProperties);
+        vespalib::string diversity_attribute = DiversityAttribute::lookup(rankProperties);
+        uint32_t diversity_min_groups = DiversityMinGroups::lookup(rankProperties);
+        double diversity_cutoff_factor = DiversityCutoffFactor::lookup(rankProperties);
+        vespalib::string diversity_cutoff_strategy = DiversityCutoffStrategy::lookup(rankProperties);
+        if (!limit_attribute.empty() && limit_maxhits > 0) {
+            _match_limiter.reset(new MatchPhaseLimiter(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext,
+                            limit_attribute, limit_maxhits, !limit_ascending, limit_max_filter_coverage,
+                            samplePercentage, postFilterMultiplier,
+                            diversity_attribute, diversity_min_groups,
+                            diversity_cutoff_factor,
+                            AttributeLimiter::toDiversityCutoffStrategy(diversity_cutoff_strategy)));
+        } else if (_rankSetup.hasMatchPhaseDegradation()) {
+            _match_limiter.reset(new MatchPhaseLimiter(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext,
+                            _rankSetup.getDegradationAttribute(), _rankSetup.getDegradationMaxHits(), !_rankSetup.isDegradationOrderAscending(),
+                            _rankSetup.getDegradationMaxFilterCoverage(),
+                            _rankSetup.getDegradationSamplePercentage(), _rankSetup.getDegradationPostFilterMultiplier(),
+                            _rankSetup.getDiversityAttribute(), _rankSetup.getDiversityMinGroups(),
+                            _rankSetup.getDiversityCutoffFactor(),
+                            AttributeLimiter::toDiversityCutoffStrategy(_rankSetup.getDiversityCutoffStrategy())));
+        }
+    }
+    if (_match_limiter.get() == nullptr) {
+        _match_limiter.reset(new NoMatchPhaseLimiter());
+    }
+}
+
+MatchTools::UP
+MatchToolsFactory::createMatchTools() const
+{
+    assert(_valid);
+    return MatchTools::UP(
+            new MatchTools(_queryLimiter, _requestContext.getDoom(), _query, *_match_limiter, _queryEnv,
+                           _mdl, _rankSetup, _featureOverrides));
+}
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h
new file mode 100644
index 00000000000..c75797852dd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "queryenvironment.h"
+#include "isearchcontext.h"
+#include "query.h"
+#include "viewresolver.h"
+#include <vespa/vespalib/util/doom.h>
+#include "querylimiter.h"
+#include "match_phase_limiter.h"
+#include "handlerecorder.h"
+#include "requestcontext.h"
+
+#include <vespa/vespalib/util/clock.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/fef/fef.h>
+#include <vespa/searchlib/common/idocumentmetastore.h>
+
+namespace proton {
+namespace matching {
+
+class MatchTools : public vespalib::noncopyable
+{
+private:
+    QueryLimiter                       & _queryLimiter;
+    const vespalib::Doom               & _doom;
+    const Query                        & _query;
+    MaybeMatchPhaseLimiter             & _match_limiter;
+    const QueryEnvironment             & _queryEnv;
+    const search::fef::RankSetup       & _rankSetup;
+    const search::fef::Properties      & _featureOverrides;
+    search::fef::MatchDataLayout         _mdl;
+    HandleRecorder                       _handleRecorder;
+
+public:
+    typedef std::unique_ptr<MatchTools> UP;
+    MatchTools(QueryLimiter & queryLimiter,
+               const vespalib::Doom & requestContext,
+               const Query &query,
+               MaybeMatchPhaseLimiter &match_limiter_in,
+               const QueryEnvironment &queryEnv,
+               const search::fef::MatchDataLayout &mdl,
+               const search::fef::RankSetup &rankSetup,
+               const search::fef::Properties &featureOverrides);
+    const vespalib::Doom &doom() const { return _doom; }
+    QueryLimiter & getQueryLimiter() { return _queryLimiter; }
+    MaybeMatchPhaseLimiter &match_limiter() { return _match_limiter; }
+    search::queryeval::SearchIterator::UP
+    createSearch(search::fef::MatchData &matchData) const {
+        return _query.createSearch(matchData);
+    }
+    bool has_first_phase_rank() const { return !_rankSetup.getFirstPhaseRank().empty(); }
+    bool has_second_phase_rank() const { return !_rankSetup.getSecondPhaseRank().empty(); }
+    search::fef::RankProgram::UP first_phase_program() const;
+    search::fef::RankProgram::UP second_phase_program() const;
+    search::fef::RankProgram::UP summary_program() const;
+    search::fef::RankProgram::UP dump_program() const;
+};
+
+class MatchToolsFactory : public vespalib::noncopyable
+{
+private:
+    QueryLimiter                  & _queryLimiter;
+    RequestContext                  _requestContext;
+    Query                           _query;
+    MaybeMatchPhaseLimiter::UP      _match_limiter;
+    QueryEnvironment                _queryEnv;
+    search::fef::MatchDataLayout    _mdl;
+    const search::fef::RankSetup  & _rankSetup;
+    const search::fef::Properties & _featureOverrides;
+    bool                            _valid;
+
+public:
+    typedef std::unique_ptr<MatchToolsFactory> UP;
+
+    MatchToolsFactory(QueryLimiter & queryLimiter,
+                      const vespalib::Doom &doom_in,
+                      ISearchContext &searchContext,
+                      search::attribute::IAttributeContext &attributeContext,
+                      const vespalib::stringref &queryStack,
+                      const vespalib::string &location,
+                      const ViewResolver &viewResolver,
+                      const search::IDocumentMetaStore &metaStore,
+                      const search::fef::IIndexEnvironment &indexEnv,
+                      const search::fef::RankSetup &rankSetup,
+                      const search::fef::Properties &rankProperties,
+                      const search::fef::Properties &featureOverrides);
+    bool valid() const { return _valid; }
+    const MaybeMatchPhaseLimiter &match_limiter() const { return *_match_limiter; }
+    MatchTools::UP createMatchTools() const;
+};
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h b/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h
new file mode 100644
index 00000000000..f0006925bfb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "querynodes.h"
+#include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/searchlib/query/tree/templatetermvisitor.h>
+
+namespace proton {
+namespace matching {
+
+/**
+ * Visits all terms of a node tree, and allocates MatchData space for
+ * each.
+ */
+class MatchDataReserveVisitor
+    : public search::query::TemplateTermVisitor<MatchDataReserveVisitor,
+                                                ProtonNodeTypes>
+{
+    search::fef::MatchDataLayout &_mdl;
+
+public:
+    template <class TermNode>
+    void visitTerm(TermNode &n) { n.allocateTerms(_mdl); }
+
+    virtual void visit(ProtonNodeTypes::Equiv &n) {
+        MatchDataReserveVisitor subAllocator(n.children_mdl);
+        for (size_t i = 0; i < n.getChildren().size(); ++i) {
+            n.getChildren()[i]->accept(subAllocator);
+        }
+        n.allocateTerms(_mdl);
+    }
+
+    MatchDataReserveVisitor(search::fef::MatchDataLayout &mdl) : _mdl(mdl) {}
+};
+
+}  // namespace matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
new file mode 100644
index 00000000000..99fc96b43b2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
@@ -0,0 +1,275 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.matcher");
+
+#include "isearchcontext.h"
+#include "match_master.h"
+#include "match_params.h"
+#include "matcher.h"
+#include "query.h"
+#include "queryenvironment.h"
+#include "result_processor.h"
+#include "sessionmanager.h"
+#include <vespa/searchlib/common/resultset.h>
+#include <vespa/searchlib/engine/errorcodes.h>
+#include <vespa/searchlib/features/setup.h>
+#include <vespa/searchlib/fef/properties.h>
+#include <vespa/searchlib/fef/test/plugin/setup.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+using search::fef::Properties;
+using search::fef::Property;
+using namespace search;
+using namespace search::engine;
+using search::attribute::IAttributeContext;
+using search::fef::MatchDataLayout;
+using search::fef::MatchData;
+using search::queryeval::Blueprint;
+using search::queryeval::SearchIterator;
+using vespalib::Doom;
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+// used to give out empty blacklist blueprints
+struct StupidMetaStore : IDocumentMetaStore {
+    bool getGid(DocId, GlobalId &) const override { return false; }
+    bool getLid(const GlobalId &, DocId &) const override { return false; }
+    DocumentMetaData getMetaData(const GlobalId &) const override { return DocumentMetaData(); }
+    void getMetaData(const BucketId &, DocumentMetaData::Vector &) const override { }
+    DocId getCommittedDocIdLimit() const override { return 1; }
+    DocId getNumUsedLids() const override { return 0; }
+    DocId getNumActiveLids() const override { return 0; }
+    search::LidUsageStats getLidUsageStats() const override { return search::LidUsageStats(); }
+    Blueprint::UP createBlackListBlueprint() const override {
+        return Blueprint::UP();
+    }
+};
+
+FeatureSet::SP findFeatureSet(const DocsumRequest &req,
+                              MatchToolsFactory &mtf, bool summaryFeatures) {
+    std::vector<uint32_t> docs;
+    for (size_t i = 0; i < req.hits.size(); ++i) {
+        if (req.hits[i].docid != search::endDocId) {
+            docs.push_back(req.hits[i].docid);
+        }
+    }
+    std::sort(docs.begin(), docs.end());
+    return MatchMaster::getFeatureSet(mtf, docs, summaryFeatures);
+}
+
+}  // namespace proton::matching::<unnamed>
+
+FeatureSet::SP
+Matcher::getFeatureSet(const DocsumRequest & req,
+                       ISearchContext & searchCtx,
+                       IAttributeContext & attrCtx,
+                       SessionManager & sessionMgr,
+                       bool summaryFeatures)
+{
+    search::grouping::SessionId
+        sessionId(&req.sessionId[0], req.sessionId.size());
+    if (!sessionId.empty()) {
+        const Properties &cache_props = req.propertiesMap.cacheProperties();
+        bool searchSessionCached = cache_props.lookup("query").found();
+        if (searchSessionCached) {
+            SearchSession::SP session(sessionMgr.pickSearch(sessionId));
+            if (session.get()) {
+                MatchToolsFactory &mtf = session->getMatchToolsFactory();
+                FeatureSet::SP result = findFeatureSet(req, mtf, summaryFeatures);
+                session->releaseEnumGuards();
+                return result;
+            }
+        }
+    }
+
+    StupidMetaStore metaStore;
+    MatchToolsFactory mtf(_queryLimiter, Doom(_clock, req.getTimeOfDoom()), searchCtx, attrCtx,
+                          req.getStackRef(), req.location, _viewResolver,
+                          metaStore, _indexEnv,
+                          *_rankSetup, req.propertiesMap.rankProperties(),
+                          req.propertiesMap.featureOverrides());
+    if (!mtf.valid()) {
+        LOG(warning, "getFeatureSet(%s): query execution failed "
+            "(invalid query). Returning empty feature set",
+            (summaryFeatures ? "summary features" : "rank features"));
+        return FeatureSet::SP(new FeatureSet());
+    }
+    return findFeatureSet(req, mtf, summaryFeatures);
+}
+
+Matcher::Matcher(const search::index::Schema &schema,
+                 const search::fef::Properties &props,
+                 const vespalib::Clock & clock,
+                 QueryLimiter & queryLimiter,
+                 uint32_t distributionKey)
+    : _indexEnv(schema, props),
+      _blueprintFactory(),
+      _rankSetup(),
+      _viewResolver(ViewResolver::createFromSchema(schema)),
+      _statsLock(),
+      _stats(),
+      _clock(clock),
+      _queryLimiter(queryLimiter),
+      _distributionKey(distributionKey)
+{
+    search::features::setup_search_features(_blueprintFactory);
+    search::fef::test::setup_fef_test_plugin(_blueprintFactory);
+    _rankSetup.reset(new search::fef::RankSetup(_blueprintFactory, _indexEnv));
+    _rankSetup->configure(); // reads config values from the property map
+    if (!_rankSetup->compile()) {
+        throw vespalib::IllegalArgumentException(
+                "failed to compile rank setup", VESPA_STRLOC);
+    }
+}
+
+MatchingStats
+Matcher::getStats()
+{
+    vespalib::LockGuard guard(_statsLock);
+    MatchingStats stats = _stats;
+    _stats = MatchingStats();
+    return stats;
+}
+
+search::engine::SearchReply::UP
+Matcher::handleGroupingSession(proton::matching::SessionManager &sessionMgr,
+                               search::grouping::GroupingContext & groupingContext,
+                               search::grouping::GroupingSession::UP groupingSession)
+{
+    search::engine::SearchReply::UP reply(new search::engine::SearchReply());
+    groupingSession->continueExecution(groupingContext);
+    groupingContext.getResult().swap(reply->groupResult);
+    if (!groupingSession->finished()) {
+        sessionMgr.insert(std::move(groupingSession));
+    }
+    return reply;
+}
+
+class LimitedThreadBundleWrapper : public vespalib::ThreadBundle
+{
+public:
+    LimitedThreadBundleWrapper(vespalib::ThreadBundle &threadBundle, uint32_t maxThreads) :
+        _threadBundle(threadBundle),
+        _maxThreads(std::min(maxThreads, static_cast<uint32_t>(threadBundle.size())))
+    { }
+private:
+    size_t size() const override { return _maxThreads; }
+    void run(const std::vector<vespalib::Runnable*> &targets) override {
+        _threadBundle.run(targets);
+    }
+    vespalib::ThreadBundle &_threadBundle;
+    const uint32_t          _maxThreads;
+};
+
+search::engine::SearchReply::UP
+Matcher::match(const search::engine::SearchRequest &request,
+               vespalib::ThreadBundle &threadBundle,
+               ISearchContext &searchContext,
+               IAttributeContext &attrContext,
+               proton::matching::SessionManager &sessionMgr,
+               const search::IDocumentMetaStore &metaStore,
+               SearchSession::OwnershipBundle &&owned_objects)
+{
+    fastos::StopWatch total_matching_time;
+    total_matching_time.start();
+    MatchingStats my_stats;
+    search::engine::SearchReply::UP reply(new search::engine::SearchReply());
+    { // we want to measure full set-up and tear-down time as part of
+      // collateral time
+        search::grouping::GroupingContext
+            groupingContext(_clock, request.getTimeOfDoom(),
+                            &request.groupSpec[0], request.groupSpec.size());
+        search::grouping::SessionId sessionId(&request.sessionId[0], request.sessionId.size());
+        bool shouldCacheSearchSession = false;
+        bool shouldCacheGroupingSession = false;
+        if (!sessionId.empty()) {
+            const Properties &cache_props = request.propertiesMap.cacheProperties();
+            shouldCacheGroupingSession = cache_props.lookup("grouping").found();
+            shouldCacheSearchSession = cache_props.lookup("query").found();
+            if (shouldCacheGroupingSession) {
+                search::grouping::GroupingSession::UP
+                    session(sessionMgr.pickGrouping(sessionId));
+                if (session.get()) {
+                    return handleGroupingSession(
+                            sessionMgr, groupingContext, std::move(session));
+                }
+            }
+        }
+        const Properties *feature_overrides = &request.propertiesMap.featureOverrides();
+        if (shouldCacheSearchSession) {
+            owned_objects.feature_overrides.reset(new Properties(*feature_overrides));
+            feature_overrides = owned_objects.feature_overrides.get();
+        }
+        MatchToolsFactory::UP mtf = create_match_tools_factory(request, searchContext, attrContext, metaStore, *feature_overrides);
+        if (!mtf->valid()) {
+            reply->errorCode = ECODE_QUERY_PARSE_ERROR;
+            reply->errorMessage = "query execution failed (invalid query)";
+            return reply;
+        }
+        bool willNotNeedRanking = (!groupingContext.needRanking() && (request.maxhits == 0)) ||
+                                  (!request.sortSpec.empty() && (request.sortSpec.find("[rank]") == vespalib::string::npos));
+        MatchParams params(searchContext.getDocIdLimit(),
+                           _rankSetup->getHeapSize(),
+                           _rankSetup->getArraySize(),
+                           _rankSetup->getRankScoreDropLimit(),
+                           request.offset, request.maxhits,
+                           !_rankSetup->getSecondPhaseRank().empty(),
+                           !willNotNeedRanking);
+        ResultProcessor
+            rp(attrContext, metaStore, sessionMgr, groupingContext,
+               sessionId, request.sortSpec, params.offset, params.hits);
+        MatchMaster master;
+        LimitedThreadBundleWrapper limitedThreadBundle(threadBundle, _rankSetup->getNumThreadsPerSearch());
+        ResultProcessor::Result::UP result = master.match(params, limitedThreadBundle, *mtf, rp, _distributionKey, _rankSetup->getNumSearchPartitions());
+        my_stats = master.getStats();
+        size_t estimate = std::min(static_cast<size_t>(metaStore.getCommittedDocIdLimit()), mtf->match_limiter().getDocIdSpaceEstimate());
+        if (shouldCacheSearchSession && ((result->_numFs4Hits != 0) || shouldCacheGroupingSession)) {
+            SearchSession::SP session(
+                    new SearchSession(sessionId, request.getTimeOfDoom(),
+                                      std::move(mtf),
+                                      std::move(owned_objects)));
+            session->releaseEnumGuards();
+            sessionMgr.insert(std::move(session));
+        }
+        reply = std::move(result->_reply);
+        reply->useCoverage = true;
+        reply->coverage.setActive(metaStore.getNumActiveLids());
+        reply->coverage.setCovered(std::min(static_cast<size_t>(metaStore.getNumActiveLids()),
+                                            (estimate * metaStore.getNumActiveLids())/metaStore.getCommittedDocIdLimit()));
+    }
+    total_matching_time.stop();
+    my_stats.queryCollateralTime(total_matching_time.elapsed().sec()
+                                 - my_stats.queryLatencyAvg());
+    {
+        vespalib::LockGuard guard(_statsLock);
+        _stats.add(my_stats);
+    }
+    return reply;
+}
+
+FeatureSet::SP
+Matcher::getSummaryFeatures(const DocsumRequest & req,
+                            ISearchContext & searchCtx,
+                            IAttributeContext & attrCtx,
+                            SessionManager &sessionMgr)
+{
+    return getFeatureSet(req, searchCtx, attrCtx, sessionMgr, true);
+}
+
+FeatureSet::SP
+Matcher::getRankFeatures(const DocsumRequest & req,
+                         ISearchContext & searchCtx,
+                         IAttributeContext & attrCtx,
+                         SessionManager &sessionMgr)
+{
+    return getFeatureSet(req, searchCtx, attrCtx, sessionMgr, false);
+}
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h
new file mode 100644
index 00000000000..e1a55e21f92
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h
@@ -0,0 +1,180 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "indexenvironment.h"
+#include "matching_stats.h"
+#include "match_tools.h"
+#include "search_session.h"
+#include "viewresolver.h"
+#include <vespa/searchcore/proton/matching/querylimiter.h>
+#include <vespa/searchlib/common/featureset.h>
+#include <vespa/searchlib/common/resultset.h>
+#include <vespa/searchlib/engine/docsumrequest.h>
+#include <vespa/searchlib/engine/searchreply.h>
+#include <vespa/searchlib/engine/searchrequest.h>
+#include <vespa/searchlib/fef/fef.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/vespalib/util/clock.h>
+#include <vespa/vespalib/util/closure.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/thread_bundle.h>
+
+namespace search {
+namespace grouping {
+class GroupingContext;
+class GroupingSession;
+}  // namespace search::grouping;
+namespace index { class Schema; }
+namespace attribute { class IAttributeContext; }
+
+class IDocumentMetaStore;
+
+} // namespace search
+
+namespace proton {
+namespace matching {
+
+class ISearchContext;
+class SessionManager;
+
+/**
+ * The Matcher is responsible for performing searches.
+ **/
+class Matcher
+{
+private:
+    IndexEnvironment              _indexEnv;
+    search::fef::BlueprintFactory _blueprintFactory;
+    search::fef::RankSetup::SP    _rankSetup;
+    ViewResolver                  _viewResolver;
+    vespalib::Lock                _statsLock;
+    MatchingStats                 _stats;
+    const vespalib::Clock       & _clock;
+    QueryLimiter                & _queryLimiter;
+    uint32_t                      _distributionKey;
+
+    search::FeatureSet::SP
+    getFeatureSet(const search::engine::DocsumRequest & req,
+                  ISearchContext & searchCtx,
+                  search::attribute::IAttributeContext & attrCtx,
+                  SessionManager &sessionMgr,
+                  bool summaryFeatures);
+    search::engine::SearchReply::UP
+    handleGroupingSession(
+            SessionManager &sessionMgr,
+            search::grouping::GroupingContext & groupingContext,
+            std::unique_ptr<search::grouping::GroupingSession> gs);
+
+    Matcher(const Matcher &);
+    Matcher &operator=(const Matcher &);
+public:
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::unique_ptr<Matcher> UP;
+    typedef std::shared_ptr<Matcher> SP;
+
+    /**
+     * Create a new matcher. The schema represents the current index
+     * layout.
+     *
+     * @param schema index schema
+     * @param props ranking configuration
+     * @param clock used for timeout handling
+     **/
+    Matcher(const search::index::Schema &schema,
+            const search::fef::Properties &props,
+            const vespalib::Clock & clock,
+            QueryLimiter & queryLimiter,
+            uint32_t distributionKey);
+
+    const search::fef::IIndexEnvironment &get_index_env() const { return _indexEnv; }
+
+    /**
+     * Observe and reset stats for this object.
+     *
+     * @return stats
+     **/
+    MatchingStats getStats();
+
+    /**
+     * Create the low-level tools needed to perform matching. This
+     * function is exposed for testing purposes.
+     **/
+    MatchToolsFactory::UP create_match_tools_factory(const search::engine::SearchRequest &request,
+                                                     ISearchContext &searchContext,
+                                                     search::attribute::IAttributeContext &attrContext,
+                                                     const search::IDocumentMetaStore &metaStore,
+                                                     const search::fef::Properties &feature_overrides) const
+    {
+        return MatchToolsFactory::UP(new MatchToolsFactory(
+                        _queryLimiter, vespalib::Doom(_clock, request.getTimeOfDoom()),
+                        searchContext, attrContext, request.getStackRef(),
+                        request.location, _viewResolver, metaStore, _indexEnv,
+                        *_rankSetup, request.propertiesMap.rankProperties(),
+                        feature_overrides));
+    }
+
+    /**
+     * Perform a search against this matcher.
+     *
+     * @return search reply
+     * @param request the search request
+     * @param threadBundle bundle of threads to use for multi-threaded execution
+     * @param searchContext abstract view of searchable data
+     * @param attrContext abstract view of attribute data
+     * @param sessionManager multilevel grouping session cache
+     * @param metaStore the document meta store used to map from lid to gid
+     **/
+    search::engine::SearchReply::UP
+    match(const search::engine::SearchRequest &request,
+          vespalib::ThreadBundle &threadBundle,
+          ISearchContext &searchContext,
+          search::attribute::IAttributeContext &attrContext,
+          SessionManager &sessionManager,
+          const search::IDocumentMetaStore &metaStore,
+          SearchSession::OwnershipBundle &&owned_objects);
+
+    /**
+     * Perform matching for the documents in the given docsum request
+     * to calculate the summary features specified in the rank setup
+     * of this matcher.
+     *
+     * @param req the docsum request
+     * @param searchCtx abstract view of searchable data
+     * @param attrCtx abstract view of attribute data
+     * @return calculated summary features.
+     **/
+    search::FeatureSet::SP
+    getSummaryFeatures(const search::engine::DocsumRequest & req,
+                       ISearchContext & searchCtx,
+                       search::attribute::IAttributeContext & attrCtx,
+                       SessionManager &sessionManager);
+
+    /**
+     * Perform matching for the documents in the given docsum request
+     * to calculate the rank features specified in the rank setup of
+     * this matcher.
+     *
+     * @param req the docsum request
+     * @param searchCtx abstract view of searchable data
+     * @param attrCtx abstract view of attribute data
+     * @return calculated rank features.
+     **/
+    search::FeatureSet::SP
+    getRankFeatures(const search::engine::DocsumRequest & req,
+                    ISearchContext & searchCtx,
+                    search::attribute::IAttributeContext & attrCtx,
+                    SessionManager &sessionManager);
+
+    /**
+     * @return true if this rankprofile has summary-features enabled
+     **/
+    bool canProduceSummaryFeatures() const { return ! _rankSetup->getSummaryFeatures().empty(); }
+    double get_termwise_limit() const { return _rankSetup->get_termwise_limit(); }
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
new file mode 100644
index 00000000000..79227cf9d88
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "matching_stats.h"
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+using vespalib::make_string;
+using vespalib::IllegalStateException;
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+MatchingStats::Partition &get_writable_partition(std::vector<MatchingStats::Partition> &state, size_t id) {
+    if (state.size() <= id) {
+        state.resize(id + 1);
+    }
+    return state[id];
+}
+
+} // namespace proton::matching::<unnamed>
+
+MatchingStats &
+MatchingStats::merge_partition(const Partition &partition, size_t id)
+{
+    get_writable_partition(_partitions, id) = partition;
+
+    _docsMatched += partition.docsMatched();
+    _docsRanked += partition.docsRanked();
+    _docsReRanked += partition.docsReRanked();
+
+    return *this;
+}
+
+MatchingStats &
+MatchingStats::add(const MatchingStats &rhs)
+{
+
+    _queries += rhs._queries;
+    _limited_queries += rhs._limited_queries;
+
+    _docsMatched += rhs._docsMatched;
+    _docsRanked += rhs._docsRanked;
+    _docsReRanked += rhs._docsReRanked;
+
+    _queryCollateralTime.add(rhs._queryCollateralTime);
+    _queryLatency.add(rhs._queryLatency);
+    _matchTime.add(rhs._matchTime);
+    _groupingTime.add(rhs._groupingTime);
+    _rerankTime.add(rhs._rerankTime);
+    for (size_t id = 0; id < rhs.getNumPartitions(); ++id) {
+        get_writable_partition(_partitions, id).add(rhs.getPartition(id));
+    }
+    return *this;
+}
+
+}
+} // namespace searchcore
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h
new file mode 100644
index 00000000000..7cc8f0d3ef2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton {
+namespace matching {
+
+/**
+ * Statistics for the matching pipeline. Used for internal aggregation
+ * before inserting numbers into the metrics framework. The values
+ * produced by a single search are set on a single object. Values are
+ * aggregated by adding objects together.
+ **/
+struct MatchingStats
+{
+private:
+    class Avg {
+        double _value;
+        size_t _count;
+    public:
+        Avg() : _value(0.0), _count(0) {}
+        void set(double value) { _value = value; _count = 1; }
+        double avg() const { return !_count ? 0 : _value / _count; }
+        size_t count() const { return _count; }
+        void add(const Avg &other) {
+            _value += other._value;
+            _count += other._count;
+        }
+    };
+
+public:
+
+    /**
+     * Matching statistics that are tracked separately for each match
+     * thread.
+     **/
+    class Partition {
+        size_t _docsMatched;
+        size_t _docsRanked;
+        size_t _docsReRanked;
+        Avg    _active_time;
+        Avg    _wait_time;
+    public:
+        Partition()
+            : _docsMatched(0),
+              _docsRanked(0),
+              _docsReRanked(0),
+              _active_time(),
+              _wait_time() {}
+
+        Partition &docsMatched(size_t value) { _docsMatched = value; return *this; }
+        size_t docsMatched() const { return _docsMatched; }
+        Partition &docsRanked(size_t value) { _docsRanked = value; return *this; }
+        size_t docsRanked() const { return _docsRanked; }
+        Partition &docsReRanked(size_t value) { _docsReRanked = value; return *this; }
+        size_t docsReRanked() const { return _docsReRanked; }
+
+        Partition &active_time(double time_s) { _active_time.set(time_s); return *this; }
+        double active_time_avg() const { return _active_time.avg(); }
+        size_t active_time_count() const { return _active_time.count(); }
+        Partition &wait_time(double time_s) { _wait_time.set(time_s); return *this; }
+        double wait_time_avg() const { return _wait_time.avg(); }
+        size_t wait_time_count() const { return _wait_time.count(); }
+
+        Partition &add(const Partition &rhs) {
+            _docsMatched += rhs._docsMatched;
+            _docsRanked += rhs._docsRanked;
+            _docsReRanked += rhs._docsReRanked;
+
+            _active_time.add(rhs._active_time);
+            _wait_time.add(rhs._wait_time);
+            return *this;
+        }
+    };
+
+private:
+    size_t                 _queries;
+    size_t                 _limited_queries;
+    size_t                 _docsMatched;
+    size_t                 _docsRanked;
+    size_t                 _docsReRanked;
+    Avg                    _queryCollateralTime;
+    Avg                    _queryLatency;
+    Avg                    _matchTime;
+    Avg                    _groupingTime;
+    Avg                    _rerankTime;
+    std::vector<Partition> _partitions;
+
+public:
+    MatchingStats()
+        : _queries(0),
+          _limited_queries(0),
+          _docsMatched(0),
+          _docsRanked(0),
+          _docsReRanked(0),
+          _queryCollateralTime(),
+          _queryLatency(),
+          _matchTime(),
+          _groupingTime(),
+          _rerankTime(),
+          _partitions() {}
+
+    MatchingStats &queries(size_t value) { _queries = value; return *this; }
+    size_t queries() const { return _queries; }
+
+    MatchingStats &limited_queries(size_t value) { _limited_queries = value; return *this; }
+    size_t limited_queries() const { return _limited_queries; }
+
+    MatchingStats &docsMatched(size_t value) { _docsMatched = value; return *this; }
+    size_t docsMatched() const { return _docsMatched; }
+
+    MatchingStats &docsRanked(size_t value) { _docsRanked = value; return *this; }
+    size_t docsRanked() const { return _docsRanked; }
+
+    MatchingStats &docsReRanked(size_t value) { _docsReRanked = value; return *this; }
+    size_t docsReRanked() const { return _docsReRanked; }
+
+    MatchingStats &queryCollateralTime(double time_s) { _queryCollateralTime.set(time_s); return *this; }
+    double queryCollateralTimeAvg() const { return _queryCollateralTime.avg(); }
+    size_t queryCollateralTimeCount() const { return _queryCollateralTime.count(); }
+
+    MatchingStats &queryLatency(double time_s) { _queryLatency.set(time_s); return *this; }
+    double queryLatencyAvg() const { return _queryLatency.avg(); }
+    size_t queryLatencyCount() const { return _queryLatency.count(); }
+
+    MatchingStats &matchTime(double time_s) { _matchTime.set(time_s); return *this; }
+    double matchTimeAvg() const { return _matchTime.avg(); }
+    size_t matchTimeCount() const { return _matchTime.count(); }
+
+    MatchingStats &groupingTime(double time_s) { _groupingTime.set(time_s); return *this; }
+    double groupingTimeAvg() const { return _groupingTime.avg(); }
+    size_t groupingTimeCount() const { return _groupingTime.count(); }
+
+    MatchingStats &rerankTime(double time_s) { _rerankTime.set(time_s); return *this; }
+    double rerankTimeAvg() const { return _rerankTime.avg(); }
+    size_t rerankTimeCount() const { return _rerankTime.count(); }
+
+    // used to merge in stats from each match thread
+    MatchingStats &merge_partition(const Partition &partition, size_t id);
+    size_t getNumPartitions() const { return _partitions.size(); }
+    const Partition &getPartition(size_t index) const { return _partitions[index]; }
+
+    // used to aggregate accross searches (and configurations)
+    MatchingStats &add(const MatchingStats &rhs);
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp b/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp
new file mode 100644
index 00000000000..8f5e09fac7d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp
@@ -0,0 +1,131 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "partial_result.h"
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+bool before(const search::RankedHit &a, const search::RankedHit &b) {
+    if (a._rankValue != b._rankValue) {
+        return (a._rankValue > b._rankValue);
+    }
+    return (a._docId < b._docId);
+}
+
+void mergeHits(size_t maxHits,
+               std::vector<search::RankedHit> &hits,
+               const std::vector<search::RankedHit> &rhs_hits)
+{
+    std::vector<search::RankedHit> my_hits;
+    std::swap(hits, my_hits);
+    hits.reserve(maxHits);
+    const search::RankedHit *a_pos = &my_hits[0];
+    const search::RankedHit *a_end = a_pos + my_hits.size();
+    const search::RankedHit *b_pos = &rhs_hits[0];
+    const search::RankedHit *b_end = b_pos + rhs_hits.size();
+    while (a_pos < a_end && b_pos < b_end && hits.size() < maxHits) {
+        if (before(*a_pos, *b_pos)) {
+            hits.push_back(*a_pos++);
+        } else {
+            hits.push_back(*b_pos++);
+        }
+    }
+    while (a_pos < a_end && hits.size() < maxHits) {
+        hits.push_back(*a_pos++);
+    }
+    while (b_pos < b_end && hits.size() < maxHits) {
+        hits.push_back(*b_pos++);
+    }
+}
+
+bool before(const PartialResult::SortRef &a, uint32_t docid_a,
+            const PartialResult::SortRef &b, uint32_t docid_b) {
+    int res = memcmp(a.first, b.first, std::min(a.second, b.second));
+    if (res != 0) {
+        return (res < 0);
+    }
+    if (a.second != b.second) {
+        return (a.second < b.second);
+    }
+    return (docid_a < docid_b);
+}
+
+size_t mergeHits(size_t maxHits,
+                 std::vector<search::RankedHit> &hits,
+                 std::vector<PartialResult::SortRef> &sortData,
+                 const std::vector<search::RankedHit> &rhs_hits,
+                 const std::vector<PartialResult::SortRef> &rhs_sortData)
+{
+    size_t sortDataSize = 0;
+    std::vector<search::RankedHit> my_hits;
+    std::vector<PartialResult::SortRef> my_sortData;
+    std::swap(hits, my_hits);
+    std::swap(sortData, my_sortData);
+    hits.reserve(maxHits);
+    sortData.reserve(maxHits);
+    const search::RankedHit *a_pos = &my_hits[0];
+    const search::RankedHit *a_end = a_pos + my_hits.size();
+    const search::RankedHit *b_pos = &rhs_hits[0];
+    const search::RankedHit *b_end = b_pos + rhs_hits.size();
+    const PartialResult::SortRef *a_sort_pos = &my_sortData[0];
+    const PartialResult::SortRef *b_sort_pos = &rhs_sortData[0];
+    while (a_pos < a_end && b_pos < b_end && hits.size() < maxHits) {
+        if (before(*a_sort_pos, a_pos->_docId,
+                   *b_sort_pos, b_pos->_docId))
+        {
+            hits.push_back(*a_pos++);
+            sortDataSize += a_sort_pos->second;
+            sortData.push_back(*a_sort_pos++);
+        } else {
+            hits.push_back(*b_pos++);
+            sortDataSize += b_sort_pos->second;
+            sortData.push_back(*b_sort_pos++);
+        }
+    }
+    while (a_pos < a_end && hits.size() < maxHits) {
+        hits.push_back(*a_pos++);
+        sortDataSize += a_sort_pos->second;
+        sortData.push_back(*a_sort_pos++);
+    }
+    while (b_pos < b_end && hits.size() < maxHits) {
+        hits.push_back(*b_pos++);
+        sortDataSize += b_sort_pos->second;
+        sortData.push_back(*b_sort_pos++);
+    }
+    return sortDataSize;
+}
+
+} // namespace proton::matching::<unnamed>
+
+PartialResult::PartialResult(size_t maxSize_in, bool hasSortData_in)
+    : _hits(),
+      _sortData(),
+      _maxSize(maxSize_in),
+      _totalHits(0),
+      _hasSortData(hasSortData_in),
+      _sortDataSize(0)
+{
+    _hits.reserve(_maxSize);
+    if (_hasSortData) {
+        _sortData.reserve(_maxSize);
+    }
+}
+
+void
+PartialResult::merge(Source &rhs)
+{
+    PartialResult &r = static_cast<PartialResult&>(rhs);
+    assert(_hasSortData == r._hasSortData);
+    _totalHits += r._totalHits;
+    if (_hasSortData) {
+        _sortDataSize = mergeHits(_maxSize, _hits, _sortData, r._hits, r._sortData);
+    } else {
+        mergeHits(_maxSize, _hits, r._hits);
+    }
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h
new file mode 100644
index 00000000000..7264057c223
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/linkedptr.h>
+#include <vespa/vespalib/util/dual_merge_director.h>
+#include <vespa/searchlib/common/rankedhit.h>
+#include <vector>
+#include <algorithm>
+#include <memory>
+
+namespace proton {
+namespace matching {
+
+/**
+ * The best hits from each match thread are put into a partial result
+ * and merged with results from other threads.
+ **/
+class PartialResult : public vespalib::DualMergeDirector::Source
+{
+public:
+    typedef vespalib::LinkedPtr<PartialResult> LP;
+    typedef std::pair<const char *, size_t> SortRef;
+
+private:
+    std::vector<search::RankedHit> _hits;
+    std::vector<SortRef>           _sortData;
+    size_t                         _maxSize;
+    size_t                         _totalHits;
+    bool                           _hasSortData;
+    size_t                         _sortDataSize;
+
+public:
+    PartialResult(size_t maxSize_in, bool hasSortData_in);
+    size_t size() const { return _hits.size(); }
+    size_t maxSize() const { return _maxSize; }
+    size_t totalHits() const { return _totalHits; }
+    bool hasSortData() const { return _hasSortData; }
+    size_t sortDataSize() const { return _sortDataSize; }
+    const search::RankedHit &hit(size_t i) const { return _hits[i]; }
+    const SortRef &sortData(size_t i) const { return _sortData[i]; }
+    void totalHits(size_t th) { _totalHits = th; }
+    void add(const search::RankedHit &h) {
+        assert(!_hasSortData);
+        _hits.push_back(h);
+    }
+    void add(const search::RankedHit &h, const SortRef &sd) {
+        assert(_hasSortData);
+        _hits.push_back(h);
+        _sortData.push_back(sd);
+        _sortDataSize += sd.second;
+    }
+    virtual void merge(Source &rhs);
+};
+
+} // namespace proton::matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp
new file mode 100644
index 00000000000..ab61515d0fb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp
@@ -0,0 +1,170 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.query");
+
+#include "query.h"
+
+#include "blueprintbuilder.h"
+#include "matchdatareservevisitor.h"
+#include "querynodes.h"
+#include "resolveviewvisitor.h"
+#include "termdataextractor.h"
+#include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/searchlib/common/location.h>
+#include <vespa/searchlib/fef/iindexenvironment.h>
+#include <vespa/searchlib/parsequery/stackdumpiterator.h>
+#include <vespa/searchlib/query/tree/location.h>
+#include <vespa/searchlib/query/tree/point.h>
+#include <vespa/searchlib/query/tree/querytreecreator.h>
+#include <vespa/searchlib/query/tree/rectangle.h>
+#include <vespa/searchlib/query/weight.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/queryeval/intermediate_blueprints.h>
+
+using document::PositionDataType;
+using search::SimpleQueryStackDumpIterator;
+using search::fef::IIndexEnvironment;
+using search::fef::ITermData;
+using search::fef::MatchData;
+using search::fef::MatchDataLayout;
+using search::query::Node;
+using search::query::QueryTreeCreator;
+using search::query::Weight;
+using search::queryeval::AndNotBlueprint;
+using search::queryeval::Blueprint;
+using search::queryeval::IRequestContext;
+using search::queryeval::SearchIterator;
+using vespalib::string;
+using std::vector;
+
+namespace proton {
+namespace matching {
+
+namespace {
+void AddLocationNode(const string &location_str, Node::UP &query_tree,
+                     search::fef::Location &fef_location) {
+    if (location_str.empty()) {
+        return;
+    }
+    string::size_type pos = location_str.find(':');
+    if (pos == string::npos) {
+        LOG(warning, "Location string lacks attribute vector specification. loc='%s'",
+            location_str.c_str());
+        return;
+    }
+    const string view = PositionDataType::getZCurveFieldName(
+            location_str.substr(0, pos));
+    const string loc = location_str.substr(pos + 1);
+
+    search::common::Location locationSpec;
+    if (!locationSpec.parse(loc)) {
+        LOG(warning, "Location parse error (location: '%s'): %s",
+            location_str.c_str(),
+            locationSpec.getParseError());
+        return;
+    }
+
+    int32_t id = -1;
+    Weight weight(100);
+
+    ProtonAnd::UP new_base(new ProtonAnd);
+    new_base->append(std::move(query_tree));
+
+    if (locationSpec.getRankOnDistance()) {
+        new_base->append(Node::UP(new ProtonLocationTerm(loc, view, id, weight)));
+        fef_location.setAttribute(view);
+        fef_location.setXPosition(locationSpec.getX());
+        fef_location.setYPosition(locationSpec.getY());
+        fef_location.setXAspect(locationSpec.getXAspect());
+        fef_location.setValid(true);
+    } else if (locationSpec.getPruneOnDistance()) {
+        new_base->append(Node::UP(new ProtonLocationTerm(loc, view, id, weight)));
+    }
+    query_tree = std::move(new_base);
+}
+}  // namespace
+
+bool
+Query::buildTree(const vespalib::stringref &stack, const string &location,
+                 const ViewResolver &resolver,
+                 const search::fef::IIndexEnvironment &indexEnv)
+{
+    SimpleQueryStackDumpIterator stack_dump_iterator(stack);
+    _query_tree =
+        QueryTreeCreator<ProtonNodeTypes>::create(stack_dump_iterator);
+    if (_query_tree.get()) {
+        AddLocationNode(location, _query_tree, _location);
+        ResolveViewVisitor resolve_visitor(resolver, indexEnv);
+        _query_tree->accept(resolve_visitor);
+        return true;
+    } else {
+        // TODO(havardpe): log warning or pass message upwards
+        return false;
+    }
+}
+
+void
+Query::extractTerms(vector<const ITermData *> &terms)
+{
+    TermDataExtractor::extractTerms(*_query_tree, terms);
+}
+
+void
+Query::extractLocations(vector<const search::fef::Location *> &locations)
+{
+    locations.clear();
+    locations.push_back(&_location);
+}
+
+void
+Query::setBlackListBlueprint(Blueprint::UP blackListBlueprint)
+{
+    _blackListBlueprint = std::move(blackListBlueprint);
+}
+
+void
+Query::reserveHandles(const IRequestContext & requestContext,
+                      ISearchContext &context,
+                      MatchDataLayout &mdl)
+{
+    MatchDataReserveVisitor reserve_visitor(mdl);
+    _query_tree->accept(reserve_visitor);
+
+    _blueprint = BlueprintBuilder::build(requestContext, *_query_tree, context);
+    LOG(debug, "original blueprint:\n%s\n", _blueprint->asString().c_str());
+    if (_blackListBlueprint.get() != NULL) {
+        std::unique_ptr<AndNotBlueprint> andNotBlueprint(new AndNotBlueprint());
+        (*andNotBlueprint)
+            .addChild(std::move(_blueprint))
+            .addChild(std::move(_blackListBlueprint));
+        _blueprint = std::move(andNotBlueprint);
+        _blueprint->setDocIdLimit(context.getDocIdLimit());
+        LOG(debug, "blueprint after black listing:\n%s\n", _blueprint->asString().c_str());
+    }
+}
+
+void
+Query::optimize()
+{
+    _blueprint = Blueprint::optimize(std::move(_blueprint));
+    LOG(debug, "optimized blueprint:\n%s\n", _blueprint->asString().c_str());
+}
+
+void
+Query::fetchPostings(void)
+{
+    _blueprint->getState(); // ensure final state is cached to ensure thread safety
+    _blueprint->fetchPostings(true);
+}
+
+SearchIterator::UP
+Query::createSearch(MatchData &md) const
+{
+    return _blueprint->createSearch(md, true);
+}
+
+}  // namespace matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h
new file mode 100644
index 00000000000..dcc902cf5b0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/query.h
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/fef/location.h>
+#include <vespa/searchlib/query/tree/node.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/queryeval/irequestcontext.h>
+
+namespace search { namespace fef { class ITermData; }}
+namespace search { namespace fef { class MatchDataLayout; }}
+namespace search { namespace fef { class IIndexEnvironment; }}
+
+namespace proton {
+namespace matching {
+
+class ViewResolver;
+class ISearchContext;
+
+class Query
+{
+private:
+    search::query::Node::UP          _query_tree;
+    search::queryeval::Blueprint::UP _blueprint;
+    search::fef::Location            _location;
+    search::queryeval::Blueprint::UP _blackListBlueprint;
+
+public:
+    /**
+     * Build query tree from a stack dump.
+     *
+     * @return success(true)/failure(false)
+     **/
+    bool buildTree(const vespalib::stringref &stack,
+                   const vespalib::string &location,
+                   const ViewResolver &resolver,
+                   const search::fef::IIndexEnvironment &idxEnv);
+
+    /**
+     * Extract query terms from the query tree; to be used to build
+     * the query environment.
+     *
+     * @param terms where to collect terms
+     **/
+    void extractTerms(std::vector<const search::fef::ITermData *> &terms);
+
+    /**
+     * Extract locations from the query tree; to be used to build
+     * the query environment.
+     *
+     * @param locs where to collect locations
+     **/
+    void extractLocations(std::vector<const search::fef::Location *> &locs);
+
+    /**
+     * Use the given blueprint as black list node in the blueprint tree.
+     * The search iterator created by this blueprint should return all
+     * invisible / inactive documents as hits. These hits will then not be
+     * part of the result set for the query executed.
+     *
+     * @param blackListBlueprint the blueprint used for black listing.
+     **/
+    void setBlackListBlueprint(search::queryeval::Blueprint::UP blackListBlueprint);
+
+    /**
+     * Reserve room for terms in the query in the given match data
+     * layout. This function also prepares the createSearch function
+     * for use.
+     *
+     * @param context search context
+     * @param mdl match data layout
+     **/
+    void reserveHandles(const search::queryeval::IRequestContext & requestContext,
+                        ISearchContext &context,
+                        search::fef::MatchDataLayout &mdl);
+
+    /**
+     * Optimize the query to be executed. This function should be
+     * called after the reserveHandles function and before the
+     * fetchPostings function. The only reason this is a separate
+     * function is that the query optimization is so awesome that
+     * testing becomes harder. Not calling this function enables the
+     * test to verify the original query without optimization.
+     **/
+    void optimize();
+    void fetchPostings(void);
+
+    /**
+     * Create the actual search iterator tree used to find matches.
+     *
+     * @return iterator tree
+     * @param md match data used for feature unpacking
+     **/
+    search::queryeval::SearchIterator::UP
+    createSearch(search::fef::MatchData &md) const;
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp
new file mode 100644
index 00000000000..e1ad0bd0400
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.queryenvironment");
+
+#include "queryenvironment.h"
+
+using search::attribute::IAttributeContext;
+using search::fef::IIndexEnvironment;
+using search::fef::Location;
+using search::fef::Properties;
+
+namespace proton {
+namespace matching {
+
+QueryEnvironment::QueryEnvironment(const IIndexEnvironment &indexEnv,
+                                   const IAttributeContext &attrContext,
+                                   const Properties &properties)
+    : _indexEnv(indexEnv),
+      _attrContext(attrContext),
+      _properties(properties),
+      _locations(1),
+      _terms()
+{
+}
+
+const search::fef::Properties &
+QueryEnvironment::getProperties() const
+{
+    return _properties;
+}
+
+uint32_t
+QueryEnvironment::getNumTerms() const
+{
+    return _terms.size();
+}
+
+const search::fef::ITermData *
+QueryEnvironment::getTerm(uint32_t idx) const
+{
+    if (idx >= _terms.size()) {
+        return 0;
+    }
+    return _terms[idx];
+}
+
+const search::fef::Location &
+QueryEnvironment::getLocation() const
+{
+    return *_locations[0];
+}
+
+const IAttributeContext &
+QueryEnvironment::getAttributeContext() const
+{
+    return _attrContext;
+}
+
+const search::fef::IIndexEnvironment &
+QueryEnvironment::getIndexEnvironment() const
+{
+    return _indexEnv;
+}
+
+QueryEnvironment::~QueryEnvironment()
+{
+}
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h
new file mode 100644
index 00000000000..490379de7ae
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/fef/iqueryenvironment.h>
+#include <vespa/searchlib/fef/properties.h>
+#include <vespa/searchlib/fef/location.h>
+
+namespace proton {
+namespace matching {
+
+/**
+ * Query environment implementation for the proton matching pipeline.
+ **/
+class QueryEnvironment : public search::fef::IQueryEnvironment
+{
+private:
+    const search::fef::IIndexEnvironment       &_indexEnv;
+    const search::attribute::IAttributeContext &_attrContext;
+    search::fef::Properties                     _properties;
+    std::vector<const search::fef::Location *>  _locations;
+    std::vector<const search::fef::ITermData *> _terms;
+
+    QueryEnvironment(const QueryEnvironment &);
+    QueryEnvironment &operator=(const QueryEnvironment &);
+
+public:
+    /**
+     * Set up a new query environment.
+     *
+     * @param indexEnv index environment; referenced, not copied
+     * @param attrContext attribute context; referenced, not copied
+     * @param properties properties; copied
+     **/
+    QueryEnvironment(const search::fef::IIndexEnvironment &indexEnv,
+                     const search::attribute::IAttributeContext &attrContext,
+                     const search::fef::Properties &properties);
+
+    /**
+     * Used to edit the list of terms by the one setting up this query
+     * environment.
+     *
+     * @return modifiable list of terms data pointers
+     **/
+    std::vector<const search::fef::ITermData *> &terms() { return _terms; }
+
+    /**
+     * Used to edit the list of locations by the one setting up this
+     * query environment.
+     *
+     * Initially, only the first location in this list is made
+     * available through the IQueryEnvironment interface.
+     *
+     * @return modifiable list of location data pointers
+     **/
+    std::vector<const search::fef::Location *> &locations() {
+        return _locations;
+    }
+
+    // inherited from search::fef::IQueryEnvironment
+    virtual const search::fef::Properties &getProperties() const;
+
+    // inherited from search::fef::IQueryEnvironment
+    virtual uint32_t getNumTerms() const;
+
+    // inherited from search::fef::IQueryEnvironment
+    virtual const search::fef::ITermData *getTerm(uint32_t idx) const;
+
+    // inherited from search::fef::IQueryEnvironment
+    virtual const search::fef::Location & getLocation() const;
+
+    // inherited from search::fef::IQueryEnvironment
+    virtual const search::attribute::IAttributeContext & getAttributeContext() const;
+
+    // inherited from search::fef::IQueryEnvironment
+    virtual const search::fef::IIndexEnvironment & getIndexEnvironment() const;
+
+    virtual ~QueryEnvironment();
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/querylimiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/querylimiter.cpp
new file mode 100644
index 00000000000..77232db1d5a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/querylimiter.cpp
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/searchcore/proton/matching/querylimiter.h>
+#include <limits>
+
+namespace proton {
+namespace matching {
+
+QueryLimiter::LimitedToken::LimitedToken(const Doom & doom, QueryLimiter & limiter) :
+    _limiter(limiter)
+{
+    _limiter.grabToken(doom);
+}
+
+QueryLimiter::LimitedToken::~LimitedToken()
+{
+    _limiter.releaseToken();
+}
+
+void
+QueryLimiter::grabToken(const Doom & doom)
+{
+    vespalib::MonitorGuard guard(_monitor);
+    while ((_maxThreads > 0) && (_activeThreads >= _maxThreads) && !doom.doom()) {
+        int left = doom.left().ms();
+        if (left > 0) {
+            guard.wait(left);
+        }
+    }
+    _activeThreads++;
+}
+
+void
+QueryLimiter::releaseToken()
+{
+    vespalib::MonitorGuard guard(_monitor);
+    _activeThreads--;
+    guard.signal();
+}
+
+QueryLimiter::QueryLimiter() :
+    _monitor(),
+    _activeThreads(0),
+    _maxThreads(-1),
+    _coverage(1.0),
+    _minHits(std::numeric_limits<uint32_t>::max())
+{
+}
+
+void
+QueryLimiter::configure(int maxThreads, double coverage, uint32_t minHits)
+{
+    _maxThreads = maxThreads;
+    _coverage = coverage;
+    _minHits = minHits;
+}
+
+QueryLimiter::Token::UP
+QueryLimiter::getToken(const Doom & doom, uint32_t numDocs, uint32_t numHits, bool hasSorting, bool hasGrouping)
+{
+    if (_maxThreads > 0) {
+        if (hasSorting || hasGrouping) {
+            if (numHits > _minHits) {
+                if (numDocs * _coverage < numHits) {
+                    return Token::UP(new LimitedToken(doom, *this));
+                }
+            }
+        }
+    }
+    return Token::UP(new NoLimitToken());
+}
+
+}
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/querylimiter.h b/searchcore/src/vespa/searchcore/proton/matching/querylimiter.h
new file mode 100644
index 00000000000..11c05673a9b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/querylimiter.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/doom.h>
+
+namespace proton {
+namespace matching {
+
+class QueryLimiter
+{
+private:
+    typedef vespalib::Doom Doom;
+public:
+    class Token {
+    public:
+        typedef std::unique_ptr<Token> UP;
+        virtual ~Token() { }
+    };
+public:
+    QueryLimiter();
+    void configure(int maxThreads, double coverage, uint32_t minHits);
+    Token::UP getToken(const Doom & doom, uint32_t numDocs, uint32_t numHits, bool hasSorting, bool hasGrouping);
+private:
+    class NoLimitToken : public Token {
+    };
+    class LimitedToken : public Token {
+    private:
+        QueryLimiter & _limiter;
+    public:
+        LimitedToken(const Doom & doom, QueryLimiter & limiter);
+        virtual ~LimitedToken();
+    };
+    void grabToken(const Doom & doom);
+    void releaseToken();
+    vespalib::Monitor _monitor;
+    volatile int _activeThreads;
+
+    // These are updated asynchronously at reconfig.
+    volatile int      _maxThreads;
+    volatile double   _coverage;
+    volatile uint32_t _minHits;
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp
new file mode 100644
index 00000000000..2acb9c3a165
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp
@@ -0,0 +1,146 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.querynodes");
+
+#include "querynodes.h"
+#include "termdatafromnode.h"
+#include "viewresolver.h"
+#include "handlerecorder.h"
+#include <vespa/searchlib/fef/fieldinfo.h>
+#include <vespa/searchlib/fef/iindexenvironment.h>
+#include <vespa/searchlib/fef/matchdata.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/searchlib/fef/simpletermdata.h>
+#include <vespa/searchlib/query/tree/templatetermvisitor.h>
+#include <vespa/searchlib/queryeval/orsearch.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
+#include <map>
+#include <vector>
+
+using search::fef::FieldInfo;
+using search::fef::FieldType;
+using search::fef::IIndexEnvironment;
+using search::fef::MatchData;
+using search::fef::MatchDataLayout;
+using search::fef::SimpleTermData;
+using search::fef::TermFieldHandle;
+using search::query::Node;
+using search::query::TemplateTermVisitor;
+using search::query::Weight;
+using search::queryeval::OrSearch;
+using search::queryeval::SearchIterator;
+using std::map;
+using std::vector;
+using vespalib::string;
+
+namespace proton {
+namespace matching {
+
+void
+ProtonTermData::setDocumentFrequency(double freq)
+{
+    for (size_t i = 0; i < _fields.size(); ++i) {
+        _fields[i].setDocFreq(freq);
+    }
+}
+
+void
+ProtonTermData::resolve(const ViewResolver &resolver,
+                         const IIndexEnvironment &idxEnv,
+                         const string &view,
+                         bool forceFilter)
+{
+    std::vector<string> fields;
+    resolver.resolve(((view == "") ? "default" : view), fields);
+    _fields.clear();
+    _fields.reserve(fields.size());
+    for (size_t i = 0; i < fields.size(); ++i) {
+        const FieldInfo *info = idxEnv.getFieldByName(fields[i]);
+        if (info != 0) {
+            _fields.push_back(FieldEntry(fields[i], info->id()));
+            _fields.back().attribute_field =
+                (info->type() == FieldType::ATTRIBUTE) ||
+                (info->type() == FieldType::HIDDEN_ATTRIBUTE);
+            _fields.back().filter_field = forceFilter ? true : info->isFilter();
+        } else {
+            LOG(debug, "ignoring undefined field: '%s'", fields[i].c_str());
+        }
+    }
+}
+
+void
+ProtonTermData::resolveFromChildren(const std::vector<Node *> &subterms)
+{
+    for (size_t i = 0; i < subterms.size(); ++i) {
+        const ProtonTermData *child = termDataFromNode(*subterms[i]);
+        if (child == 0) {
+            LOG(warning, "child of equiv is not a term");
+            continue;
+        }
+        for (size_t j = 0; j < child->numFields(); ++j) {
+            FieldSpec subSpec = child->field(j).fieldSpec();
+            if (lookupField(subSpec.getFieldId()) == 0) {
+                // this must happen before handles are reserved
+                LOG_ASSERT(subSpec.getHandle() == search::fef::IllegalHandle);
+                _fields.push_back(FieldEntry(subSpec.getName(), subSpec.getFieldId()));
+            }
+        }
+    }
+}
+
+void
+ProtonTermData::allocateTerms(MatchDataLayout &mdl)
+{
+    for (size_t i = 0; i < _fields.size(); ++i) {
+        _fields[i].setHandle(mdl.allocTermField(_fields[i].getFieldId()));
+    }
+}
+
+void
+ProtonTermData::setDocumentFrequency(uint32_t estHits, uint32_t docIdLimit)
+{
+    if (docIdLimit > 1) {
+        double hits = estHits;
+        setDocumentFrequency(hits / (docIdLimit - 1));
+    } else {
+        setDocumentFrequency(0.0);
+    }
+}
+
+size_t
+ProtonTermData::numFields() const
+{
+    return _fields.size();
+}
+
+const ProtonTermData::FieldEntry &
+ProtonTermData::field(size_t i) const
+{
+    assert(i < _fields.size());
+    return _fields[i];
+}
+
+const ProtonTermData::FieldEntry *
+ProtonTermData::lookupField(uint32_t fieldId) const
+{
+    for (size_t i = 0; i < numFields(); ++i) {
+        if (field(i).getFieldId() == fieldId) {
+            return &field(i);
+        }
+    }
+    return 0;
+}
+
+TermFieldHandle
+ProtonTermData::FieldEntry::getHandle() const
+{
+    TermFieldHandle handle(search::fef::SimpleTermFieldData::getHandle());
+    HandleRecorder::registerHandle(handle);
+    return handle;
+}
+
+
+}  // namespace matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h
new file mode 100644
index 00000000000..63052d633d1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h
@@ -0,0 +1,152 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/fef/iindexenvironment.h>
+#include <vespa/searchlib/fef/itermdata.h>
+#include <vespa/searchlib/fef/simpletermdata.h>
+#include <vespa/searchlib/fef/simpletermfielddata.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/searchlib/query/tree/intermediatenodes.h>
+#include <vespa/searchlib/query/tree/termnodes.h>
+#include <vespa/searchlib/query/tree/simplequery.h>
+#include <vespa/searchlib/query/weight.h>
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <memory>
+#include <vector>
+
+namespace proton {
+namespace matching {
+
+class ViewResolver;
+
+class ProtonTermData : public search::fef::ITermData
+{
+public:
+    typedef search::queryeval::FieldSpec FieldSpec;
+
+    struct FieldEntry : search::fef::SimpleTermFieldData {
+        vespalib::string field_name;
+        bool attribute_field;
+        bool filter_field;
+
+        FieldEntry(const vespalib::string &name, uint32_t fieldId)
+            : SimpleTermFieldData(fieldId),
+              field_name(name),
+              attribute_field(false),
+              filter_field(false) {}
+
+        FieldSpec fieldSpec() const {
+            return FieldSpec(field_name, getFieldId(),
+                             getHandle(), filter_field);
+        }
+        virtual search::fef::TermFieldHandle getHandle() const;
+    };
+
+private:
+    std::vector<FieldEntry> _fields;
+
+    void setDocumentFrequency(double docFreq);
+
+protected:
+    void resolve(const ViewResolver &resolver,
+                 const search::fef::IIndexEnvironment &idxEnv,
+                 const vespalib::string &view,
+                 bool forceFilter);
+
+public:
+    void resolveFromChildren(const std::vector<search::query::Node *> &children);
+    void allocateTerms(search::fef::MatchDataLayout &mdl);
+    void setDocumentFrequency(uint32_t estHits, uint32_t numDocs);
+
+    // ITermData interface
+    virtual size_t numFields() const;
+    virtual const FieldEntry &field(size_t i) const;
+    virtual const FieldEntry *lookupField(uint32_t fieldId) const;
+};
+
+namespace {
+template <typename NodeType> uint32_t numTerms(const NodeType &) { return 1; }
+template <>
+uint32_t numTerms<search::query::Phrase>(const search::query::Phrase &n) {
+    return n.getChildren().size();
+}
+} // namespace proton::matching::<unnamed>
+
+template <typename Base>
+struct ProtonTerm : public Base,
+                    public ProtonTermData
+{
+    using Base::Base;
+
+    void resolve(const ViewResolver &resolver,
+                 const search::fef::IIndexEnvironment &idxEnv)
+    {
+        bool forceFilter = !Base::usePositionData();
+        ProtonTermData::resolve(resolver, idxEnv, Base::getView(), forceFilter);
+    }
+
+    // ITermData interface
+    virtual uint32_t getPhraseLength() const { return numTerms<Base>(*this); }
+    virtual uint32_t getTermIndex() const { return -1; }
+    virtual search::query::Weight getWeight() const { return Base::getWeight(); }
+    virtual uint32_t getUniqueId() const { return Base::getId(); }
+};
+
+typedef search::query::SimpleAnd     ProtonAnd;
+typedef search::query::SimpleAndNot  ProtonAndNot;
+typedef search::query::SimpleNear    ProtonNear;
+typedef search::query::SimpleONear   ProtonONear;
+typedef search::query::SimpleOr      ProtonOr;
+typedef search::query::SimpleRank    ProtonRank;
+typedef search::query::SimpleWeakAnd ProtonWeakAnd;
+
+struct ProtonEquiv : public ProtonTerm<search::query::Equiv>
+{
+    search::fef::MatchDataLayout children_mdl;
+    using ProtonTerm::ProtonTerm;
+};
+
+typedef ProtonTerm<search::query::LocationTerm>    ProtonLocationTerm;
+typedef ProtonTerm<search::query::NumberTerm>      ProtonNumberTerm;
+typedef ProtonTerm<search::query::Phrase>          ProtonPhrase;
+typedef ProtonTerm<search::query::PrefixTerm>      ProtonPrefixTerm;
+typedef ProtonTerm<search::query::RangeTerm>       ProtonRangeTerm;
+typedef ProtonTerm<search::query::StringTerm>      ProtonStringTerm;
+typedef ProtonTerm<search::query::SubstringTerm>   ProtonSubstringTerm;
+typedef ProtonTerm<search::query::SuffixTerm>      ProtonSuffixTerm;
+typedef ProtonTerm<search::query::WeightedSetTerm> ProtonWeightedSetTerm;
+typedef ProtonTerm<search::query::DotProduct>      ProtonDotProduct;
+typedef ProtonTerm<search::query::WandTerm>        ProtonWandTerm;
+typedef ProtonTerm<search::query::PredicateQuery>  ProtonPredicateQuery;
+typedef ProtonTerm<search::query::RegExpTerm>      ProtonRegExpTerm;
+
+struct ProtonNodeTypes {
+    typedef ProtonAnd             And;
+    typedef ProtonAndNot          AndNot;
+    typedef ProtonEquiv           Equiv;
+    typedef ProtonLocationTerm    LocationTerm;
+    typedef ProtonNear            Near;
+    typedef ProtonNumberTerm      NumberTerm;
+    typedef ProtonONear           ONear;
+    typedef ProtonOr              Or;
+    typedef ProtonPhrase          Phrase;
+    typedef ProtonPrefixTerm      PrefixTerm;
+    typedef ProtonRangeTerm       RangeTerm;
+    typedef ProtonRank            Rank;
+    typedef ProtonStringTerm      StringTerm;
+    typedef ProtonSubstringTerm   SubstringTerm;
+    typedef ProtonSuffixTerm      SuffixTerm;
+    typedef ProtonWeakAnd         WeakAnd;
+    typedef ProtonWeightedSetTerm WeightedSetTerm;
+    typedef ProtonDotProduct      DotProduct;
+    typedef ProtonWandTerm        WandTerm;
+    typedef ProtonPredicateQuery  PredicateQuery;
+    typedef ProtonRegExpTerm      RegExpTerm;
+};
+
+}  // namespace matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/requestcontext.cpp b/searchcore/src/vespa/searchcore/proton/matching/requestcontext.cpp
new file mode 100644
index 00000000000..9c35ab316af
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/requestcontext.cpp
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "requestcontext.h"
+
+namespace proton {
+
+RequestContext::RequestContext(const vespalib::Doom & doom, search::attribute::IAttributeContext & attributeContext) :
+    _doom(doom),
+    _attributeContext(attributeContext)
+{ }
+
+const vespalib::Doom &
+RequestContext::getDoom() const
+{
+    return _doom;
+}
+
+const search::AttributeVector *
+RequestContext::getAttribute(const vespalib::string & name) const
+{
+    const search::attribute::IAttributeVector * iav = _attributeContext.getAttribute(name);
+    return dynamic_cast<const search::AttributeVector *>(iav);
+}
+
+const search::AttributeVector *
+RequestContext::getAttributeStableEnum(const vespalib::string & name) const
+{
+    const search::attribute::IAttributeVector * iav = _attributeContext.getAttributeStableEnum(name);
+    return dynamic_cast<const search::AttributeVector *>(iav);
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/requestcontext.h b/searchcore/src/vespa/searchcore/proton/matching/requestcontext.h
new file mode 100644
index 00000000000..9f52ae23ac3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/requestcontext.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/queryeval/irequestcontext.h>
+#include <vespa/searchcommon/attribute/iattributecontext.h>
+
+namespace proton {
+
+class RequestContext : public search::queryeval::IRequestContext
+{
+public:
+    RequestContext(const vespalib::Doom & doom, search::attribute::IAttributeContext & attributeContext);
+    const vespalib::Doom & getDoom() const override;
+    const search::AttributeVector * getAttribute(const vespalib::string & name) const override;
+    const search::AttributeVector * getAttributeStableEnum(const vespalib::string & name) const override;
+private:
+    const vespalib::Doom                   _doom;
+    search::attribute::IAttributeContext & _attributeContext;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.h b/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.h
new file mode 100644
index 00000000000..475d058be9f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "querynodes.h"
+#include "viewresolver.h"
+#include <vespa/searchlib/query/tree/templatetermvisitor.h>
+#include <vespa/searchlib/fef/iindexenvironment.h>
+
+namespace proton {
+namespace matching {
+
+class ResolveViewVisitor
+    : public search::query::TemplateTermVisitor<ResolveViewVisitor,
+                                                ProtonNodeTypes>
+{
+    const ViewResolver &_resolver;
+    const search::fef::IIndexEnvironment &_indexEnv;
+
+public:
+    ResolveViewVisitor(const matching::ViewResolver &resolver,
+                       const search::fef::IIndexEnvironment &indexEnv)
+        : _resolver(resolver), _indexEnv(indexEnv) {}
+
+    template <class TermNode>
+    void visitTerm(TermNode &n) { n.resolve(_resolver, _indexEnv); }
+
+    virtual void visit(ProtonNodeTypes::Equiv &n) {
+        visitChildren(n);
+        n.resolveFromChildren(n.getChildren());
+    }
+};
+
+}  // namespace matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
new file mode 100644
index 00000000000..e4e12fc10c2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
@@ -0,0 +1,122 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.result_processor");
+#include "result_processor.h"
+#include <vespa/searchlib/common/sortresults.h>
+#include <vespa/searchlib/common/docstamp.h>
+
+namespace proton {
+namespace matching {
+
+ResultProcessor::ResultProcessor(search::attribute::IAttributeContext &attrContext,
+                                 const search::IDocumentMetaStore &metaStore,
+                                 SessionManager &sessionMgr,
+                                 search::grouping::GroupingContext &groupingContext,
+                                 const search::grouping::SessionId &sessionId,
+                                 const vespalib::string &sortSpec,
+                                 size_t offset, size_t hits)
+    : _attrContext(attrContext),
+      _metaStore(metaStore),
+      _sessionMgr(sessionMgr),
+      _groupingContext(groupingContext),
+      _groupingSession(),
+      _sortSpec(sortSpec),
+      _offset(offset),
+      _hits(hits),
+      _result(),
+      _wasMerged(false)
+{
+    if (!_groupingContext.empty()) {
+        _groupingSession.reset(new search::grouping::GroupingSession(sessionId,
+                                       _groupingContext, attrContext));
+    }
+}
+
+void
+ResultProcessor::prepareThreadContextCreation(size_t num_threads)
+{
+    if (num_threads > 1) {
+        _wasMerged = true;
+    }
+    if (_groupingSession.get() != 0) {
+        _groupingSession->prepareThreadContextCreation(num_threads);
+    }
+}
+
+ResultProcessor::Context::UP
+ResultProcessor::createThreadContext(const vespalib::Doom & doom, size_t thread_id)
+{
+    Sort::UP sort(new Sort(doom, _attrContext, _sortSpec));
+    PartialResult::LP result(new PartialResult((_offset + _hits), sort->hasSortData()));
+    if (thread_id == 0) {
+        _result = result;
+    }
+    search::grouping::GroupingContext::UP groupingContext;
+    if (_groupingSession.get() != 0) {
+        groupingContext = _groupingSession->createThreadContext(thread_id, _attrContext);
+    }
+    return Context::UP(new Context(std::move(sort), result, std::move(groupingContext)));
+}
+
+ResultProcessor::Result::UP
+ResultProcessor::makeReply()
+{
+    search::engine::SearchReply::UP reply(new search::engine::SearchReply());
+    const search::IDocumentMetaStore &metaStore = _metaStore;
+    search::engine::SearchReply &r = *reply;
+    PartialResult &result = *_result;
+    size_t numFs4Hits(0);
+    if (_groupingSession) {
+        if (_wasMerged) {
+            _groupingSession->getGroupingManager().prune();
+        }
+        _groupingSession->getGroupingManager().convertToGlobalId(metaStore);
+        _groupingSession->continueExecution(_groupingContext);
+        numFs4Hits = _groupingContext.countFS4Hits();
+        _groupingContext.getResult().swap(r.groupResult);
+        if (!_groupingSession->getSessionId().empty() && !_groupingSession->finished()) {
+            _sessionMgr.insert(std::move(_groupingSession));
+        }
+    }
+    uint32_t hitOffset = _offset;
+    uint32_t hitcnt    = (result.size() > hitOffset) ? (result.size() - hitOffset) : 0;
+    r.offset           = hitOffset;
+    r.totalHitCount    = result.totalHits();
+    r.hits.resize(hitcnt);
+    document::GlobalId gid;
+    for (size_t i = 0; i < hitcnt; ++i) {
+        search::engine::SearchReply::Hit &dst = r.hits[i];
+        const search::RankedHit &src = result.hit(hitOffset + i);
+        uint32_t docId = src._docId;
+        if (metaStore.getGid(docId, gid)) {
+            dst.gid = gid;
+        }
+        dst.metric = src._rankValue;
+        LOG(debug, "convertLidToGid: hit[%zu]: lid(%u) -> gid(%s)",
+            i, docId, dst.gid.toString().c_str());
+    }
+    if (result.hasSortData() && hitcnt > 0) {
+        size_t sortDataSize = result.sortDataSize();
+        for (size_t i = 0; i < hitOffset; ++i) {
+            sortDataSize -= result.sortData(i).second;
+        }
+        r.sortIndex.resize(hitcnt + 1);
+        r.sortData.resize(sortDataSize);
+        uint32_t sortOffset = 0;
+        for (size_t i = 0; i < hitcnt; ++i) {
+            const PartialResult::SortRef &sr = result.sortData(hitOffset + i);
+            r.sortIndex[i] = sortOffset;
+            memcpy(&r.sortData[0] + sortOffset, sr.first, sr.second);
+            sortOffset += sr.second;
+        }
+        r.sortIndex[hitcnt] = sortOffset;
+        assert(sortOffset == sortDataSize);
+    }
+    numFs4Hits += reply->hits.size();
+    return Result::UP(new Result(std::move(reply), numFs4Hits));
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h
new file mode 100644
index 00000000000..6d64138e549
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h
@@ -0,0 +1,115 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "partial_result.h"
+#include "result_processor.h"
+#include "sessionmanager.h"
+#include <vespa/searchcore/grouping/groupingcontext.h>
+#include <vespa/searchcore/grouping/groupingmanager.h>
+#include <vespa/searchcore/grouping/groupingsession.h>
+#include <vespa/searchlib/attribute/attributecontext.h>
+#include <vespa/searchlib/common/sortresults.h>
+#include <vespa/searchlib/common/idocumentmetastore.h>
+#include <vespa/searchlib/common/resultset.h>
+#include <vespa/searchlib/engine/searchreply.h>
+#include <vespa/vespalib/util/dual_merge_director.h>
+#include <vespa/vespalib/util/noncopyable.hpp>
+
+namespace proton {
+namespace matching {
+
+class ResultProcessor
+{
+public:
+    /**
+     * Sorter selection and owner of additional data needed for
+     * multi-level sorting.
+     **/
+    struct Sort : vespalib::noncopyable {
+        typedef std::unique_ptr<Sort> UP;
+        FastS_IResultSorter *sorter;
+        FastS_SortSpec       sortSpec;
+        Sort(const vespalib::Doom & doom, search::attribute::IAttributeContext &ac, const vespalib::string &ss)
+            : sorter(FastS_DefaultResultSorter::instance()),
+              sortSpec(doom)
+        {
+            if (!ss.empty() && sortSpec.Init(ss.c_str(), ac)) {
+                sorter = &sortSpec;
+            }
+        }
+        bool hasSortData() const {
+            return (sorter == (const FastS_IResultSorter *) &sortSpec);
+        }
+    };
+
+    /**
+     * Adapter to use grouping contexts as merging sources.
+     **/
+    struct GroupingSource : vespalib::DualMergeDirector::Source {
+        search::grouping::GroupingContext *ctx;
+        GroupingSource(search::grouping::GroupingContext *g) : ctx(g) {}
+        virtual void merge(Source &s) {
+            GroupingSource &rhs = static_cast<GroupingSource&>(s);
+            assert((ctx == 0) == (rhs.ctx == 0));
+            if (ctx != 0) {
+                search::grouping::GroupingManager man(*ctx);
+                man.merge(*rhs.ctx);
+            }
+        }
+    };
+
+    /**
+     * Context per thread used for result processing.
+     **/
+    struct Context {
+        typedef std::unique_ptr<Context> UP;
+
+        Sort::UP                              sort;
+        PartialResult::LP                     result;
+        search::grouping::GroupingContext::UP grouping;
+        GroupingSource                        groupingSource;
+
+        Context(Sort::UP s, PartialResult::LP r,
+                search::grouping::GroupingContext::UP g)
+            : sort(std::move(s)), result(r), grouping(std::move(g)),
+              groupingSource(grouping.get()) {}
+    };
+
+    struct Result {
+        typedef std::unique_ptr<Result> UP;
+        Result(search::engine::SearchReply::UP reply, size_t numFs4Hits) : _reply(std::move(reply)), _numFs4Hits(numFs4Hits) { }
+        search::engine::SearchReply::UP _reply;
+        size_t _numFs4Hits;
+    };
+
+private:
+    search::attribute::IAttributeContext  &_attrContext;
+    const search::IDocumentMetaStore      &_metaStore;
+    SessionManager                        &_sessionMgr;
+    search::grouping::GroupingContext     &_groupingContext;
+    search::grouping::GroupingSession::UP  _groupingSession;
+    const vespalib::string                &_sortSpec;
+    size_t                                 _offset;
+    size_t                                 _hits;
+    PartialResult::LP                      _result;
+    bool                                   _wasMerged;
+
+public:
+    ResultProcessor(search::attribute::IAttributeContext &attrContext,
+                    const search::IDocumentMetaStore &metaStore,
+                    SessionManager &sessionMgr,
+                    search::grouping::GroupingContext &groupingContext,
+                    const search::grouping::SessionId &sessionId,
+                    const vespalib::string &sortSpec,
+                    size_t offset, size_t hits);
+
+    size_t countFS4Hits();
+    void prepareThreadContextCreation(size_t num_threads);
+    Context::UP createThreadContext(const vespalib::Doom & doom, size_t thread_id);
+    Result::UP makeReply();
+};
+
+} // namespace proton::matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/search_session.h b/searchcore/src/vespa/searchcore/proton/matching/search_session.h
new file mode 100644
index 00000000000..5e0a50ce774
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/search_session.h
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "match_context.h"
+#include "match_tools.h"
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <memory>
+
+namespace proton {
+namespace matching {
+
+/**
+ * Holds enough data to perform a GetDocSum request. Makes sure the
+ * data is kept alive.
+ */
+class SearchSession {
+public:
+    struct OwnershipBundle {
+        ISearchHandler::SP search_handler;
+        search::fef::Properties::UP feature_overrides;
+        MatchContext::UP context;
+    };
+private:
+    typedef vespalib::string SessionId;
+
+    SessionId _session_id;
+    fastos::TimeStamp _create_time;
+    fastos::TimeStamp _time_of_doom;
+    OwnershipBundle _owned_objects;
+    MatchToolsFactory::UP _match_tools_factory;
+
+public:
+    typedef std::shared_ptr<SearchSession> SP;
+
+    SearchSession(const SessionId &id, fastos::TimeStamp time_of_doom,
+                  MatchToolsFactory::UP match_tools_factory,
+                  OwnershipBundle &&owned_objects)
+        : _session_id(id),
+          _create_time(fastos::ClockSystem::now()),
+          _time_of_doom(time_of_doom),
+          _owned_objects(std::move(owned_objects)),
+          _match_tools_factory(std::move(match_tools_factory)) {
+    }
+
+    const SessionId &getSessionId() const { return _session_id; }
+
+    void releaseEnumGuards() { _owned_objects.context->releaseEnumGuards(); }
+
+    /**
+     * Gets this session's create time.
+     */
+    fastos::TimeStamp getCreateTime() const { return _create_time; }
+
+    /**
+     * Gets this session's timeout.
+     */
+    fastos::TimeStamp getTimeOfDoom() const { return _time_of_doom; }
+
+    MatchToolsFactory &getMatchToolsFactory() { return *_match_tools_factory; }
+};
+
+}  // namespace proton::matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp
new file mode 100644
index 00000000000..33f002f6bf8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "session_manager_explorer.h"
+#include <vespa/vespalib/data/slime/slime.h>
+
+using vespalib::slime::Inserter;
+using vespalib::slime::Cursor;
+using vespalib::StateExplorer;
+
+namespace proton {
+namespace matching {
+
+namespace {
+
+const vespalib::string SEARCH = "search";
+
+class SearchSessionExplorer : public vespalib::StateExplorer
+{
+private:
+    const SessionManager &_manager;
+
+public:
+    SearchSessionExplorer(const SessionManager &manager) : _manager(manager) {}
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override {        
+        Cursor &state = inserter.insertObject();
+        state.setLong("numSessions", _manager.getNumSearchSessions());
+        if (full) {
+            std::vector<SessionManager::SearchSessionInfo> sessions = _manager.getSortedSearchSessionInfo();
+            Cursor &array = state.setArray("sessions");
+            for (const auto &session: sessions) {
+                Cursor &entry = array.addObject();
+                entry.setString("id", session.id);
+                entry.setString("created", session.created.toString());
+                entry.setString("doom", session.doom.toString());
+            }
+        }
+    }
+};
+
+} // namespace proton::matching::<unnamed>
+
+void
+SessionManagerExplorer::get_state(const Inserter &, bool) const
+{
+}
+
+std::vector<vespalib::string>
+SessionManagerExplorer::get_children_names() const
+{
+    return std::vector<vespalib::string>({SEARCH});
+}
+
+std::unique_ptr<StateExplorer>
+SessionManagerExplorer::get_child(vespalib::stringref name) const
+{
+    if (name == SEARCH) {
+        return std::unique_ptr<StateExplorer>(new SearchSessionExplorer(_manager));
+    }
+    return std::unique_ptr<StateExplorer>(nullptr);
+}
+
+} // namespace proton::matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h
new file mode 100644
index 00000000000..e00a95f628e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "sessionmanager.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+namespace matching {
+
+/**
+ * Class used to explore the state of a session manager
+ */
+class SessionManagerExplorer : public vespalib::StateExplorer
+{
+private:
+    const SessionManager &_manager;
+
+public:
+    SessionManagerExplorer(const SessionManager &manager) : _manager(manager) {}
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+    virtual std::vector<vespalib::string> get_children_names() const override;
+    virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override;
+};
+
+}  // namespace proton::matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp
new file mode 100644
index 00000000000..15d68ecfb08
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".sessionmanager");
+#include "sessionmanager.h"
+
+using search::grouping::GroupingSession;
+
+namespace proton {
+namespace matching {
+
+void SessionManager::SessionCacheBase::entryDropped(const SessionId &id) {
+    LOG(debug, "Session cache is full, dropping entry to fit "
+        "session '%s'", id.c_str());
+    _stats.numDropped++;
+}
+
+SessionManager::SessionManager(uint32_t maxSize)
+    : _grouping_cache(maxSize),
+      _search_map() {
+}
+
+void SessionManager::insert(search::grouping::GroupingSession::UP session) {
+    _grouping_cache.insert(std::move(session));
+}
+
+void SessionManager::insert(SearchSession::SP session) {
+    _search_map.insert(std::move(session));
+}
+
+GroupingSession::UP SessionManager::pickGrouping(const SessionId &id) {
+    return _grouping_cache.pick(id);
+}
+
+SearchSession::SP SessionManager::pickSearch(const SessionId &id) {
+    return _search_map.pick(id);
+}
+
+std::vector<SessionManager::SearchSessionInfo>
+SessionManager::getSortedSearchSessionInfo() const
+{
+    std::vector<SearchSessionInfo> sessions;
+    _search_map.each([&sessions](const SearchSession &session)
+                     {
+                         sessions.emplace_back(session.getSessionId(),
+                                 session.getCreateTime(),
+                                 session.getTimeOfDoom());
+                     });
+    std::sort(sessions.begin(), sessions.end(),
+              [](const SearchSessionInfo &a,
+                 const SearchSessionInfo &b)
+              {
+                  return (a.created < b.created);
+              });
+    return sessions;
+}
+
+void SessionManager::pruneTimedOutSessions(fastos::TimeStamp currentTime) {
+    _grouping_cache.pruneTimedOutSessions(currentTime);
+    _search_map.pruneTimedOutSessions(currentTime);
+}
+
+void SessionManager::close() {
+    pruneTimedOutSessions(fastos::TimeStamp::FUTURE);
+    assert(_grouping_cache.empty());
+    assert(_search_map.empty());
+}
+
+}  // namespace proton::matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h
new file mode 100644
index 00000000000..56b48047611
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h
@@ -0,0 +1,206 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "search_session.h"
+#include "isessioncachepruner.h"
+#include <vespa/searchcore/grouping/groupingsession.h>
+#include <vespa/searchcore/grouping/sessionid.h>
+#include <vespa/vespalib/stllike/lrucache_map.h>
+
+namespace proton {
+namespace matching {
+
+typedef vespalib::string SessionId;
+
+class SessionManager : public ISessionCachePruner {
+public:
+    struct Stats {
+        Stats()
+            : numInsert(0),
+              numPick(0),
+              numDropped(0),
+              numCached(0),
+              numTimedout(0)
+        {}
+        uint32_t numInsert;
+        uint32_t numPick;
+        uint32_t numDropped;
+        uint32_t numCached;
+        uint32_t numTimedout;
+    };
+
+    struct SearchSessionInfo {
+        vespalib::string id;
+        fastos::TimeStamp created;
+        fastos::TimeStamp doom;
+        SearchSessionInfo(const vespalib::string &id_in,
+                          fastos::TimeStamp created_in,
+                          fastos::TimeStamp doom_in)
+            : id(id_in), created(created_in), doom(doom_in) {}
+    };
+
+private:
+    struct SessionCacheBase {
+    protected:
+        Stats _stats;
+        vespalib::Lock _lock;
+
+        void entryDropped(const SessionId &id);
+        ~SessionCacheBase() {}
+    };
+
+    template <typename T>
+    struct SessionCache : SessionCacheBase {
+        typedef typename T::LP EntryLP;
+        typedef typename T::UP EntryUP;
+        vespalib::lrucache_map<vespalib::LruParam<SessionId, EntryLP> > _cache;
+
+        SessionCache(uint32_t max_size) : _cache(max_size) {}
+
+        void insert(EntryUP session) {
+            vespalib::LockGuard guard(_lock);
+            const SessionId &id(session->getSessionId());
+            if (_cache.size() >= _cache.capacity()) {
+                entryDropped(id);
+            }
+            _cache.insert(id, EntryLP(session.release()));
+            _stats.numInsert++;
+        }
+        EntryUP pick(const SessionId & id) {
+            vespalib::LockGuard guard(_lock);
+            EntryUP ret;
+            if (_cache.hasKey(id)) {
+                _stats.numPick++;
+                EntryLP session(_cache.get(id));
+                _cache.erase(id);
+                ret.reset(session.release());
+            }
+            return ret;
+        }
+        void pruneTimedOutSessions(fastos::TimeStamp currentTime) {
+            std::vector<EntryLP> toDestruct = stealTimedOutSessions(currentTime);
+            toDestruct.clear();
+        }
+        std::vector<EntryLP> stealTimedOutSessions(fastos::TimeStamp currentTime) {
+            std::vector<EntryLP> toDestruct;
+            vespalib::LockGuard guard(_lock);
+            toDestruct.reserve(_cache.size());
+            for (auto it(_cache.begin()), mt(_cache.end()); it != mt;) {
+                EntryLP session = *it;
+                if (session->getTimeOfDoom() < currentTime) {
+                    toDestruct.push_back(session);
+                    it = _cache.erase(it);
+                    _stats.numTimedout++;
+                } else {
+                    it++;
+                }
+            }
+            return toDestruct;
+        }
+        Stats getStats() {
+            vespalib::LockGuard guard(_lock);
+            Stats stats = _stats;
+            stats.numCached = _cache.size();
+            _stats = Stats();
+            return stats;
+        }
+        bool empty() const {
+            vespalib::LockGuard guard(_lock);
+            return _cache.empty();
+        }
+    };
+
+    template <typename T>
+    struct SessionMap : SessionCacheBase {
+        typedef typename T::SP EntrySP;
+        vespalib::hash_map<SessionId, EntrySP> _map;
+
+        void insert(EntrySP session) {
+            vespalib::LockGuard guard(_lock);
+            const SessionId &id(session->getSessionId());
+            _map.insert(std::make_pair(id, session));
+            _stats.numInsert++;
+        }
+        EntrySP pick(const SessionId & id) {
+            vespalib::LockGuard guard(_lock);
+            auto it = _map.find(id);
+            if (it != _map.end()) {
+                _stats.numPick++;
+                return it->second;
+            }
+            return EntrySP();
+        }
+        void pruneTimedOutSessions(fastos::TimeStamp currentTime) {
+            std::vector<EntrySP> toDestruct = stealTimedOutSessions(currentTime);
+            toDestruct.clear();
+        }
+        std::vector<EntrySP> stealTimedOutSessions(fastos::TimeStamp currentTime) {
+            std::vector<EntrySP> toDestruct;
+            std::vector<SessionId> keys;
+            vespalib::LockGuard guard(_lock);
+            keys.reserve(_map.size());
+            toDestruct.reserve(_map.size());
+            for (auto & it : _map) {
+                EntrySP &session = it.second;
+                if (session->getTimeOfDoom() < currentTime) {
+                    keys.push_back(it.first);
+                    toDestruct.push_back(EntrySP());
+                    toDestruct.back().swap(session);
+                }
+            }
+            for (auto key : keys) {
+                _map.erase(key);
+                _stats.numTimedout++;
+            }
+            return toDestruct;
+        }
+        Stats getStats() {
+            vespalib::LockGuard guard(_lock);
+            Stats stats = _stats;
+            stats.numCached = _map.size();
+            _stats = Stats();
+            return stats;
+        }
+        size_t size() const {
+            vespalib::LockGuard guard(_lock);
+            return _map.size();
+        }
+        bool empty() const {
+            vespalib::LockGuard guard(_lock);
+            return _map.empty();
+        }
+        template <typename F>
+        void each(F f) const {
+            vespalib::LockGuard guard(_lock);
+            for (const auto &entry: _map) {
+                f(*entry.second);
+            }
+        }
+    };
+
+    SessionCache<search::grouping::GroupingSession> _grouping_cache;
+    SessionMap<SearchSession> _search_map;
+
+public:
+    typedef std::unique_ptr<SessionManager> UP;
+    typedef std::shared_ptr<SessionManager> SP;
+
+    SessionManager(uint32_t maxSizeGrouping);
+
+    void insert(search::grouping::GroupingSession::UP session);
+    search::grouping::GroupingSession::UP pickGrouping(const SessionId &id);
+    Stats getGroupingStats() { return _grouping_cache.getStats(); }
+
+    void insert(SearchSession::SP session);
+    SearchSession::SP pickSearch(const SessionId &id);
+    Stats getSearchStats() { return _search_map.getStats(); }
+    size_t getNumSearchSessions() const { return _search_map.size(); }
+    std::vector<SearchSessionInfo> getSortedSearchSessionInfo() const;
+
+    void pruneTimedOutSessions(fastos::TimeStamp currentTime);
+    void close();
+};
+
+}  // namespace proton::matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp b/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp
new file mode 100644
index 00000000000..020809d080f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.termdataextractor");
+
+#include "termdataextractor.h"
+
+#include "querynodes.h"
+#include <vespa/searchlib/query/tree/templatetermvisitor.h>
+
+using search::fef::ITermData;
+using search::query::Node;
+using search::query::TemplateTermVisitor;
+using std::vector;
+
+namespace proton {
+namespace matching {
+
+namespace {
+class TermDataExtractorVisitor
+    : public TemplateTermVisitor<TermDataExtractorVisitor, ProtonNodeTypes>
+{
+    vector<const ITermData *> &_term_data;
+
+public:
+    TermDataExtractorVisitor(vector<const ITermData *> &term_data)
+        : _term_data(term_data) {
+    }
+
+    template <class TermType>
+    void visitTerm(TermType &n) {
+        if (n.isRanked()) {
+            _term_data.push_back(&n);
+        }
+    }
+
+    void visit(ProtonLocationTerm &) {}
+
+    virtual void visit(ProtonNodeTypes::AndNot &n) {
+        assert(n.getChildren().size() > 0);
+        n.getChildren()[0]->accept(*this);
+    }
+
+    virtual void visit(ProtonNodeTypes::Equiv &n) {
+        // XXX: unranked equiv not supported
+        _term_data.push_back(&n);
+    }
+};
+}  // namespace
+
+void TermDataExtractor::extractTerms(const Node &node,
+                                     vector<const ITermData *> &term_data) {
+    TermDataExtractorVisitor visitor(term_data);
+    // The visitor doesn't deal with const nodes. However, we are
+    // not changing the node, so we can safely remove the const.
+    const_cast<Node &>(node).accept(visitor);
+}
+
+}  // namespace matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.h b/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.h
new file mode 100644
index 00000000000..37187aef566
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vector>
+
+namespace search {
+namespace query { class Node; }
+namespace fef { class ITermData; }
+}  // namespace search
+
+namespace proton {
+namespace matching {
+
+struct TermDataExtractor {
+    /**
+     * Extracts pointers to all TermData objects stored in the term
+     * nodes of the node tree.
+     */
+    static void extractTerms(const search::query::Node &node,
+                             std::vector<const search::fef::ITermData *> &td);
+};
+
+}  // namespace matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp
new file mode 100644
index 00000000000..6914e9f8702
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.termdatafromnode");
+
+#include "termdatafromnode.h"
+#include <vespa/searchlib/query/tree/customtypevisitor.h>
+#include "querynodes.h"
+
+namespace proton {
+namespace matching {
+
+namespace {
+struct TermDataFromTermVisitor
+    : public search::query::CustomTypeVisitor<ProtonNodeTypes>
+{
+    const ProtonTermData *data;
+    TermDataFromTermVisitor() : data(0) {}
+
+    template <class TermNode>
+    void visitTerm(const TermNode &n) {
+        data = &n;
+    }
+
+    virtual void visit(ProtonAnd &) {}
+    virtual void visit(ProtonAndNot &) {}
+    virtual void visit(ProtonNear &) {}
+    virtual void visit(ProtonONear &) {}
+    virtual void visit(ProtonOr &) {}
+    virtual void visit(ProtonRank &) {}
+    virtual void visit(ProtonWeakAnd &) {}
+
+    virtual void visit(ProtonWeightedSetTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonDotProduct &n) { visitTerm(n); }
+    virtual void visit(ProtonWandTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonPhrase &n) { visitTerm(n); }
+    virtual void visit(ProtonEquiv &n) { visitTerm(n); }
+
+    virtual void visit(ProtonNumberTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonLocationTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonPrefixTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonRangeTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonStringTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonSubstringTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonSuffixTerm &n) { visitTerm(n); }
+    virtual void visit(ProtonPredicateQuery &) { }
+    virtual void visit(ProtonRegExpTerm &n) { visitTerm(n); }
+};
+}  // namespace
+
+const ProtonTermData *
+termDataFromNode(const search::query::Node &node)
+{
+    TermDataFromTermVisitor visitor;
+    const_cast<search::query::Node &>(node).accept(visitor);
+    return visitor.data;
+}
+
+}  // namespace matching
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h
new file mode 100644
index 00000000000..12b1a6fda2a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+
+namespace search { namespace query { class Node; } }
+
+namespace proton {
+namespace matching {
+
+class ProtonTermData;
+
+const ProtonTermData *termDataFromNode(const search::query::Node &node);
+
+}  // namespace matching
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/matching/viewresolver.cpp b/searchcore/src/vespa/searchcore/proton/matching/viewresolver.cpp
new file mode 100644
index 00000000000..44b7cfe20c1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/viewresolver.cpp
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.matching.viewresolver");
+
+#include "viewresolver.h"
+#include <vespa/searchcommon/common/schema.h>
+
+namespace proton {
+namespace matching {
+
+ViewResolver &
+ViewResolver::add(const vespalib::stringref &view,
+                  const vespalib::stringref &field)
+{
+    _map[view].push_back(field);
+    return *this;
+}
+
+bool
+ViewResolver::resolve(const vespalib::stringref &view,
+                      std::vector<vespalib::string> &fields) const
+{
+    Map::const_iterator pos = _map.find(view);
+    if (pos == _map.end()) {
+        fields.push_back(view);
+        return false;
+    }
+    fields = pos->second;
+    return true;
+}
+
+ViewResolver
+ViewResolver::createFromSchema(const search::index::Schema &schema)
+{
+    ViewResolver resolver;
+    for (uint32_t i = 0; i < schema.getNumFieldSets(); ++i) {
+        const search::index::Schema::FieldSet
+            &f = schema.getFieldSet(i);
+        const vespalib::string &view = f.getName();
+        const std::vector<vespalib::string> &fields = f.getFields();
+        for (uint32_t j = 0; j < fields.size(); ++j) {
+            resolver.add(view, fields[j]);
+        }
+    }
+    return resolver;
+}
+
+} // namespace matching
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/viewresolver.h b/searchcore/src/vespa/searchcore/proton/matching/viewresolver.h
new file mode 100644
index 00000000000..f0f12f9a8e5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/viewresolver.h
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <vector>
+#include <map>
+
+namespace search {
+namespace index { class Schema; }
+} // namespace search
+
+namespace proton {
+namespace matching {
+
+/**
+ * A small utility class used to resolve views into fields when
+ * setting up the query tree. A view contains a set of fields. An
+ * undefined view is considered empty. A view will resolve to the set
+ * of fields it contains. If the view is empty, it will resolve to a
+ * field with the same name as the view itself.
+ **/
+class ViewResolver
+{
+private:
+    typedef std::map<vespalib::string, std::vector<vespalib::string> > Map;
+    Map _map;
+
+public:
+    /**
+     * Add a field to the given view. This function is public to
+     * facilitate testing. Duplicate detection is not performed here,
+     * so adding the same field to a view multiple times is not a good
+     * idea.
+     *
+     * @return this object, for chaining
+     * @param view the name of the view
+     * @param field the name of the field
+     **/
+    ViewResolver &add(const vespalib::stringref &view,
+                      const vespalib::stringref &field);
+
+    /**
+     * Resolve a view to obtain the set of fields it
+     * contains. Undefined views are considered empty and will resolve
+     * to a field with the same name as the view itself.
+     *
+     * @return true if the view was non-empty
+     * @param view the name of the view
+     * @param fields vector that will be filled out with the fields
+     *               that are part of the requested view.
+     **/
+    bool resolve(const vespalib::stringref &view,
+                 std::vector<vespalib::string> &fields) const;
+
+    /**
+     * Create a view resolver based on the field collections defined
+     * in the given schema. View definitions should be completely
+     * separate from how fields are combined into collections in the
+     * index, but this is a good start, as view and index correlate
+     * 1-to-1 in the current model.
+     *
+     * @return view resolver
+     * @param schema index schema
+     **/
+    static ViewResolver createFromSchema(const search::index::Schema &schema);
+};
+
+} // namespace matching
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt
new file mode 100644
index 00000000000..57a15be5535
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt
@@ -0,0 +1,21 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_proton_metrics STATIC
+    SOURCES
+    attribute_metrics.cpp
+    content_proton_metrics.cpp
+    documentdb_job_trackers.cpp
+    documentdb_tagged_metrics.cpp
+    executor_metrics.cpp
+    feed_metrics.cpp
+    job_load_sampler.cpp
+    job_tracker.cpp
+    job_tracked_flush_target.cpp
+    job_tracked_flush_task.cpp
+    legacy_documentdb_metrics.cpp
+    legacy_proton_metrics.cpp
+    metrics_engine.cpp
+    resource_usage_metrics.cpp
+    sessionmanager_metrics.cpp
+    trans_log_server_metrics.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp
new file mode 100644
index 00000000000..e1299c990bb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "attribute_metrics.h"
+
+namespace proton {
+
+AttributeMetrics::List::Entry::Entry(const std::string &name)
+    : metrics::MetricSet(name, "", "Attribute vector metrics", 0),
+      memoryUsage("memoryusage", "", "Memory usage", this),
+      bitVectors("bitvectors", "", "Number of bitvectors", this)
+{
+}
+
+AttributeMetrics::List::Entry::LP
+AttributeMetrics::List::add(const std::string &name)
+{
+    if (metrics.find(name) != metrics.end()) {
+        return Entry::LP(0);
+    }
+    Entry::LP entry(new Entry(name));
+    metrics[name] = entry;
+    return entry;
+}
+
+AttributeMetrics::List::Entry::LP
+AttributeMetrics::List::get(const std::string &name) const
+{
+    std::map<std::string, Entry::LP>::const_iterator pos = metrics.find(name);
+    if (pos == metrics.end()) {
+        return Entry::LP(0);
+    }
+    return pos->second;
+}
+
+AttributeMetrics::List::Entry::LP
+AttributeMetrics::List::remove(const std::string &name)
+{
+    std::map<std::string, Entry::LP>::const_iterator pos = metrics.find(name);
+    if (pos == metrics.end()) {
+        return Entry::LP(0);
+    }
+    Entry::LP retval = pos->second;
+    metrics.erase(name);
+    return retval;
+}
+
+std::vector<AttributeMetrics::List::Entry::LP>
+AttributeMetrics::List::release()
+{
+    std::vector<Entry::LP> entries;
+    std::map<std::string, Entry::LP>::const_iterator pos = metrics.begin();
+    for (; pos != metrics.end(); ++pos) {
+        entries.push_back(pos->second);
+    }
+    metrics.clear();
+    return entries;
+}
+
+AttributeMetrics::List::List(metrics::MetricSet *parent)
+    : metrics::MetricSet("list", "", "Metrics per attribute vector", parent),
+      metrics()
+{
+}
+
+AttributeMetrics::AttributeMetrics(metrics::MetricSet *parent)
+    : metrics::MetricSet("attributes", "", "Attribute metrics", parent),
+      list(this),
+      memoryUsage("memoryusage", "", "Memory usage for attributes", this),
+      bitVectors("bitvectors", "", "Number of bitvectors for attributes", this)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.h
new file mode 100644
index 00000000000..263ef693c07
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.h
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/vespalib/util/linkedptr.h>
+
+namespace proton {
+
+struct LegacyDocumentDBMetrics;
+
+struct AttributeMetrics : metrics::MetricSet {
+
+    // The metric set also owns the actual metrics for individual
+    // attribute vectors. Another way to do this would be to let the
+    // attribute vectors own their own metrics, but this would
+    // generate more dependencies and reduce locality of code changes.
+
+    struct List : metrics::MetricSet {
+        struct Entry : metrics::MetricSet {
+            typedef vespalib::LinkedPtr<Entry> LP;
+            metrics::LongValueMetric memoryUsage;
+            metrics::LongValueMetric bitVectors;
+            Entry(const std::string &name);
+        };
+        Entry::LP add(const std::string &name);
+        Entry::LP get(const std::string &name) const;
+        Entry::LP remove(const std::string &name);
+        std::vector<Entry::LP> release();
+
+        // per attribute metrics will be wired in here (by the metrics engine)
+        List(metrics::MetricSet *parent);
+
+    private:
+        std::map<std::string, Entry::LP> metrics;
+    };
+
+    List                     list;
+    metrics::LongValueMetric memoryUsage;
+    metrics::LongValueMetric bitVectors;
+
+    AttributeMetrics(metrics::MetricSet *parent);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp
new file mode 100644
index 00000000000..ba86b563f61
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.content_proton_metrics");
+#include "content_proton_metrics.h"
+
+namespace proton {
+
+ContentProtonMetrics::ContentProtonMetrics()
+    : metrics::MetricSet("content.proton", "", "Search engine metrics", nullptr),
+      transactionLog(this),
+      resourceUsage(this)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h
new file mode 100644
index 00000000000..bf720d59a36
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include "resource_usage_metrics.h"
+#include "trans_log_server_metrics.h"
+
+namespace proton {
+
+/**
+ * Metric set for all metrics reported by proton.
+ *
+ * This class uses the new metric naming scheme decided in architect meeting 2014-10-30.
+ * All proton metrics use the prefix "content.proton." and dimensions where appropriate.
+ * For instance, all document db metrics use the dimension "documenttype":"mydoctype"
+ * instead of using the document type name as part of metric names.
+ */
+struct ContentProtonMetrics : metrics::MetricSet
+{
+    TransLogServerMetrics transactionLog;
+    ResourceUsageMetrics resourceUsage;
+
+    ContentProtonMetrics();
+
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp
new file mode 100644
index 00000000000..cb4d29af497
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.documentdb_job_trackers");
+#include "documentdb_job_trackers.h"
+#include "job_tracked_flush_target.h"
+
+using searchcorespi::IFlushTarget;
+typedef IFlushTarget::Type FTT;
+typedef IFlushTarget::Component FTC;
+
+namespace proton {
+
+DocumentDBJobTrackers::DocumentDBJobTrackers()
+    : _lock(),
+      _now(fastos::ClockSystem::now()),
+      _attributeFlush(new JobTracker(_now.sec(), _lock)),
+      _memoryIndexFlush(new JobTracker(_now.sec(), _lock)),
+      _diskIndexFusion(new JobTracker(_now.sec(), _lock)),
+      _documentStoreFlush(new JobTracker(_now.sec(), _lock)),
+      _documentStoreCompact(new JobTracker(_now.sec(), _lock)),
+      _bucketMove(new JobTracker(_now.sec(), _lock)),
+      _lidSpaceCompact(new JobTracker(_now.sec(), _lock)),
+      _removedDocumentsPrune(new JobTracker(_now.sec(), _lock))
+{
+}
+
+namespace {
+
+IFlushTarget::SP
+trackFlushTarget(const IJobTracker::SP &tracker,
+                 const IFlushTarget::SP &target)
+{
+    return IFlushTarget::SP(new JobTrackedFlushTarget(tracker, target));
+}
+
+}
+
+IFlushTarget::List
+DocumentDBJobTrackers::trackFlushTargets(const IFlushTarget::List &flushTargets)
+{
+    IFlushTarget::List retval;
+    for (const auto &ft : flushTargets) {
+        if (ft->getComponent() == FTC::ATTRIBUTE && ft->getType() == FTT::SYNC) {
+            retval.push_back(trackFlushTarget(_attributeFlush, ft));
+        } else if (ft->getComponent() == FTC::INDEX && ft->getType() == FTT::FLUSH) {
+            retval.push_back(trackFlushTarget(_memoryIndexFlush, ft));
+        } else if (ft->getComponent() == FTC::INDEX && ft->getType() == FTT::GC) {
+            retval.push_back(trackFlushTarget(_diskIndexFusion, ft));
+        } else if (ft->getComponent() == FTC::DOCUMENT_STORE && ft->getType() == FTT::SYNC) {
+            retval.push_back(trackFlushTarget(_documentStoreFlush, ft));
+        } else if (ft->getComponent() == FTC::DOCUMENT_STORE && ft->getType() == FTT::GC) {
+            retval.push_back(trackFlushTarget(_documentStoreCompact, ft));
+        } else {
+            LOG(warning, "trackFlushTargets(): Flush target '%s' with type '%d' and component '%d' "
+                    "is not known and will not be tracked",
+                    ft->getName().c_str(), static_cast<int>(ft->getType()),
+                    static_cast<int>(ft->getComponent()));
+            retval.push_back(ft);
+        }
+    }
+    return retval;
+}
+
+namespace {
+
+double
+updateMetric(metrics::DoubleAverageMetric &metric,
+             JobTracker &tracker,
+             double nowInSec,
+             const vespalib::LockGuard &guard)
+{
+    double load = tracker.sampleLoad(nowInSec, guard);
+    metric.addValue(load);
+    return load;
+}
+
+}
+
+void
+DocumentDBJobTrackers::updateMetrics(DocumentDBTaggedMetrics::JobMetrics &metrics)
+{
+    vespalib::LockGuard guard(_lock);
+    _now = fastos::ClockSystem::now();
+    double nowInSec = _now.sec();
+    double load = 0.0;
+    load += updateMetric(metrics.attributeFlush, *_attributeFlush, nowInSec, guard);
+    load += updateMetric(metrics.memoryIndexFlush, *_memoryIndexFlush, nowInSec, guard);
+    load += updateMetric(metrics.diskIndexFusion, *_diskIndexFusion, nowInSec, guard);
+    load += updateMetric(metrics.documentStoreFlush, *_documentStoreFlush, nowInSec, guard);
+    load += updateMetric(metrics.documentStoreCompact, *_documentStoreCompact, nowInSec, guard);
+    load += updateMetric(metrics.bucketMove, *_bucketMove, nowInSec, guard);
+    load += updateMetric(metrics.lidSpaceCompact, *_lidSpaceCompact, nowInSec, guard);
+    load += updateMetric(metrics.removedDocumentsPrune, *_removedDocumentsPrune, nowInSec, guard);
+    metrics.total.addValue(load);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.h
new file mode 100644
index 00000000000..4fd0b20688e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentdb_tagged_metrics.h"
+#include "job_tracker.h"
+#include <vespa/fastos/timestamp.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton {
+
+/**
+ * Class that handles all job trackers for a document db and
+ * connects them to the job metrics.
+ */
+class DocumentDBJobTrackers
+{
+private:
+    vespalib::Lock    _lock;
+    fastos::TimeStamp _now;
+    JobTracker::SP    _attributeFlush;
+    JobTracker::SP    _memoryIndexFlush;
+    JobTracker::SP    _diskIndexFusion;
+    JobTracker::SP    _documentStoreFlush;
+    JobTracker::SP    _documentStoreCompact;
+    JobTracker::SP    _bucketMove;
+    JobTracker::SP    _lidSpaceCompact;
+    JobTracker::SP    _removedDocumentsPrune;
+
+public:
+    DocumentDBJobTrackers();
+
+    IJobTracker &getAttributeFlush() { return *_attributeFlush; }
+    IJobTracker &getMemoryIndexFlush() { return *_memoryIndexFlush; }
+    IJobTracker &getDiskIndexFusion() { return *_diskIndexFusion; }
+    IJobTracker &getDocumentStoreFlush() { return *_documentStoreFlush; }
+    IJobTracker &getDocumentStoreCompact() { return *_documentStoreCompact; }
+    IJobTracker::SP getBucketMove() { return _bucketMove; }
+    IJobTracker::SP getLidSpaceCompact() { return _lidSpaceCompact; }
+    IJobTracker::SP getRemovedDocumentsPrune() { return _removedDocumentsPrune; }
+
+    searchcorespi::IFlushTarget::List
+    trackFlushTargets(const searchcorespi::IFlushTarget::List &flushTargets);
+
+    void updateMetrics(DocumentDBTaggedMetrics::JobMetrics &metrics);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h
new file mode 100644
index 00000000000..20e72a23edf
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentdb_tagged_metrics.h"
+#include "legacy_documentdb_metrics.h"
+
+namespace proton {
+
+/**
+ * A collection of all the metrics for a document db (both tagged and no-tagged).
+ */
+class DocumentDBMetricsCollection
+{
+private:
+    LegacyDocumentDBMetrics _metrics;
+    DocumentDBTaggedMetrics _taggedMetrics;
+
+public:
+    DocumentDBMetricsCollection(const vespalib::string &docTypeName, size_t maxNumThreads)
+        : _metrics(docTypeName, maxNumThreads),
+          _taggedMetrics(docTypeName)
+    {
+    }
+    LegacyDocumentDBMetrics &getMetrics() { return _metrics; }
+    DocumentDBTaggedMetrics &getTaggedMetrics() { return _taggedMetrics; }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
new file mode 100644
index 00000000000..27ddcb0bf2e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.documentdb_tagged_metrics");
+#include "documentdb_tagged_metrics.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using vespalib::make_string;
+
+namespace proton {
+
+DocumentDBTaggedMetrics::JobMetrics::JobMetrics(metrics::MetricSet* parent)
+    : MetricSet("job", "", "Job load average for various jobs in a document database", parent),
+      attributeFlush("attribute_flush", "", "Flushing of attribute vector(s) to disk", this),
+      memoryIndexFlush("memory_index_flush", "", "Flushing of memory index to disk", this),
+      diskIndexFusion("disk_index_fusion", "", "Fusion of disk indexes", this),
+      documentStoreFlush("document_store_flush", "", "Flushing of document store to disk", this),
+      documentStoreCompact("document_store_compact", "",
+              "Compaction of document store on disk", this),
+      bucketMove("bucket_move", "",
+              "Moving of buckets between 'ready' and 'notready' sub databases", this),
+      lidSpaceCompact("lid_space_compact", "",
+              "Compaction of lid space in document meta store and attribute vectors", this),
+      removedDocumentsPrune("removed_documents_prune", "",
+              "Pruning of removed documents in 'removed' sub database", this),
+      total("total", "", "The job load average total of all job metrics", this)
+{
+}
+
+DocumentDBTaggedMetrics::SubDBMetrics::SubDBMetrics(const vespalib::string &name,
+                                                    MetricSet *parent)
+    : MetricSet(name, "", "Sub database metrics", parent),
+      lidSpace(this),
+      documentStore(this)
+{
+}
+
+DocumentDBTaggedMetrics::SubDBMetrics::LidSpaceMetrics::LidSpaceMetrics(MetricSet *parent)
+    : MetricSet("lid_space", "", "Local document id (lid) space metrics for this document sub DB", parent),
+      lidLimit("lid_limit", "", "The size of the allocated lid space", this),
+      usedLids("used_lids", "", "The number of lids used", this),
+      lowestFreeLid("lowest_free_lid", "", "The lowest free lid", this),
+      highestUsedLid("highest_used_lid", "", "The highest used lid", this),
+      lidBloatFactor("lid_bloat_factor", "", "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space "
+              "((lid_limit - used_lids) / lid_limit)", this),
+      lidFragmentationFactor("lid_fragmentation_factor", "",
+              "The fragmentation factor of this lid space, indicating the amount of holes in the currently used part of the lid space "
+              "((highest_used_lid - used_lids) / highest_used_lid)", this)
+{
+}
+
+DocumentDBTaggedMetrics::SubDBMetrics::DocumentStoreMetrics::DocumentStoreMetrics(MetricSet *parent)
+    : MetricSet("document_store", "", "document store metrics for this document sub DB", parent),
+      diskUsage("disk_usage", "", "Disk space usage in bytes", this),
+      diskBloat("disk_bloat", "", "Disk space bloat in bytes", this),
+      maxBucketSpread("max_bucket_spread", "", "Max bucket spread in underlying files (sum(unique buckets in each chunk)/unique buckets in file)", this)
+{
+}
+
+DocumentDBTaggedMetrics::AttributeMetrics::AttributeMetrics(MetricSet *parent)
+    : MetricSet("attribute", "", "Attribute vector metrics for this document db", parent),
+      resourceUsage(this)
+{
+}
+
+DocumentDBTaggedMetrics::AttributeMetrics::ResourceUsageMetrics::ResourceUsageMetrics(MetricSet *parent)
+    : MetricSet("resource_usage", "", "Usage metrics for various attribute vector resources", parent),
+      enumStore("enum_store", "", "The highest relative amount of enum store address space used among "
+              "all enumerated attribute vectors in this document db (value in the range [0, 1])", this),
+      multiValue("multi_value", "", "The highest relative amount of multi-value address space used among "
+              "all multi-value attribute vectors in this document db (value in the range [0, 1])", this),
+      feedingBlocked("feeding_blocked", "", "Whether feeding is blocked due to attribute resource limits being reached (value is either 0 or 1)", this)
+{
+}
+
+DocumentDBTaggedMetrics::DocumentDBTaggedMetrics(const vespalib::string &docTypeName)
+    : MetricSet("documentdb", {{"documenttype", docTypeName}}, "Document DB metrics", nullptr),
+      job(this),
+      attribute(this),
+      ready("ready", this),
+      notReady("notready", this),
+      removed("removed", this)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
new file mode 100644
index 00000000000..f19a0c4dce0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/metrics/metricset.h>
+#include <vespa/metrics/valuemetric.h>
+
+namespace proton {
+
+/**
+ * Metrics for a document db that are tagged with at least "documenttype":"name".
+ * These tags are exposed as dimensions together with the metrics.
+ */
+struct DocumentDBTaggedMetrics : metrics::MetricSet
+{
+    struct JobMetrics : metrics::MetricSet
+    {
+        metrics::DoubleAverageMetric attributeFlush;
+        metrics::DoubleAverageMetric memoryIndexFlush;
+        metrics::DoubleAverageMetric diskIndexFusion;
+        metrics::DoubleAverageMetric documentStoreFlush;
+        metrics::DoubleAverageMetric documentStoreCompact;
+        metrics::DoubleAverageMetric bucketMove;
+        metrics::DoubleAverageMetric lidSpaceCompact;
+        metrics::DoubleAverageMetric removedDocumentsPrune;
+        metrics::DoubleAverageMetric total;
+
+        JobMetrics(metrics::MetricSet *parent);
+    };
+
+    struct SubDBMetrics : metrics::MetricSet
+    {
+        struct LidSpaceMetrics : metrics::MetricSet
+        {
+            metrics::LongValueMetric lidLimit;
+            metrics::LongValueMetric usedLids;
+            metrics::LongValueMetric lowestFreeLid;
+            metrics::LongValueMetric highestUsedLid;
+            metrics::DoubleValueMetric lidBloatFactor;
+            metrics::DoubleValueMetric lidFragmentationFactor;
+
+            LidSpaceMetrics(metrics::MetricSet *parent);
+        };
+
+        struct DocumentStoreMetrics : metrics::MetricSet
+        {
+            metrics::LongValueMetric diskUsage;
+            metrics::LongValueMetric diskBloat;
+            metrics::DoubleValueMetric maxBucketSpread;
+
+            DocumentStoreMetrics(metrics::MetricSet *parent);
+        };
+
+        LidSpaceMetrics lidSpace;
+        DocumentStoreMetrics documentStore;
+
+        SubDBMetrics(const vespalib::string &name, metrics::MetricSet *parent);
+    };
+
+    struct AttributeMetrics : metrics::MetricSet
+    {
+        struct ResourceUsageMetrics : metrics::MetricSet
+        {
+            metrics::DoubleValueMetric enumStore;
+            metrics::DoubleValueMetric multiValue;
+            metrics::LongValueMetric   feedingBlocked;
+
+            ResourceUsageMetrics(metrics::MetricSet *parent);
+        };
+
+        ResourceUsageMetrics resourceUsage;
+
+        AttributeMetrics(metrics::MetricSet *parent);
+    };
+
+    JobMetrics job;
+    AttributeMetrics attribute;
+    SubDBMetrics ready;
+    SubDBMetrics notReady;
+    SubDBMetrics removed;
+
+    DocumentDBTaggedMetrics(const vespalib::string &docTypeName);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp
new file mode 100644
index 00000000000..30be90d637a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.executormetrics");
+#include "executor_metrics.h"
+
+namespace proton {
+
+void
+ExecutorMetrics::update(const vespalib::ThreadStackExecutorBase::Stats &stats)
+{
+    maxPending.set(stats.maxPendingTasks);
+    accepted.inc(stats.acceptedTasks);
+    rejected.inc(stats.rejectedTasks);
+}
+
+ExecutorMetrics::ExecutorMetrics(const std::string &name, metrics::MetricSet *parent)
+    : metrics::MetricSet(name, "", "Instance specific thread executor metrics", parent),
+      maxPending("maxpending", "", "Maximum number of pending (active + queued) tasks", this),
+      accepted("accepted", "", "Number of accepted tasks", this),
+      rejected("rejected", "", "Number of rejected tasks", this)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h
new file mode 100644
index 00000000000..526c4fd6985
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/vespalib/util/threadstackexecutorbase.h>
+
+namespace proton {
+
+struct ExecutorMetrics : metrics::MetricSet
+{
+    metrics::LongValueMetric maxPending;
+    metrics::LongCountMetric accepted;
+    metrics::LongCountMetric rejected;
+
+    void update(const vespalib::ThreadStackExecutorBase::Stats &stats);
+    ExecutorMetrics(const std::string &name, metrics::MetricSet *parent);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.cpp
new file mode 100644
index 00000000000..bb6e1fdf50d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.cpp
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.feedengine.feedmetrics");
+#include "feed_metrics.h"
+#include <vespa/vespalib/util/sync.h>
+
+using vespalib::LockGuard;
+
+namespace proton {
+
+FeedMetrics::FeedMetrics()
+    : metrics::MetricSet("feed", "", "Feed metrics", 0),
+      count("count", "logdefault", "Feed messages handled", this),
+      latency("latency", "logdefault", "Feed message latency", this)
+{
+}
+
+PerDocTypeFeedMetrics::PerDocTypeFeedMetrics(MetricSet *parent)
+    : MetricSet("feedmetrics", "", "Feed metrics", parent),
+      _update_lock(),
+      _puts("puts", "", "Number of feed put operations", this),
+      _updates("updates", "", "Number of feed update operations", this),
+      _removes("removes", "", "Number of feed remove operations", this),
+      _moves("moves", "", "Number of feed move operations", this),
+      _put_latency("put_latency", "", "Latency for feed puts", this),
+      _update_latency("update_latency", "", "Latency for feed updates", this),
+      _remove_latency("remove_latency", "", "Latency for feed removes", this),
+      _move_latency("move_latency", "", "Latency for feed moves", this)
+{
+}
+
+void PerDocTypeFeedMetrics::RegisterPut(const FastOS_Time &start_time) {
+    LockGuard lock(_update_lock);
+    _puts.inc(1);
+    _put_latency.addValue(start_time.MilliSecsToNow() / 1000.0);
+}
+
+void PerDocTypeFeedMetrics::RegisterUpdate(const FastOS_Time &start_time) {
+    LockGuard lock(_update_lock);
+    _updates.inc(1);
+    _update_latency.addValue(start_time.MilliSecsToNow() / 1000.0);
+}
+
+void PerDocTypeFeedMetrics::RegisterRemove(const FastOS_Time &start_time) {
+    LockGuard lock(_update_lock);
+    _removes.inc(1);
+    _remove_latency.addValue(start_time.MilliSecsToNow() / 1000.0);
+}
+
+void
+PerDocTypeFeedMetrics::RegisterMove(const FastOS_Time &start_time)
+{
+    LockGuard lock(_update_lock);
+    _moves.inc(1);
+    _move_latency.addValue(start_time.MilliSecsToNow() / 1000.0);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.h
new file mode 100644
index 00000000000..541a9cec548
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton {
+
+struct FeedMetrics : metrics::MetricSet
+{
+    vespalib::Lock               updateLock;
+    metrics::LongCountMetric     count;
+    metrics::DoubleAverageMetric latency;
+
+    FeedMetrics();
+};
+
+class PerDocTypeFeedMetrics : metrics::MetricSet {
+    vespalib::Lock _update_lock;
+    metrics::LongCountMetric _puts;
+    metrics::LongCountMetric _updates;
+    metrics::LongCountMetric _removes;
+    metrics::LongCountMetric _moves;
+    metrics::DoubleAverageMetric _put_latency;
+    metrics::DoubleAverageMetric _update_latency;
+    metrics::DoubleAverageMetric _remove_latency;
+    metrics::DoubleAverageMetric _move_latency;
+
+public:
+    PerDocTypeFeedMetrics(metrics::MetricSet *parent);
+    void RegisterPut(const FastOS_Time &start_time);
+    void RegisterUpdate(const FastOS_Time &start_time);
+    void RegisterRemove(const FastOS_Time &start_time);
+    void RegisterMove(const FastOS_Time &start_time);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/i_job_tracker.h b/searchcore/src/vespa/searchcore/proton/metrics/i_job_tracker.h
new file mode 100644
index 00000000000..b3f8c215bf1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/i_job_tracker.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton {
+
+/**
+ * Interface for tracking the start and end of jobs.
+ */
+struct IJobTracker
+{
+    typedef std::shared_ptr<IJobTracker> SP;
+
+    virtual ~IJobTracker() {}
+
+    virtual void start() = 0;
+    virtual void end() = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.cpp
new file mode 100644
index 00000000000..cf70fb8a4a8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.cpp
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.job_load_sampler");
+#include "job_load_sampler.h"
+
+namespace proton {
+
+void
+JobLoadSampler::updateIntegral(double now, uint32_t jobCnt)
+{
+    assert(now >= _lastUpdateTime);
+    _loadIntegral += (now - _lastUpdateTime) * jobCnt;
+    _lastUpdateTime = now;
+}
+
+JobLoadSampler::JobLoadSampler(double now)
+    : _lastSampleTime(now),
+      _lastUpdateTime(now),
+      _currJobCnt(0),
+      _loadIntegral(0)
+{
+}
+
+void
+JobLoadSampler::startJob(double now)
+{
+    updateIntegral(now, _currJobCnt++);
+}
+
+void
+JobLoadSampler::endJob(double now)
+{
+    updateIntegral(now, _currJobCnt--);
+}
+
+double
+JobLoadSampler::sampleLoad(double now)
+{
+    assert(now >= _lastSampleTime);
+    updateIntegral(now, _currJobCnt);
+    double load = (now - _lastSampleTime > 0) ? (_loadIntegral / (now - _lastSampleTime)) : 0;
+    _lastSampleTime = now;
+    _loadIntegral = 0;
+    return load;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.h b/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.h
new file mode 100644
index 00000000000..6f078a8d9b2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.h
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton {
+
+/**
+ * Class the samples job load average of jobs running in a given time interval.
+ *
+ * If 1 job runs during a complete interval the sampled load is 1.0,
+ * if 2 jobs run for 0.7 intervals each the load is 1.4.
+ */
+class JobLoadSampler
+{
+private:
+    double _lastSampleTime;
+    double _lastUpdateTime;
+    uint32_t _currJobCnt;
+    double _loadIntegral;
+
+    void updateIntegral(double now, uint32_t jobCnt);
+
+public:
+    /**
+     * Start the sampler with now (in seconds).
+     */
+    JobLoadSampler(double now);
+
+    /**
+     * Signal that a job starts now (in seconds).
+     */
+    void startJob(double now);
+
+    /**
+     * Signal that a job ends now (in seconds).
+     */
+    void endJob(double now);
+
+    /**
+     * Samples the average load from previous sample time to now (in seconds).
+     */
+    double sampleLoad(double now);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp
new file mode 100644
index 00000000000..8ed767da162
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.job_tracked_flush_target");
+#include "job_tracked_flush_target.h"
+#include "job_tracked_flush_task.h"
+#include <memory>
+
+using searchcorespi::IFlushTarget;
+using searchcorespi::FlushTask;
+
+namespace proton {
+
+JobTrackedFlushTarget::JobTrackedFlushTarget(const IJobTracker::SP &tracker,
+                                             const IFlushTarget::SP &target)
+    : IFlushTarget(target->getName(), target->getType(), target->getComponent()),
+      _tracker(tracker),
+      _target(target)
+{
+}
+
+FlushTask::UP
+JobTrackedFlushTarget::initFlush(SerialNum currentSerial)
+{
+    _tracker->start();
+    FlushTask::UP targetTask = _target->initFlush(currentSerial);
+    _tracker->end();
+    if (targetTask.get() != nullptr) {
+        return FlushTask::UP(new JobTrackedFlushTask(_tracker, std::move(targetTask)));
+    }
+    return FlushTask::UP();
+}
+
+uint64_t
+JobTrackedFlushTarget::getApproxBytesToWriteToDisk() const
+{
+    return _target->getApproxBytesToWriteToDisk();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h
new file mode 100644
index 00000000000..c877d5ed4c7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_job_tracker.h"
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+
+namespace proton {
+
+/**
+ * Class that tracks the start and end of an init flush in a flush target.
+ * The returned flush task is also tracked.
+ */
+class JobTrackedFlushTarget : public searchcorespi::IFlushTarget
+{
+private:
+    IJobTracker::SP                 _tracker;
+    searchcorespi::IFlushTarget::SP _target;
+
+public:
+    JobTrackedFlushTarget(const IJobTracker::SP &tracker,
+                          const searchcorespi::IFlushTarget::SP &target);
+
+    const IJobTracker &getTracker() const { return *_tracker; }
+    const searchcorespi::IFlushTarget &getTarget() const { return *_target; }
+
+    // Implements searchcorespi::IFlushTarget
+    virtual MemoryGain getApproxMemoryGain() const {
+        return _target->getApproxMemoryGain();
+    }
+    virtual DiskGain getApproxDiskGain() const {
+        return _target->getApproxDiskGain();
+    }
+    virtual SerialNum getFlushedSerialNum() const {
+        return _target->getFlushedSerialNum();
+    }
+    virtual Time getLastFlushTime() const {
+        return _target->getLastFlushTime();
+    }
+    virtual bool needUrgentFlush() const {
+        return _target->needUrgentFlush();
+    }
+    virtual searchcorespi::FlushTask::UP initFlush(SerialNum currentSerial);
+    virtual searchcorespi::FlushStats getLastFlushStats() const {
+        return _target->getLastFlushStats();
+    }
+
+    virtual uint64_t getApproxBytesToWriteToDisk() const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp
new file mode 100644
index 00000000000..a96164d506c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.job_tracked_flush_task");
+#include "job_tracked_flush_task.h"
+
+using searchcorespi::FlushTask;
+
+namespace proton {
+
+JobTrackedFlushTask::JobTrackedFlushTask(const IJobTracker::SP &tracker,
+                                         FlushTask::UP task)
+    : _tracker(tracker),
+      _task(std::move(task))
+{
+}
+
+void
+JobTrackedFlushTask::run()
+{
+    _tracker->start();
+    _task->run();
+    _tracker->end();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.h b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.h
new file mode 100644
index 00000000000..fdd11a66dbb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_job_tracker.h"
+#include <vespa/searchcorespi/flush/flushtask.h>
+
+namespace proton {
+
+/**
+ * Class that tracks the start and end of a flush task.
+ */
+class JobTrackedFlushTask : public searchcorespi::FlushTask
+{
+private:
+    IJobTracker::SP              _tracker;
+    searchcorespi::FlushTask::UP _task;
+
+public:
+    JobTrackedFlushTask(const IJobTracker::SP &tracker,
+                        searchcorespi::FlushTask::UP task);
+
+    // Implements searchcorespi::FlushTask
+    virtual void run();
+    virtual search::SerialNum getFlushSerial() const {
+        return _task->getFlushSerial();
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp
new file mode 100644
index 00000000000..5dc3acd1f89
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.job_tracker");
+#include "job_tracker.h"
+#include <vespa/fastos/timestamp.h>
+
+using fastos::TimeStamp;
+using fastos::ClockSystem;
+
+namespace proton {
+
+JobTracker::JobTracker(double now, vespalib::Lock &lock)
+    : _sampler(now),
+      _lock(lock)
+{
+}
+
+double
+JobTracker::sampleLoad(double now, const vespalib::LockGuard &guard)
+{
+    (void) guard;
+    return _sampler.sampleLoad(now);
+}
+
+void
+JobTracker::start()
+{
+    vespalib::LockGuard guard(_lock);
+    _sampler.startJob(TimeStamp(ClockSystem::now()).sec());
+}
+
+void
+JobTracker::end()
+{
+    vespalib::LockGuard guard(_lock);
+    _sampler.endJob(TimeStamp(ClockSystem::now()).sec());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.h b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.h
new file mode 100644
index 00000000000..5d1ba336a71
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "job_load_sampler.h"
+#include "i_job_tracker.h"
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton {
+
+/**
+ * Class that tracks the start and end of jobs and makes average job load available.
+ */
+class JobTracker : public IJobTracker
+{
+private:
+    JobLoadSampler  _sampler;
+    vespalib::Lock &_lock;
+
+public:
+    typedef std::shared_ptr<JobTracker> SP;
+
+    JobTracker(double now, vespalib::Lock &lock);
+
+    /**
+     * Samples the average job load from previous sample time to now (in seconds).
+     * The caller of this function must take the guard on the lock referenced by this class.
+     */
+    double sampleLoad(double now, const vespalib::LockGuard &guard);
+
+    // Implements IJobTracker
+    virtual void start();
+    virtual void end();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.cpp
new file mode 100644
index 00000000000..ade6599ab65
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.cpp
@@ -0,0 +1,183 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.legacy_documentdb_metrics");
+#include "legacy_documentdb_metrics.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using vespalib::asciistream;
+using vespalib::make_string;
+using metrics::MetricSet;
+
+namespace proton {
+
+using matching::MatchingStats;
+
+LegacyDocumentDBMetrics::IndexMetrics::IndexMetrics(MetricSet *parent)
+    : MetricSet("index", "", "Index metrics", parent),
+      memoryUsage("memoryusage", "", "Memory usage for memory indexes", this),
+      docsInMemory("docsinmemory", "", "Number of documents in memory", this),
+      diskUsage("diskusage", "", "Disk usage for disk indexes", this)
+{
+}
+
+LegacyDocumentDBMetrics::DocstoreMetrics::DocstoreMetrics(MetricSet *parent)
+    : MetricSet("docstore", "", "Document store metrics", parent),
+      memoryUsage("memoryusage", "", "Memory usage for docstore", this),
+      cacheLookups("cachelookups", "", "Number of lookups in summary cache",
+                   this),
+      hits(0),
+      cacheHitRate("cachehitrate", "", "Rate of cache hits in summary cache",
+                   this),
+      cacheElements("cacheelements", "", "Number of elements in summary cache",
+                    this),
+      cacheMemoryUsed("cachememoryused", "", "Memory used by summary cache",
+                      this)
+{
+}
+
+void
+LegacyDocumentDBMetrics::MatchingMetrics::update(const MatchingStats &stats)
+{
+    docsMatched.inc(stats.docsMatched());
+    docsRanked.inc(stats.docsRanked());
+    docsReRanked.inc(stats.docsReRanked());
+    queries.inc(stats.queries());
+    queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(),
+                                      stats.queryCollateralTimeCount());
+    queryLatency.addValueBatch(stats.queryLatencyAvg(),
+                               stats.queryLatencyCount());
+}
+
+LegacyDocumentDBMetrics::MatchingMetrics::MatchingMetrics(MetricSet *parent)
+    : MetricSet("matching", "", "Matching metrics", parent),
+      docsMatched("docsmatched", "", "Number of documents matched", this),
+      docsRanked("docsranked", "", "Number of documents ranked (first phase)", this),
+      docsReRanked("docsreranked", "",
+                   "Number of documents re-ranked (second phase)", this),
+      queries("queries", "", "Number of queries executed", this),
+      queryCollateralTime("querycollateraltime", "", "Average time spent setting up and tearing down queries", this),
+      queryLatency("querylatency", "", "Average latency when matching a query", this)
+{
+}
+
+LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::RankProfileMetrics(
+        const std::string &name, size_t numDocIdPartitions, MetricSet *parent)
+    : MetricSet(name, "", "Rank profile metrics", parent),
+      queries("queries", "", "Number of queries executed", this),
+      limited_queries("limitedqueries", "", "Number of queries limited in match phase", this),
+      matchTime("match_time", "", "Average time for matching a query", this),
+      groupingTime("grouping_time", "", "Average time spent on grouping", this),
+      rerankTime("rerank_time", "", "Average time spent on 2nd phase ranking", this)
+{
+    for (size_t i=0; i < numDocIdPartitions; i++) {
+        vespalib::string s(make_string("docid_part%02ld", i));
+        partitions.push_back(DocIdPartition::LP(new DocIdPartition(s, this)));
+    }
+}
+
+LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::DocIdPartition::DocIdPartition(const std::string &name, MetricSet *parent) :
+    MetricSet(name, "", "DocId Partition profile metrics", parent),
+    docsMatched("docsmatched", "", "Number of documents matched", this),
+    docsRanked("docsranked", "", "Number of documents ranked (first phase)", this),
+    docsReRanked("docsreranked", "",
+                 "Number of documents re-ranked (second phase)", this),
+    active_time("activetime", "", "Time spent doing actual work", this),
+    wait_time("waittime", "", "Time spent waiting for other external threads and resources", this)
+{
+}
+
+void
+LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::DocIdPartition::update(const MatchingStats::Partition &stats)
+{
+    docsMatched.inc(stats.docsMatched());
+    docsRanked.inc(stats.docsRanked());
+    docsReRanked.inc(stats.docsReRanked());
+    active_time.addValueBatch(stats.active_time_avg(), stats.active_time_count());
+    wait_time.addValueBatch(stats.wait_time_avg(), stats.wait_time_count());
+}
+
+void
+LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::update(const MatchingStats &stats)
+{
+    queries.inc(stats.queries());
+    limited_queries.inc(stats.limited_queries());
+    matchTime.addValueBatch(stats.matchTimeAvg(), stats.matchTimeCount());
+    groupingTime.addValueBatch(stats.groupingTimeAvg(),
+                               stats.groupingTimeCount());
+    rerankTime.addValueBatch(stats.rerankTimeAvg(), stats.rerankTimeCount());
+    if (stats.getNumPartitions() > 0) {
+        if (stats.getNumPartitions() == partitions.size()) {
+            for (size_t i(0), m(stats.getNumPartitions()); i < m; i++) {
+                DocIdPartition & partition(*partitions[i]);
+                const MatchingStats::Partition & s(stats.getPartition(i));
+                partition.update(s);
+            }
+        } else {
+            vespalib::string msg(make_string("Num partitions use '%ld' is not equal to number of partitions '%ld' configured.",
+                                             stats.getNumPartitions(),
+                                             partitions.size()));
+            throw vespalib::IllegalStateException(msg, VESPA_STRLOC);
+        }
+    }
+}
+
+LegacyDocumentDBMetrics::SubDBMetrics::DocumentMetaStoreMetrics::
+DocumentMetaStoreMetrics(MetricSet *parent)
+    : MetricSet("docmetastore", "", "Document meta store metrics", parent),
+      lidLimit("lidlimit", "", "The size of the allocated lid space", this),
+      usedLids("usedlids", "", "The number of lids used", this),
+      lowestFreeLid("lowestfreelid", "", "The lowest free lid", this),
+      highestUsedLid("highestusedlid", "", "The highest used lid", this),
+      lidBloatFactor("lidbloatfactor", "", "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space "
+              "((lidlimit - usedlids) / lidlimit)", this),
+      lidFragmentationFactor("lid_fragmentation_factor", "",
+              "The fragmentation factor of this lid space, indicating the amount of holes in the currently used part of the lid space "
+              "((highestusedlid - usedlids) / highestusedlid)", this)
+{
+}
+
+LegacyDocumentDBMetrics::SubDBMetrics::SubDBMetrics(const vespalib::string &name,
+                                              MetricSet *parent)
+    : MetricSet(name, "", "Sub database metrics", parent),
+      attributes(this),
+      docMetaStore(this)
+{
+}
+
+LegacyDocumentDBMetrics::LegacyDocumentDBMetrics(const std::string &docTypeName, size_t maxNumThreads)
+    : MetricSet(make_string("%s", docTypeName.c_str()), "", "Document DB Metrics", 0),
+      index(this),
+      attributes(this),
+      docstore(this),
+      matching(this),
+      executor("executor", this),
+      indexExecutor("indexexecutor", this),
+      feed(this),
+      sessionManager(this),
+      ready("ready", this),
+      notReady("notready", this),
+      removed("removed", this),
+      memoryUsage("memoryusage", "", "Memory usage for this Document DB",
+                  this),
+      numDocs("numdocs", "",
+              "Number of ready/indexed documents in this Document DB (aka number of documents in the 'ready' sub db)", this),
+      numActiveDocs("numactivedocs", "",
+                    "Number of active/searchable documents in this Document DB (aka number of active/searchable documents in the 'ready' sub db)", this),
+      numIndexedDocs("numindexeddocs", "",
+                     "Number of ready/indexed documents in this Document DB (aka number of documents in the 'ready' sub db)", this),
+      numStoredDocs("numstoreddocs", "",
+                    "Total number of documents stored in this Document DB (aka number of documents in the 'ready' and 'notready' sub dbs)", this),
+      numRemovedDocs("numremoveddocs", "",
+                    "Number of removed documents in this Document DB (aka number of documents in the 'removed' sub db)", this),
+      numBadConfigs("numBadConfigs", "",
+                    "Number of bad configs for this Document DB", this),
+      _maxNumThreads(maxNumThreads)
+{
+    memoryUsage.addMetricToSum(index.memoryUsage);
+    memoryUsage.addMetricToSum(attributes.memoryUsage);
+    memoryUsage.addMetricToSum(docstore.memoryUsage);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h
new file mode 100644
index 00000000000..424eea31c10
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attribute_metrics.h"
+#include "executor_metrics.h"
+#include <vespa/metrics/metrics.h>
+#include "sessionmanager_metrics.h"
+#include "feed_metrics.h"
+#include <vespa/searchcore/proton/matching/matching_stats.h>
+
+namespace proton {
+
+/**
+ * Metric set for all legacy metrics reported for a document db.
+ *
+ * All these metrics have the document type name as part of the metric name,
+ * which is not flexible for setting up default metric graph dashboards.
+ *
+ * @deprecated Use DocumentDBTaggedMetrics for all new metrics.
+ */
+struct LegacyDocumentDBMetrics : metrics::MetricSet
+{
+    struct IndexMetrics : metrics::MetricSet {
+        metrics::LongValueMetric memoryUsage;
+        metrics::LongValueMetric docsInMemory;
+        metrics::LongValueMetric diskUsage;
+
+        IndexMetrics(metrics::MetricSet *parent);
+    };
+
+    struct DocstoreMetrics : metrics::MetricSet {
+        metrics::LongValueMetric memoryUsage;
+        metrics::LongCountMetric cacheLookups;
+        size_t hits;
+        metrics::LongAverageMetric cacheHitRate;
+        metrics::LongValueMetric cacheElements;
+        metrics::LongValueMetric cacheMemoryUsed;
+
+        DocstoreMetrics(metrics::MetricSet *parent);
+    };
+
+    struct MatchingMetrics : metrics::MetricSet {
+        metrics::LongCountMetric docsMatched;
+        metrics::LongCountMetric docsRanked;
+        metrics::LongCountMetric docsReRanked;
+        metrics::LongCountMetric queries;
+        metrics::DoubleAverageMetric queryCollateralTime;
+        metrics::DoubleAverageMetric queryLatency;
+
+        struct RankProfileMetrics : metrics::MetricSet {
+            struct DocIdPartition : metrics::MetricSet {
+                metrics::LongCountMetric docsMatched;
+                metrics::LongCountMetric docsRanked;
+                metrics::LongCountMetric docsReRanked;
+                metrics::DoubleAverageMetric active_time;
+                metrics::DoubleAverageMetric wait_time;
+
+                typedef vespalib::LinkedPtr<DocIdPartition> LP;
+                DocIdPartition(const std::string &name, metrics::MetricSet *parent);
+                void update(const matching::MatchingStats::Partition &stats);
+            };
+            typedef std::vector<DocIdPartition::LP> DocIdPartitions;
+
+            typedef vespalib::LinkedPtr<RankProfileMetrics> LP;
+
+            metrics::LongCountMetric     queries;
+            metrics::LongCountMetric     limited_queries;        
+            metrics::DoubleAverageMetric matchTime;
+            metrics::DoubleAverageMetric groupingTime;
+            metrics::DoubleAverageMetric rerankTime;
+            DocIdPartitions              partitions;
+
+            RankProfileMetrics(const std::string &name,
+                               size_t numDocIdPartitions,
+                               metrics::MetricSet *parent);
+            void update(const matching::MatchingStats &stats);
+
+        };
+        typedef std::map<std::string, RankProfileMetrics::LP> RankProfileMap;
+        RankProfileMap rank_profiles;
+
+        void update(const matching::MatchingStats &stats);
+        MatchingMetrics(metrics::MetricSet *parent);
+    };
+
+    struct SubDBMetrics : metrics::MetricSet
+    {
+        struct DocumentMetaStoreMetrics : metrics::MetricSet
+        {
+            metrics::LongValueMetric lidLimit;
+            metrics::LongValueMetric usedLids;
+            metrics::LongValueMetric lowestFreeLid;
+            metrics::LongValueMetric highestUsedLid;
+            metrics::DoubleValueMetric lidBloatFactor;
+            metrics::DoubleValueMetric lidFragmentationFactor;
+
+            DocumentMetaStoreMetrics(metrics::MetricSet *parent);
+        };
+
+        AttributeMetrics attributes;
+        DocumentMetaStoreMetrics docMetaStore;
+        SubDBMetrics(const vespalib::string &name, metrics::MetricSet *parent);
+    };
+
+    IndexMetrics                                 index;
+    AttributeMetrics                             attributes;
+    DocstoreMetrics                              docstore;
+    MatchingMetrics                              matching;
+    ExecutorMetrics                              executor;
+    ExecutorMetrics                              indexExecutor;
+    PerDocTypeFeedMetrics                        feed;
+    search::grouping::SessionManagerMetrics      sessionManager;
+    SubDBMetrics                                 ready;
+    SubDBMetrics                                 notReady;
+    SubDBMetrics                                 removed;
+    metrics::SumMetric<metrics::LongValueMetric> memoryUsage;
+    metrics::LongValueMetric                     numDocs;
+    metrics::LongValueMetric                     numActiveDocs;
+    metrics::LongValueMetric                     numIndexedDocs;
+    metrics::LongValueMetric                     numStoredDocs;
+    metrics::LongValueMetric                     numRemovedDocs;
+    metrics::LongValueMetric                     numBadConfigs;
+    size_t                                      _maxNumThreads;
+
+    LegacyDocumentDBMetrics(const std::string &docTypeName, size_t maxNumThreads);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp
new file mode 100644
index 00000000000..dc16d324d98
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.metrics.legacy_proton_metrics");
+#include "legacy_proton_metrics.h"
+
+namespace proton {
+
+LegacyProtonMetrics::DocumentTypeMetrics::DocumentTypeMetrics(metrics::MetricSet *parent)
+    : metrics::MetricSet("doctypes", "", "Metrics per document type", parent)
+{
+}
+
+LegacyProtonMetrics::LegacyProtonMetrics()
+    : metrics::MetricSet("proton", "", "Search engine metrics", 0),
+      docTypes(this),
+      executor("executor", this),
+      flushExecutor("flushexecutor", this),
+      matchExecutor("matchexecutor", this),
+      summaryExecutor("summaryexecutor", this),
+      memoryUsage("memoryusage", "logdefault", "Total tracked memory usage", this),
+      diskUsage("diskusage", "logdefault", "Total tracked disk usage", this),
+      docsInMemory("docsinmemory", "logdefault", "Total Number of documents in memory", this),
+      numDocs("numdocs", "logdefault", "Total number of ready/indexed documents among all document dbs (equal as numindexeddocs)", this),
+      numActiveDocs("numactivedocs", "logdefault",
+                    "Total number of active/searchable documents among all document dbs", this),
+      numIndexedDocs("numindexeddocs", "logdefault",
+                     "Total number of ready/indexed documents among all document dbs (equal as numdocs)", this),
+      numStoredDocs("numstoreddocs", "logdefault",
+                    "Total number of stored documents among all document dbs", this),
+      numRemovedDocs("numremoveddocs", "logdefault",
+                    "Total number of removed documents among all document dbs", this)
+{
+    // supply start value to support sum without any document types
+    metrics::LongValueMetric start("start", "", "", 0);
+    memoryUsage.setStartValue(start);
+    diskUsage.setStartValue(start);
+    docsInMemory.setStartValue(start);
+    numDocs.setStartValue(start);
+    numActiveDocs.setStartValue(start);
+    numIndexedDocs.setStartValue(start);
+    numStoredDocs.setStartValue(start);
+    numRemovedDocs.setStartValue(start);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.h
new file mode 100644
index 00000000000..81df510ec18
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include "executor_metrics.h"
+
+namespace proton {
+
+/**
+ * Metric set for all legacy metrics reported by proton.
+ *
+ * @deprecated Use ContentProtonMetrics for all new metrics.
+ */
+struct LegacyProtonMetrics : metrics::MetricSet
+{
+    struct DocumentTypeMetrics : metrics::MetricSet {
+        // documentdb metrics will be wired in here (by the metrics engine)
+        DocumentTypeMetrics(metrics::MetricSet *parent);
+    };
+
+    DocumentTypeMetrics                          docTypes;
+    ExecutorMetrics                              executor;
+    ExecutorMetrics                              flushExecutor;
+    ExecutorMetrics                              matchExecutor;
+    ExecutorMetrics                              summaryExecutor;
+    metrics::SumMetric<metrics::LongValueMetric> memoryUsage;
+    metrics::SumMetric<metrics::LongValueMetric> diskUsage;
+    metrics::SumMetric<metrics::LongValueMetric> docsInMemory;
+    metrics::SumMetric<metrics::LongValueMetric> numDocs;
+    metrics::SumMetric<metrics::LongValueMetric> numActiveDocs;
+    metrics::SumMetric<metrics::LongValueMetric> numIndexedDocs;
+    metrics::SumMetric<metrics::LongValueMetric> numStoredDocs;
+    metrics::SumMetric<metrics::LongValueMetric> numRemovedDocs;
+    // transport metrics will be wired in here
+
+    LegacyProtonMetrics();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp
new file mode 100644
index 00000000000..c5581822e9d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp
@@ -0,0 +1,234 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.metricsengine");
+#include "metrics_engine.h"
+#include <vespa/metrics/jsonwriter.h>
+
+namespace proton {
+
+MetricsEngine::MetricsEngine()
+    : _root(),
+      _legacyRoot(),
+      _manager(),
+      _metrics_producer(_manager)
+{
+}
+
+MetricsEngine::~MetricsEngine()
+{
+}
+
+void
+MetricsEngine::start(const config::ConfigUri &)
+{
+    {
+        metrics::MetricLockGuard guard(_manager.getMetricLock());
+        _manager.registerMetric(guard, _root);
+        _manager.registerMetric(guard, _legacyRoot);
+    }
+
+    // Storage doesnt snapshot unset metrics to save memory. Currently
+    // feature seems a bit bugged. Disabling this optimalization for search.
+    // Can enable it later when it is confirmed to be working well.
+    _manager.snapshotUnsetMetrics(true);
+
+    // Currently, when injecting a metric manager into the content layer,
+    // the content layer require to be the one initializing and starting it.
+    // Thus not calling init here, but further out in the application when
+    // one knows whether we are running in row/column mode or not
+}
+
+void
+MetricsEngine::addMetricsHook(metrics::MetricManager::UpdateHook &hook)
+{
+    _manager.addMetricUpdateHook(hook, 5);
+}
+
+void
+MetricsEngine::removeMetricsHook(metrics::MetricManager::UpdateHook &hook)
+{
+    _manager.removeMetricUpdateHook(hook);
+}
+
+void
+MetricsEngine::addExternalMetrics(metrics::Metric &child)
+{
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    _legacyRoot.registerMetric(child);
+}
+
+void
+MetricsEngine::removeExternalMetrics(metrics::Metric &child)
+{
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    _legacyRoot.unregisterMetric(child);
+}
+
+namespace {
+
+void
+addLegacyDocumentDBMetrics(LegacyProtonMetrics &legacyRoot,
+                           LegacyDocumentDBMetrics &metrics)
+{
+    legacyRoot.docTypes.registerMetric(metrics);
+    // cannot use sum of sum due to metric clone issues
+    legacyRoot.memoryUsage.addMetricToSum(metrics.index.memoryUsage);
+    legacyRoot.memoryUsage.addMetricToSum(metrics.attributes.memoryUsage);
+    legacyRoot.memoryUsage.addMetricToSum(metrics.docstore.memoryUsage);
+    legacyRoot.diskUsage.addMetricToSum(metrics.index.diskUsage);
+    legacyRoot.docsInMemory.addMetricToSum(metrics.index.docsInMemory);
+    legacyRoot.numDocs.addMetricToSum(metrics.numDocs);
+    legacyRoot.numActiveDocs.addMetricToSum(metrics.numActiveDocs);
+    legacyRoot.numIndexedDocs.addMetricToSum(metrics.numIndexedDocs);
+    legacyRoot.numStoredDocs.addMetricToSum(metrics.numStoredDocs);
+    legacyRoot.numRemovedDocs.addMetricToSum(metrics.numRemovedDocs);
+}
+
+void
+removeLegacyDocumentDBMetrics(LegacyProtonMetrics &legacyRoot,
+                              LegacyDocumentDBMetrics &metrics)
+{
+    legacyRoot.docTypes.unregisterMetric(metrics);
+    // cannot use sum of sum due to metric clone issues
+    legacyRoot.memoryUsage.removeMetricFromSum(metrics.index.memoryUsage);
+    legacyRoot.memoryUsage.removeMetricFromSum(metrics.attributes.memoryUsage);
+    legacyRoot.memoryUsage.removeMetricFromSum(metrics.docstore.memoryUsage);
+    legacyRoot.diskUsage.removeMetricFromSum(metrics.index.diskUsage);
+    legacyRoot.docsInMemory.removeMetricFromSum(metrics.index.docsInMemory);
+    legacyRoot.numDocs.removeMetricFromSum(metrics.numDocs);
+    legacyRoot.numActiveDocs.removeMetricFromSum(metrics.numActiveDocs);
+    legacyRoot.numIndexedDocs.removeMetricFromSum(metrics.numIndexedDocs);
+    legacyRoot.numStoredDocs.removeMetricFromSum(metrics.numStoredDocs);
+    legacyRoot.numRemovedDocs.removeMetricFromSum(metrics.numRemovedDocs);
+}
+
+}
+
+void
+MetricsEngine::addDocumentDBMetrics(DocumentDBMetricsCollection &child)
+{
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    addLegacyDocumentDBMetrics(_legacyRoot, child.getMetrics());
+
+    _root.registerMetric(child.getTaggedMetrics());
+}
+
+void
+MetricsEngine::removeDocumentDBMetrics(DocumentDBMetricsCollection &child)
+{
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    removeLegacyDocumentDBMetrics(_legacyRoot, child.getMetrics());
+
+    _root.unregisterMetric(child.getTaggedMetrics());
+}
+
+namespace {
+
+void
+doAddAttribute(AttributeMetrics &attributes,
+               const std::string &name)
+{
+    AttributeMetrics::List::Entry::LP entry = attributes.list.add(name);
+    if (entry.get() != 0) {
+        LOG(debug, "doAddAttribute(): name='%s', attributes=%p",
+                name.c_str(), (void*)&attributes);
+        attributes.list.registerMetric(*entry);
+    } else {
+        LOG(warning, "multiple attributes have the same name: '%s'", name.c_str());
+    }
+}
+
+void
+doRemoveAttribute(AttributeMetrics &attributes,
+                  const std::string &name)
+{
+    AttributeMetrics::List::Entry::LP entry = attributes.list.remove(name);
+    if (entry.get() != 0) {
+        LOG(debug, "doRemoveAttribute(): name='%s', attributes=%p",
+                name.c_str(), (void*)&attributes);
+        attributes.list.unregisterMetric(*entry);
+    } else {
+        LOG(debug, "Could not remove attribute with name '%s', not found", name.c_str());
+    }
+}
+
+void
+doCleanAttributes(AttributeMetrics &attributes)
+{
+    std::vector<AttributeMetrics::List::Entry::LP> entries = attributes.list.release();
+    for (size_t i = 0; i < entries.size(); ++i) {
+        attributes.list.unregisterMetric(*entries[i]);
+    }
+}
+
+}
+
+void
+MetricsEngine::addAttribute(AttributeMetrics &subAttributes,
+                            AttributeMetrics *totalAttributes,
+                            const std::string &name)
+{
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    doAddAttribute(subAttributes, name);
+    if (totalAttributes != NULL) {
+        doAddAttribute(*totalAttributes, name);
+    }
+}
+
+void
+MetricsEngine::removeAttribute(AttributeMetrics &subAttributes,
+                               AttributeMetrics *totalAttributes,
+                               const std::string &name)
+{
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    doRemoveAttribute(subAttributes, name);
+    if (totalAttributes != NULL) {
+        doRemoveAttribute(*totalAttributes, name);
+    }
+}
+
+void
+MetricsEngine::cleanAttributes(AttributeMetrics &subAttributes,
+                               AttributeMetrics *totalAttributes)
+{
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    doCleanAttributes(subAttributes);
+    if (totalAttributes != NULL) {
+        doCleanAttributes(*totalAttributes);
+    }
+}
+
+void MetricsEngine::addRankProfile(LegacyDocumentDBMetrics &owner,
+                                   const std::string &name,
+                                   size_t numDocIdPartitions) {
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::LP &entry =
+        owner.matching.rank_profiles[name];
+    if (entry.get()) {
+        LOG(warning, "Two rank profiles have the same name: %s", name.c_str());
+    } else {
+        owner.matching.rank_profiles[name].reset(
+                new LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics(
+                        name, std::min(numDocIdPartitions, owner._maxNumThreads), &owner.matching));
+    }
+}
+
+void MetricsEngine::cleanRankProfiles(LegacyDocumentDBMetrics &owner) {
+    metrics::MetricLockGuard guard(_manager.getMetricLock());
+    LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMap metrics;
+    owner.matching.rank_profiles.swap(metrics);
+    for (LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMap::const_iterator
+             it = metrics.begin(); it != metrics.end(); ++it) {
+        owner.matching.unregisterMetric(*it->second);
+    }
+}
+
+void
+MetricsEngine::stop()
+{
+    _manager.stop();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.h b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.h
new file mode 100644
index 00000000000..7c8fa0627e8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.h
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "content_proton_metrics.h"
+#include "documentdb_metrics_collection.h"
+#include "legacy_proton_metrics.h"
+#include "metricswireservice.h"
+#include <algorithm>
+#include <vespa/fastlib/net/httpserver.h>
+#include <vespa/metrics/metrics.h>
+#include <vespa/metrics/state_api_adapter.h>
+#include <vespa/vespalib/net/metrics_producer.h>
+
+namespace proton {
+
+class MetricsEngine : public MetricsWireService
+{
+private:
+    ContentProtonMetrics     _root;
+    LegacyProtonMetrics      _legacyRoot;
+    metrics::MetricManager   _manager;
+    metrics::StateApiAdapter _metrics_producer;
+
+public:
+    typedef std::unique_ptr<MetricsEngine> UP;
+
+    MetricsEngine();
+    virtual ~MetricsEngine();
+    ContentProtonMetrics &root() { return _root; }
+    LegacyProtonMetrics &legacyRoot() { return _legacyRoot; }
+    void start(const config::ConfigUri & configUri);
+    void addMetricsHook(metrics::MetricManager::UpdateHook &hook);
+    void removeMetricsHook(metrics::MetricManager::UpdateHook &hook);
+    void addExternalMetrics(metrics::Metric &child);
+    void removeExternalMetrics(metrics::Metric &child);
+    void addDocumentDBMetrics(DocumentDBMetricsCollection &child);
+    void removeDocumentDBMetrics(DocumentDBMetricsCollection &child);
+    virtual void addAttribute(AttributeMetrics &subAttributes,
+                              AttributeMetrics *totalAttributes,
+                              const std::string &name);
+    virtual void removeAttribute(AttributeMetrics &subAttributes,
+                                 AttributeMetrics *totalAttributes,
+                                 const std::string &name);
+    virtual void cleanAttributes(AttributeMetrics &subAttributes,
+                                 AttributeMetrics *totalAttributes);
+    virtual void addRankProfile(LegacyDocumentDBMetrics &owner,
+                                const std::string &name,
+                                size_t numDocIdPartitions);
+    virtual void cleanRankProfiles(LegacyDocumentDBMetrics &owner);
+    void stop();
+
+    vespalib::MetricsProducer &metrics_producer() { return _metrics_producer; }
+    metrics::MetricManager &getManager() { return _manager; }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metricswireservice.h b/searchcore/src/vespa/searchcore/proton/metrics/metricswireservice.h
new file mode 100644
index 00000000000..0ca8f587b10
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/metricswireservice.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <string>
+
+namespace proton {
+class AttributeMetrics;
+class LegacyDocumentDBMetrics;
+
+struct MetricsWireService {
+    virtual void addAttribute(AttributeMetrics &subAttributes,
+                              AttributeMetrics *totalAttributes,
+                              const std::string &name) = 0;
+    virtual void removeAttribute(AttributeMetrics &subAttributes,
+                                 AttributeMetrics *totalAttributes,
+                                 const std::string &name) = 0;
+    virtual void cleanAttributes(AttributeMetrics &subAttributes,
+                                 AttributeMetrics *totalAttributes) = 0;
+    virtual void addRankProfile(LegacyDocumentDBMetrics &owner,
+                                const std::string &name,
+                                size_t numDocIdPartitions) = 0;
+    virtual void cleanRankProfiles(LegacyDocumentDBMetrics &owner) = 0;
+    virtual ~MetricsWireService() {}
+};
+
+struct DummyWireService : public MetricsWireService {
+    virtual void addAttribute(AttributeMetrics &, AttributeMetrics *, const std::string &) {}
+    virtual void removeAttribute(AttributeMetrics &, AttributeMetrics *, const std::string &) {}
+    virtual void cleanAttributes(AttributeMetrics &, AttributeMetrics *) {}
+    virtual void addRankProfile(LegacyDocumentDBMetrics &, const std::string &, size_t) {}
+    virtual void cleanRankProfiles(LegacyDocumentDBMetrics &) {}
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp
new file mode 100644
index 00000000000..1820502a235
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "resource_usage_metrics.h"
+
+namespace proton {
+
+ResourceUsageMetrics::ResourceUsageMetrics(metrics::MetricSet *parent)
+    : MetricSet("resource_usage", "", "Usage metrics for various resources in this search engine", parent),
+      disk("disk", "", "The relative amount of disk space used on this machine (value in the range [0, 1])", this),
+      memory("memory", "", "The relative amount of memory used by this process (value in the range [0, 1])", this),
+      feedingBlocked("feeding_blocked", "", "Whether feeding is blocked due to resource limits being reached (value is either 0 or 1)", this)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h
new file mode 100644
index 00000000000..df5387a6a50
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+
+namespace proton {
+
+/**
+ * Usage metrics for various resources in this search engine.
+ */
+struct ResourceUsageMetrics : metrics::MetricSet
+{
+    metrics::DoubleValueMetric disk;
+    metrics::DoubleValueMetric memory;
+    metrics::LongValueMetric feedingBlocked;
+
+    ResourceUsageMetrics(metrics::MetricSet *parent);
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/sessionmanager_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/sessionmanager_metrics.cpp
new file mode 100644
index 00000000000..61c1c970b4c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/sessionmanager_metrics.cpp
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".grouping.sessionmanagermetrics");
+#include "sessionmanager_metrics.h"
+
+namespace search {
+namespace grouping {
+
+SessionManagerMetrics::SessionManagerMetrics(metrics::MetricSet *parent)
+    : metrics::MetricSet("sessionmanager", "",
+                         "Grouping session manager metrics",
+                         parent),
+      numInsert("numinsert", "", "Number of inserted sessions", this),
+      numPick("numpick", "", "Number if picked sessions", this),
+      numDropped("numdropped", "", "Number of dropped cached sessions", this),
+      numCached("numcached", "", "Number of currently cached sessions", this),
+      numTimedout("numtimedout", "", "Number of timed out sessions", this)
+{
+}
+
+void
+SessionManagerMetrics::update(
+        const proton::matching::SessionManager::Stats &stats)
+{
+    numInsert.inc(stats.numInsert);
+    numPick.inc(stats.numPick);
+    numDropped.inc(stats.numDropped);
+    numCached.set(stats.numCached);
+    numTimedout.inc(stats.numTimedout);
+}
+
+
+}  // namespace grouping
+}  // namespace search
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/sessionmanager_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/sessionmanager_metrics.h
new file mode 100644
index 00000000000..9978db818ec
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/sessionmanager_metrics.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
+
+namespace search {
+namespace grouping {
+
+struct SessionManagerMetrics : metrics::MetricSet
+{
+    metrics::LongCountMetric numInsert;
+    metrics::LongCountMetric numPick;
+    metrics::LongCountMetric numDropped;
+    metrics::LongValueMetric numCached;
+    metrics::LongCountMetric numTimedout;
+
+    void update(const proton::matching::SessionManager::Stats &stats);
+    SessionManagerMetrics(metrics::MetricSet *parent);
+};
+
+}  // namespace grouping
+}  // namespace search
+
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp
new file mode 100644
index 00000000000..606312fcae2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "trans_log_server_metrics.h"
+
+using search::transactionlog::DomainInfo;
+using search::transactionlog::DomainStats;
+
+namespace proton {
+
+TransLogServerMetrics::DomainMetrics::DomainMetrics(metrics::MetricSet *parent,
+                                                    const vespalib::string &documentType)
+    : metrics::MetricSet("transactionlog", {{"documenttype", documentType}},
+            "Transaction log metrics for a document type", parent),
+      entries("entries", "", "The current number of entries in the transaction log", this)
+{
+}
+
+void
+TransLogServerMetrics::DomainMetrics::update(const DomainInfo &stats)
+{
+    entries.set(stats.count);
+}
+
+void
+TransLogServerMetrics::considerAddDomains(const DomainStats &stats)
+{
+    for (const auto &elem : stats) {
+        const vespalib::string &documentType = elem.first;
+        if (_domainMetrics.find(documentType) == _domainMetrics.end()) {
+            _domainMetrics[documentType] = DomainMetrics::UP(new DomainMetrics(_parent, documentType));
+        }
+    }
+}
+
+void
+TransLogServerMetrics::considerRemoveDomains(const DomainStats &stats)
+{
+    for (auto itr = _domainMetrics.begin(); itr != _domainMetrics.end(); ) {
+        const vespalib::string &documentType = itr->first;
+        if (stats.find(documentType) == stats.end()) {
+            itr = _domainMetrics.erase(itr);
+        } else {
+            ++itr;
+        }
+    }
+}
+
+void
+TransLogServerMetrics::updateDomainMetrics(const DomainStats &stats)
+{
+    for (const auto &elem : stats) {
+        const vespalib::string &documentType = elem.first;
+        _domainMetrics.find(documentType)->second->update(elem.second);
+    }
+}
+
+TransLogServerMetrics::TransLogServerMetrics(metrics::MetricSet *parent)
+    : _parent(parent)
+{
+}
+
+void
+TransLogServerMetrics::update(const DomainStats &stats)
+{
+    considerAddDomains(stats);
+    considerRemoveDomains(stats);
+    updateDomainMetrics(stats);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h
new file mode 100644
index 00000000000..0d52f6cf0d1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/searchlib/transactionlog/domain.h>
+
+namespace proton {
+
+/**
+ * Metric set for all metrics reported by transaction log server.
+ */
+class TransLogServerMetrics
+{
+public:
+    struct DomainMetrics : public metrics::MetricSet
+    {
+        metrics::LongValueMetric entries;
+
+        typedef std::unique_ptr<DomainMetrics> UP;
+        DomainMetrics(metrics::MetricSet *parent, const vespalib::string &documentType);
+        void update(const search::transactionlog::DomainInfo &stats);
+    };
+
+private:
+    metrics::MetricSet *_parent;
+    std::map<vespalib::string, DomainMetrics::UP> _domainMetrics;
+
+    void considerAddDomains(const search::transactionlog::DomainStats &stats);
+    void considerRemoveDomains(const search::transactionlog::DomainStats &stats);
+    void updateDomainMetrics(const search::transactionlog::DomainStats &stats);
+
+public:
+    TransLogServerMetrics(metrics::MetricSet *parent);
+    void update(const search::transactionlog::DomainStats &stats);
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/persistenceengine/CMakeLists.txt
new file mode 100644
index 00000000000..e847e266177
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_persistenceengine STATIC
+    SOURCES
+    document_iterator.cpp
+    i_document_retriever.cpp
+    persistenceengine.cpp
+    transport_latch.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/OWNERS b/searchcore/src/vespa/searchcore/proton/persistenceengine/OWNERS
new file mode 100644
index 00000000000..7ae1acb1be9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/OWNERS
@@ -0,0 +1 @@
+geirst
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/bucket_guard.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/bucket_guard.h
new file mode 100644
index 00000000000..04f34cefdef
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/bucket_guard.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <memory>
+#include <vespa/searchcore/proton/server/ibucketfreezer.h>
+
+namespace proton {
+
+class BucketGuard {
+    document::BucketId _bucket;
+    IBucketFreezer    &_freezer;
+
+public:
+    typedef std::unique_ptr<BucketGuard> UP;
+    BucketGuard(const BucketGuard &) = delete;
+    BucketGuard & operator=(const BucketGuard &) = delete;
+    BucketGuard(BucketGuard &&) = delete;
+    BucketGuard & operator=(BucketGuard &&) = delete;
+
+    BucketGuard(document::BucketId bucket, IBucketFreezer &freezer)
+        : _bucket(bucket),
+          _freezer(freezer)
+    {
+        freezer.freezeBucket(bucket);
+    }
+
+    ~BucketGuard() {
+        _freezer.thawBucket(_bucket);
+    }
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
new file mode 100644
index 00000000000..63f3850583c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
@@ -0,0 +1,268 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "document_iterator.h"
+#include <vespa/document/select/gid_filter.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/persistence/spi/docentry.h>
+#include <algorithm>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.persistenceengine.document_iterator");
+
+using storage::spi::IterateResult;
+using storage::spi::DocEntry;
+using storage::spi::Timestamp;
+using document::Document;
+using document::DocumentId;
+
+
+namespace proton {
+
+namespace {
+
+DocEntry *createDocEntry(Timestamp timestamp, bool removed) {
+    int flags = removed ? storage::spi::REMOVE_ENTRY : storage::spi::NONE;
+    return new DocEntry(timestamp, flags);
+}
+
+DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ssize_t defaultSerializedSize) {
+    if (doc) {
+        if (removed) {
+            return new DocEntry(timestamp, storage::spi::REMOVE_ENTRY, doc->getId());
+        } else {
+            ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->getSerializedSize();
+            return new DocEntry(timestamp, storage::spi::NONE, std::move(doc), serializedSize);
+        }
+    } else {
+        return createDocEntry(timestamp, removed);
+    }
+}
+
+} // namespace proton::<unnamed>
+
+bool
+DocumentIterator::useDocumentSelection() const
+{
+    return (!_metaOnly &&
+            !_selection.getDocumentSelection().getDocumentSelection().empty());
+}
+
+bool
+DocumentIterator::checkMeta(const search::DocumentMetaData &meta) const
+{
+    if (!meta.valid()) {
+        return false;
+    }
+    if (!_selection.getTimestampSubset().empty()) {
+        return (std::binary_search(_selection.getTimestampSubset().begin(),
+                                   _selection.getTimestampSubset().end(),
+                                   Timestamp(meta.timestamp)));
+    }
+    if ((meta.timestamp < _selection.getFromTimestamp()) ||
+        (meta.timestamp > _selection.getToTimestamp()))
+    {
+        return false;
+    }
+    if ((_versions == storage::spi::NEWEST_DOCUMENT_ONLY) && meta.removed) {
+        return false;
+    }
+    return true;
+}
+
+DocumentIterator::DocumentIterator(const storage::spi::Bucket &bucket,
+                                   const document::FieldSet& fields,
+                                   const storage::spi::Selection &selection,
+                                   storage::spi::IncludedVersions versions,
+                                   ssize_t defaultSerializedSize,
+                                   bool ignoreMaxBytes,
+                                   ReadConsistency readConsistency)
+    : _bucket(bucket),
+      _selection(selection),
+      _versions(versions),
+      _fields(fields.clone()),
+      _defaultSerializedSize(defaultSerializedSize),
+      _readConsistency(readConsistency),
+      _metaOnly(fields.getType() == document::FieldSet::NONE),
+      _ignoreMaxBytes(ignoreMaxBytes),
+      _sources(),
+      _nextItem(0),
+      _list()
+{
+}
+
+void
+DocumentIterator::add(const IDocumentRetriever::SP &retriever)
+{
+    _sources.push_back(retriever);
+}
+
+IterateResult
+DocumentIterator::iterate(size_t maxBytes)
+{
+    if (_list.empty()) {
+        for (const IDocumentRetriever::SP & source : _sources) {
+            fetchCompleteSource(*source, _list);
+        }
+    }
+    if ( _ignoreMaxBytes ) {
+        return IterateResult(std::move(_list), true);
+    } else {
+        IterateResult::List results;
+        for (size_t sz(0); (_nextItem < _list.size()) && ((sz < maxBytes) || results.empty()); _nextItem++) {
+            DocEntry::LP & item = _list[_nextItem];
+            sz += item->getSize();
+            results.push_back(item);
+            item.reset();
+        }
+        return IterateResult(results, _nextItem >= _list.size());
+    }
+}
+
+namespace {
+
+class Match {
+public:
+    Match(const IDocumentRetriever & source, bool metaOnly, const vespalib::string & selection) :
+        _dscTrue(true),
+        _metaOnly(metaOnly),
+        _willAlwaysFail(false)
+    {
+        if (!(_metaOnly || selection.empty())) {
+            LOG(spam, "ParseSelect: %s", selection.c_str());
+            _cs = source.parseSelect(selection);
+            CachedSelect &cs(*_cs);
+            _dscTrue = cs._allTrue;
+            if (cs._allFalse || cs._allInvalid) {
+                assert(!_dscTrue);
+                LOG(debug, "Nothing will ever match cs._allFalse = '%d' cs._allInvalid = '%d'", cs._allFalse, cs._allInvalid);
+                _willAlwaysFail = true;
+            } else {
+                _select = (cs._attrSelect ? cs._attrSelect->clone()
+                                          : cs._select->clone());
+                using document::select::GidFilter;
+                _gidFilter = GidFilter::for_selection_root_node(*_select);
+                _sc.reset(new SelectContext(*_cs));
+                _sc->getAttributeGuards();
+            }
+        } else {
+            _dscTrue = true;
+        }
+    }
+    
+    ~Match() {
+        if (_sc) {
+            _sc->dropAttributeGuards();
+        }
+    }
+
+    bool willAlwaysFail() const { return _willAlwaysFail; }
+
+    bool match(const search::DocumentMetaData & meta) const {
+        if (_dscTrue || _metaOnly) {
+            return true;
+        }
+        if (_sc) {
+            _sc->_docId = meta.lid;
+        }
+        if (!_gidFilter.gid_might_match_selection(meta.gid)) {
+            return false;
+        }
+        return (! _cs->_attrSelect) ||
+                (_cs->_attrSelect && (_select->contains(*_sc) == document::select::Result::True));
+    }
+    bool match(const search::DocumentMetaData & meta, const Document * doc) const {
+        if (_dscTrue || _metaOnly) {
+            return true;
+        }
+        return (doc && (doc->getId().getGlobalId() == meta.gid) &&
+               (_cs->_attrSelect || (_select->contains(*doc) == document::select::Result::True)));
+    }
+private:
+    bool                           _dscTrue;
+    bool                           _metaOnly;
+    bool                           _willAlwaysFail;
+    CachedSelect::SP               _cs;
+    document::select::Node::UP     _select;
+    document::select::GidFilter _gidFilter;
+    std::unique_ptr<SelectContext> _sc;
+};
+
+typedef vespalib::hash_map<uint32_t, uint32_t> LidIndexMap;
+
+class MatchVisitor : public search::IDocumentVisitor
+{
+public:
+    MatchVisitor(const Match & matcher, const search::DocumentMetaData::Vector & metaData,
+                 const LidIndexMap & lidIndexMap, const document::FieldSet * fields, IterateResult::List & list,
+                 ssize_t defaultSerializedSize) :
+        _matcher(matcher),
+        _metaData(metaData),
+        _lidIndexMap(lidIndexMap),
+        _fields(fields),
+        _list(list),
+        _defaultSerializedSize(defaultSerializedSize)
+    { }
+    void visit(uint32_t lid, document::Document::UP doc) override {
+        const search::DocumentMetaData & meta = _metaData[_lidIndexMap[lid]];
+        assert(lid == meta.lid);
+        if (_matcher.match(meta, doc.get())) {
+            if (doc && _fields) {
+                document::FieldSet::stripFields(*doc, *_fields);
+            }
+            _list.emplace_back(createDocEntry(meta.timestamp, meta.removed, std::move(doc), _defaultSerializedSize));
+        }
+    }
+private:
+    const Match                            & _matcher;
+    const search::DocumentMetaData::Vector & _metaData;
+    const LidIndexMap                      & _lidIndexMap;
+    const document::FieldSet               * _fields;
+    IterateResult::List                    & _list;
+    size_t                                   _defaultSerializedSize;
+};
+
+}
+
+void
+DocumentIterator::fetchCompleteSource(const IDocumentRetriever & source, IterateResult::List & list)
+{
+    search::DocumentMetaData::Vector metaData;
+    source.getBucketMetaData(_bucket, metaData);
+    if (metaData.empty()) {
+        return;
+    }
+    LOG(debug, "metadata count before filtering: %zu", metaData.size());
+
+    Match matcher(source, _metaOnly, _selection.getDocumentSelection().getDocumentSelection());
+    if (matcher.willAlwaysFail()) {
+        return;
+    }
+
+    LidIndexMap lidIndexMap(3*metaData.size());
+    IDocumentRetriever::LidVector lidsToFetch;
+    lidsToFetch.reserve(metaData.size());
+    for (size_t i(0); i < metaData.size(); i++) {
+        const search::DocumentMetaData & meta = metaData[i];
+        if (checkMeta(meta)) {
+            if (matcher.match(meta)) {
+                lidsToFetch.emplace_back(meta.lid);
+                lidIndexMap[meta.lid] = i;
+            }
+        }
+    }
+    LOG(debug, "metadata count after filtering: %zu", lidsToFetch.size());
+
+    if ( _metaOnly ) {
+        for (uint32_t lid : lidsToFetch) {
+            const search::DocumentMetaData & meta = metaData[lidIndexMap[lid]];
+            assert(lid == meta.lid);
+            list.emplace_back(createDocEntry(meta.timestamp, meta.removed));
+        }
+    } else {
+        MatchVisitor visitor(matcher, metaData, lidIndexMap, _fields.get(), list, _defaultSerializedSize);
+        source.visitDocuments(lidsToFetch, visitor, _readConsistency);
+    }
+
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h
new file mode 100644
index 00000000000..71fb435190f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/persistence/spi/bucket.h>
+#include <vespa/persistence/spi/selection.h>
+#include <vespa/persistence/spi/result.h>
+#include "i_document_retriever.h"
+#include <vespa/searchlib/common/idocumentmetastore.h>
+#include <vespa/searchcore/proton/common/cachedselect.h>
+#include <vespa/searchcore/proton/common/selectcontext.h>
+#include <vespa/persistence/spi/read_consistency.h>
+
+namespace proton {
+
+class DocumentIterator
+{
+private:
+    using ReadConsistency = storage::spi::ReadConsistency;
+    const storage::spi::Bucket            _bucket;;
+    const storage::spi::Selection         _selection;
+    const storage::spi::IncludedVersions  _versions;
+    const document::FieldSet::UP          _fields;
+    const ssize_t                         _defaultSerializedSize;
+    const ReadConsistency                 _readConsistency;
+    const bool                            _metaOnly;
+    const bool                            _ignoreMaxBytes;
+    std::vector<IDocumentRetriever::SP>   _sources;
+    size_t                                _nextItem;
+    storage::spi::IterateResult::List     _list;
+
+
+    bool useDocumentSelection() const;
+    bool checkMeta(const search::DocumentMetaData &meta) const;
+    bool checkDoc(const document::Document &doc) const;
+    bool checkDoc(const SelectContext &sc) const;
+    void fetchCompleteSource(const IDocumentRetriever & source, storage::spi::IterateResult::List & list);
+
+public:
+    DocumentIterator(const storage::spi::Bucket &bucket,
+                     const document::FieldSet& fields,
+                     const storage::spi::Selection &selection,
+                     storage::spi::IncludedVersions versions,
+                     ssize_t defaultSerializedSize,
+                     bool ignoreMaxBytes,
+                     ReadConsistency readConsistency=ReadConsistency::STRONG);
+    void add(const IDocumentRetriever::SP &retriever);
+    storage::spi::IterateResult iterate(size_t maxBytes);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp
new file mode 100644
index 00000000000..81a56f4aac6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/persistence/spi/read_consistency.h>
+#include <vespa/fastos/fastos.h>
+#include "i_document_retriever.h"
+
+namespace proton {
+
+void DocumentRetrieverBaseForTest::visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const {
+    (void) readConsistency;
+    for (uint32_t lid : lids) {
+        visitor.visit(lid, getDocument(lid));
+    }
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h
new file mode 100644
index 00000000000..cf0f8e8334c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <persistence/spi/types.h>
+#include <vespa/persistence/spi/bucket.h>
+#include <vespa/persistence/spi/read_consistency.h>
+#include <vespa/searchlib/common/idocumentmetastore.h>
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/searchcore/proton/common/cachedselect.h>
+#include <vespa/searchlib/query/base.h>
+#include <memory>
+
+namespace proton
+{
+
+/**
+ * This is an interface that allows retrieval of documents by local id and document metadata
+ * by either bucket or document id.
+ * It also provides a callback interface known in VDS as visitation.
+ **/ 
+class IDocumentRetriever
+{
+public:
+    using ReadConsistency = storage::spi::ReadConsistency;
+    typedef std::unique_ptr<IDocumentRetriever> UP;
+    typedef std::shared_ptr<IDocumentRetriever> SP;
+
+    typedef search::IDocumentStore::LidVector LidVector;
+    virtual ~IDocumentRetriever() {}
+
+    virtual const document::DocumentTypeRepo &getDocumentTypeRepo() const = 0;
+    virtual void getBucketMetaData(const storage::spi::Bucket &bucket, search::DocumentMetaData::Vector &result) const = 0;
+    virtual search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const = 0;
+    virtual document::Document::UP getDocument(search::DocumentIdT lid) const = 0;
+    /**
+     * Will visit all documents in the the given list. Visit order is undefined and will
+     * be conducted in most efficient retrieval order.
+     * @param lids to visit
+     * @param Visitor to receive callback for each document found.
+     */
+    virtual void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const = 0;
+
+    virtual CachedSelect::SP parseSelect(const vespalib::string &selection) const = 0;
+};
+
+class DocumentRetrieverBaseForTest : public IDocumentRetriever {
+public:
+    virtual void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h
new file mode 100644
index 00000000000..0e7414da4c5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+/**
+ * Interface used to deny write operations when resource limits are reached.
+ */
+struct IResourceWriteFilter
+{
+    class State
+    {
+    private:
+        bool _acceptWriteOperation;
+        vespalib::string _message;
+    public:
+        State()
+            : _acceptWriteOperation(true),
+              _message()
+        {}
+        State(bool acceptWriteOperation_, const vespalib::string &message_)
+            : _acceptWriteOperation(acceptWriteOperation_),
+              _message(message_)
+        {}
+        bool acceptWriteOperation() const { return _acceptWriteOperation; }
+        const vespalib::string &message() const { return _message; }
+    };
+
+    virtual ~IResourceWriteFilter() {}
+
+    virtual bool acceptWriteOperation() const = 0;
+    virtual State getAcceptState() const = 0;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h
new file mode 100644
index 00000000000..b8952edb990
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/persistence/spi/abstractpersistenceprovider.h>
+
+namespace proton
+{
+
+class IPersistenceEngineOwner
+{
+public:
+    virtual
+    ~IPersistenceEngineOwner(void)
+    {
+    }
+
+    virtual void
+    setClusterState(const storage::spi::ClusterState &calc) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h
new file mode 100644
index 00000000000..3e558b71e30
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "bucket_guard.h"
+#include "i_document_retriever.h"
+#include "resulthandler.h"
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/persistence/spi/abstractpersistenceprovider.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+
+namespace proton {
+
+/**
+ * This interface describes a sync persistence operation handler. It is implemented by
+ * the DocumentDB and other classes, and used by the PersistenceEngine class to delegate
+ * operations to the appropriate db.
+ */
+class IPersistenceHandler {
+public:
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::unique_ptr<IPersistenceHandler> UP;
+    typedef std::shared_ptr<IPersistenceHandler> SP;
+    typedef std::shared_ptr<
+        std::vector<IDocumentRetriever::SP> > RetrieversSP;
+
+    /**
+     * Virtual destructor to allow inheritance.
+     */
+    virtual ~IPersistenceHandler() { }
+
+    /**
+     * Called before all other functions so that the persistence handler
+     * can initialize itself before being used.
+     */
+    virtual void initialize() = 0;
+
+    virtual void handlePut(FeedToken token,
+                           const storage::spi::Bucket &bucket,
+                           storage::spi::Timestamp timestamp,
+                           const document::Document::SP &doc) = 0;
+
+    virtual void handleUpdate(FeedToken token,
+                              const storage::spi::Bucket &bucket,
+                              storage::spi::Timestamp timestamp,
+                              const document::DocumentUpdate::SP &upd) = 0;
+
+    virtual void handleRemove(FeedToken token,
+                              const storage::spi::Bucket &bucket,
+                              storage::spi::Timestamp timestamp,
+                              const document::DocumentId &id) = 0;
+
+    virtual void handleListBuckets(IBucketIdListResultHandler &resultHandler) = 0;
+
+    virtual void handleSetClusterState(const storage::spi::ClusterState &calc,
+                                       IGenericResultHandler &resultHandler) = 0;
+
+    virtual void handleSetActiveState(const storage::spi::Bucket &bucket,
+                                      storage::spi::BucketInfo::ActiveState newState,
+                                      IGenericResultHandler &resultHandler) = 0;
+
+    virtual void handleGetBucketInfo(const storage::spi::Bucket &bucket,
+                                     IBucketInfoResultHandler &resultHandler) = 0;
+
+    virtual void
+    handleCreateBucket(FeedToken token,
+                       const storage::spi::Bucket &bucket) = 0;
+
+    virtual void handleDeleteBucket(FeedToken token,
+                                    const storage::spi::Bucket &bucket) = 0;
+
+    virtual void handleGetModifiedBuckets(IBucketIdListResultHandler &resultHandler) = 0;
+
+    virtual void
+    handleSplit(FeedToken token,
+                const storage::spi::Bucket &source,
+                const storage::spi::Bucket &target1,
+                const storage::spi::Bucket &target2) = 0;
+
+    virtual void
+    handleJoin(FeedToken token,
+               const storage::spi::Bucket &source,
+               const storage::spi::Bucket &target1,
+               const storage::spi::Bucket &target2) = 0;
+
+    virtual RetrieversSP getDocumentRetrievers() = 0;
+    virtual BucketGuard::UP lockBucket(const storage::spi::Bucket &bucket) = 0;
+
+    virtual void
+    handleListActiveBuckets(IBucketIdListResultHandler &resultHandler) = 0;
+
+    virtual void
+    handlePopulateActiveBuckets(document::BucketId::List &buckets,
+                                IGenericResultHandler &resultHandler) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
new file mode 100644
index 00000000000..a81adf1134a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
@@ -0,0 +1,780 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.persistenceengine.persistenceengine");
+
+#include <vespa/documentapi/messagebus/messages/feedreply.h>
+#include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
+#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h>
+#include "ipersistenceengineowner.h"
+#include "transport_latch.h"
+#include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/sequence.h>
+
+using document::Document;
+using document::DocumentId;
+using documentapi::DocumentReply;
+using documentapi::RemoveDocumentReply;
+using mbus::Reply;
+using storage::spi::BucketChecksum;
+using storage::spi::BucketIdListResult;
+using storage::spi::BucketInfo;
+using storage::spi::BucketInfoResult;
+using storage::spi::IncludedVersions;
+using storage::spi::PartitionState;
+using storage::spi::PartitionStateList;
+using storage::spi::Result;
+using vespalib::IllegalStateException;
+using vespalib::LockGuard;
+using vespalib::Sequence;
+using vespalib::make_string;
+using vespalib::RWLockReader;
+using vespalib::RWLockWriter;
+
+namespace proton {
+
+namespace {
+
+class ResultHandlerBase {
+protected:
+    vespalib::Lock           _lock;
+    vespalib::CountDownLatch _latch;
+public:
+    ResultHandlerBase(uint32_t waitCnt) : _lock(), _latch(waitCnt) {}
+    void await() { _latch.await(); }
+};
+
+
+class GenericResultHandler : public ResultHandlerBase, public IGenericResultHandler {
+private:
+    Result _result;
+public:
+    GenericResultHandler(uint32_t waitCnt) :
+        ResultHandlerBase(waitCnt),
+        _result()
+    {
+    }
+    virtual void handle(const Result &result) {
+        if (result.hasError()) {
+            vespalib::LockGuard guard(_lock);
+            if (_result.hasError()) {
+                _result = TransportLatch::mergeErrorResults(_result, result);
+            } else {
+                _result = result;
+            }
+        }
+        _latch.countDown();
+    }
+    const Result &getResult() const { return _result; }
+};
+
+
+class BucketIdListResultHandler : public IBucketIdListResultHandler
+{
+private:
+    typedef vespalib::hash_set<document::BucketId, document::BucketId::hash> BucketIdSet;
+    BucketIdSet _bucketSet;
+public:
+    BucketIdListResultHandler()
+        : _bucketSet()
+    {
+    }
+    virtual void handle(const BucketIdListResult &result) {
+        const BucketIdListResult::List &buckets = result.getList();
+        for (size_t i = 0; i < buckets.size(); ++i) {
+            _bucketSet.insert(buckets[i]);
+        }
+    }
+    BucketIdListResult getResult() const {
+        BucketIdListResult::List buckets;
+        buckets.reserve(_bucketSet.size());
+        for (document::BucketId bucketId : _bucketSet) {
+            buckets.push_back(bucketId);
+        }
+        return BucketIdListResult(buckets);
+    }
+};
+
+
+class SynchronizedBucketIdListResultHandler : public ResultHandlerBase,
+                                              public BucketIdListResultHandler
+{
+public:
+    SynchronizedBucketIdListResultHandler(uint32_t waitCnt)
+        : ResultHandlerBase(waitCnt),
+          BucketIdListResultHandler()
+    {
+    }
+    virtual void handle(const BucketIdListResult &result) {
+        {
+            vespalib::LockGuard guard(_lock);
+            BucketIdListResultHandler::handle(result);
+        }
+        _latch.countDown();
+    }
+};
+
+
+class BucketInfoResultHandler : public IBucketInfoResultHandler {
+private:
+    BucketInfoResult _result;
+    bool             _first;
+public:
+    BucketInfoResultHandler() :
+        _result(BucketInfo()),
+        _first(true)
+    {
+    }
+    virtual void handle(const BucketInfoResult &result) {
+        if (_first) {
+            _result = result;
+            _first = false;
+        } else {
+            BucketInfo b1 = _result.getBucketInfo();
+            BucketInfo b2 = result.getBucketInfo();
+            BucketInfo::ReadyState ready =
+                    (b1.getReady() == b2.getReady() ? b1.getReady() :
+                            BucketInfo::NOT_READY);
+            BucketInfo::ActiveState active =
+                    (b1.getActive() == b2.getActive() ? b1.getActive() :
+                            BucketInfo::NOT_ACTIVE);
+            _result = BucketInfoResult(
+                    BucketInfo(BucketChecksum(b1.getChecksum() + b2.getChecksum()),
+                            b1.getDocumentCount() + b2.getDocumentCount(),
+                            b1.getDocumentSize() + b2.getDocumentSize(),
+                            b1.getEntryCount() + b2.getEntryCount(),
+                            b1.getUsedSize() + b2.getUsedSize(),
+                            ready, active));
+        }
+    }
+    const BucketInfoResult &getResult() const { return _result; }
+};
+
+}
+
+#define NOT_YET throw vespalib::IllegalArgumentException("Not implemented yet")
+
+PersistenceEngine::HandlerSnapshot::UP
+PersistenceEngine::getHandlerSnapshot() const
+{
+    LockGuard guard(_lock);
+    return std::make_unique<HandlerSnapshot>(_handlers.snapshot(), _handlers.size());
+}
+
+namespace {
+template <typename T>
+class SequenceOfOne : public Sequence<T> {
+    bool _done;
+    T _value;
+public:
+    SequenceOfOne(const T &value) : _done(false), _value(value) {}
+
+    virtual bool valid() const { return !_done; }
+    virtual T get() const { return _value; }
+    virtual void next() { _done = true; }
+};
+
+template <typename T>
+typename Sequence<T>::UP make_sequence(const T &value) {
+    return typename Sequence<T>::UP(new SequenceOfOne<T>(value));
+}
+}  // namespace
+
+PersistenceEngine::HandlerSnapshot::UP
+PersistenceEngine::getHandlerSnapshot(const DocumentId &id) const {
+    if (!id.hasDocType()) {
+        return getHandlerSnapshot();
+    }
+    IPersistenceHandler::SP handler = getHandler(DocTypeName(id.getDocType()));
+    if (!handler.get()) {
+        return HandlerSnapshot::UP();
+    }
+    return HandlerSnapshot::UP(
+            new HandlerSnapshot(make_sequence(handler.get()), 1));
+}
+
+PersistenceEngine::PersistenceEngine(IPersistenceEngineOwner &owner,
+                                     const IResourceWriteFilter &writeFilter,
+                                     ssize_t defaultSerializedSize,
+                                     bool ignoreMaxBytes)
+    : AbstractPersistenceProvider(),
+      _defaultSerializedSize(defaultSerializedSize),
+      _ignoreMaxBytes(ignoreMaxBytes),
+      _handlers(),
+      _lock(),
+      _iterators(),
+      _iterators_lock(),
+      _owner(owner),
+      _writeFilter(writeFilter),
+      _clusterState(),
+      _extraModifiedBuckets(),
+      _rwLock()
+{
+}
+
+
+PersistenceEngine::~PersistenceEngine()
+{
+    destroyIterators();
+}
+
+
+IPersistenceHandler::SP
+PersistenceEngine::putHandler(const DocTypeName &docType,
+                              const IPersistenceHandler::SP &handler)
+{
+    LockGuard guard(_lock);
+    return _handlers.putHandler(docType, handler);
+}
+
+
+IPersistenceHandler::SP
+PersistenceEngine::getHandler(const DocTypeName &docType) const
+{
+    LockGuard guard(_lock);
+    return _handlers.getHandler(docType);
+}
+
+
+IPersistenceHandler::SP
+PersistenceEngine::removeHandler(const DocTypeName &docType)
+{
+    // TODO: Grab bucket list and treat them as modified
+    LockGuard guard(_lock);
+    return _handlers.removeHandler(docType);
+}
+
+
+Result
+PersistenceEngine::initialize()
+{
+    RWLockWriter wguard(getWLock());
+    LOG(debug, "Begin initializing persistence handlers");
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        handler->initialize();
+    }
+    LOG(debug, "Done initializing persistence handlers");
+    return Result();
+}
+
+
+PartitionStateListResult
+PersistenceEngine::getPartitionStates() const
+{
+    PartitionStateList list(1);
+    return PartitionStateListResult(list);
+}
+
+
+BucketIdListResult
+PersistenceEngine::listBuckets(PartitionId id) const
+{
+    // Runs in SPI thread.
+    // No handover to write threads in persistence handlers.
+    RWLockReader rguard(getRLock());
+    if (id != 0) {
+        BucketIdListResult::List emptyList;
+        return BucketIdListResult(emptyList);
+    }
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    BucketIdListResultHandler resultHandler;
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        handler->handleListBuckets(resultHandler);
+    }
+    return resultHandler.getResult();
+}
+
+
+Result
+PersistenceEngine::setClusterState(const ClusterState &calc)
+{
+    RWLockReader rguard(getRLock());
+    saveClusterState(calc);
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    GenericResultHandler resultHandler(snap->size());
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        handler->handleSetClusterState(calc, resultHandler);
+    }
+    resultHandler.await();
+    _owner.setClusterState(calc);
+    return resultHandler.getResult();
+}
+
+
+Result
+PersistenceEngine::setActiveState(const Bucket& bucket,
+                                  storage::spi::BucketInfo::ActiveState newState)
+{
+    RWLockReader rguard(getRLock());
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    GenericResultHandler resultHandler(snap->size());
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        handler->handleSetActiveState(bucket, newState, resultHandler);
+    }
+    resultHandler.await();
+    return resultHandler.getResult();
+}
+
+
+BucketInfoResult
+PersistenceEngine::getBucketInfo(const Bucket& b) const
+{
+    // Runs in SPI thread.
+    // No handover to write threads in persistence handlers.
+    RWLockReader rguard(getRLock());
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    BucketInfoResultHandler resultHandler;
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        handler->handleGetBucketInfo(b, resultHandler);
+    }
+    return resultHandler.getResult();
+}
+
+
+Result
+PersistenceEngine::put(const Bucket& b, Timestamp t, const document::Document::SP& doc, Context&)
+{
+    if (!_writeFilter.acceptWriteOperation()) {
+        IResourceWriteFilter::State state = _writeFilter.getAcceptState();
+        if (!state.acceptWriteOperation()) {
+            return Result(Result::RESOURCE_EXHAUSTED,
+                          make_string("Put operation rejected for document '%s': '%s'",
+                                      doc->getId().toString().c_str(), state.message().c_str()));
+        }
+    }
+    RWLockReader rguard(getRLock());
+    DocTypeName docType(doc->getType());
+    LOG(spam,
+        "put(%s, %" PRIu64 ", (\"%s\", \"%s\"))",
+        b.toString().c_str(),
+        static_cast<uint64_t>(t.getValue()),
+        docType.toString().c_str(),
+        doc->getId().toString().c_str());
+    if (!doc->getId().hasDocType()) {
+        return Result(Result::PERMANENT_ERROR, make_string(
+                        "Old id scheme not supported in elastic mode (%s)",
+                        doc->getId().toString().c_str()));
+    }
+    IPersistenceHandler::SP handler = getHandler(docType);
+    if (handler.get() == NULL) {
+        return Result(Result::PERMANENT_ERROR,
+                      make_string("No handler for document type '%s'",
+                                  docType.toString().c_str()));
+    }
+    TransportLatch latch(1);
+    FeedToken token(latch, mbus::Reply::UP(new documentapi::FeedReply(
+                           documentapi::DocumentProtocol::REPLY_PUTDOCUMENT)));
+    handler->handlePut(token, b, t, doc);
+    latch.await();
+    return latch.getResult();
+}
+
+RemoveResult
+PersistenceEngine::remove(const Bucket& b, Timestamp t, const DocumentId& did, Context&)
+{
+    RWLockReader rguard(getRLock());
+    LOG(spam,
+        "remove(%s, %" PRIu64 ", \"%s\")",
+        b.toString().c_str(),
+        static_cast<uint64_t>(t.getValue()),
+        did.toString().c_str());
+    HandlerSnapshot::UP snap = getHandlerSnapshot(did);
+    if (!snap.get()) {
+        return RemoveResult(false);
+    }
+    TransportLatch latch(snap->size());
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        FeedToken token(latch, Reply::UP(new RemoveDocumentReply));
+        handler->handleRemove(token, b, t, did);
+    }
+    latch.await();
+    return latch.getRemoveResult();
+}
+
+
+UpdateResult
+PersistenceEngine::update(const Bucket& b, Timestamp t, const DocumentUpdate::SP& upd, Context&)
+{
+    if (!_writeFilter.acceptWriteOperation()) {
+        IResourceWriteFilter::State state = _writeFilter.getAcceptState();
+        if (!state.acceptWriteOperation()) {
+            return UpdateResult(Result::RESOURCE_EXHAUSTED,
+                                make_string("Update operation rejected for document '%s': '%s'",
+                                            upd->getId().toString().c_str(), state.message().c_str()));
+        }
+    }
+    RWLockReader rguard(getRLock());
+    DocTypeName docType(upd->getType());
+    LOG(spam,
+        "update(%s, %" PRIu64 ", (\"%s\", \"%s\"), createIfNonExistent='%s')",
+        b.toString().c_str(),
+        static_cast<uint64_t>(t.getValue()),
+        docType.toString().c_str(),
+        upd->getId().toString().c_str(),
+        (upd->getCreateIfNonExistent() ? "true" : "false"));
+    IPersistenceHandler::SP handler = getHandler(docType);
+    TransportLatch latch(1);
+    if (handler.get() != NULL) {
+        FeedToken token(latch, mbus::Reply::UP(new documentapi::UpdateDocumentReply()));
+        LOG(debug, "update = %s", upd->toXml().c_str());
+        handler->handleUpdate(token, b, t, upd);
+        latch.await();
+    } else {
+        return UpdateResult(Result::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str()));
+    }
+    return latch.getUpdateResult();
+}
+
+
+GetResult
+PersistenceEngine::get(const Bucket& b,
+                       const document::FieldSet& fields,
+                       const DocumentId& did,
+                       Context&) const
+{
+    RWLockReader rguard(getRLock());
+    HandlerSnapshot::UP snapshot = getHandlerSnapshot();
+
+    for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) {
+        BucketGuard::UP bucket_guard = handlers.get()->lockBucket(b);
+        IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers();
+        for (size_t i = 0; i < retrievers->size(); ++i) {
+            IDocumentRetriever &retriever = *(*retrievers)[i];
+            search::DocumentMetaData meta = retriever.getDocumentMetaData(did);
+            if (meta.timestamp != 0 && meta.bucketId == b.getBucketId()) {
+                if (meta.removed) {
+                    return GetResult();
+                }
+                document::Document::UP doc = retriever.getDocument(meta.lid);
+                if (!doc || doc->getId().getGlobalId() != meta.gid) {
+                    return GetResult();
+                }
+                document::FieldSet::stripFields(*doc, fields);
+                return GetResult(std::move(doc), meta.timestamp);
+            }
+        }
+    }
+    return GetResult();
+}
+
+
+CreateIteratorResult
+PersistenceEngine::createIterator(const Bucket &bucket,
+                                  const document::FieldSet& fields,
+                                  const Selection &selection,
+                                  IncludedVersions versions,
+                                  Context & context)
+{
+    RWLockReader rguard(getRLock());
+    HandlerSnapshot::UP snapshot = getHandlerSnapshot();
+
+    IteratorEntry *entry = new IteratorEntry(context.getReadConsistency(), bucket, fields, selection,
+                                             versions, _defaultSerializedSize, _ignoreMaxBytes);
+    entry->bucket_guards.reserve(snapshot->size());
+    for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) {
+        entry->bucket_guards.push_back(handlers.get()->lockBucket(bucket));
+        IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers();
+        for (size_t i = 0; i < retrievers->size(); ++i) {
+            entry->it.add((*retrievers)[i]);
+        }
+    }
+    entry->handler_sequence = HandlerSnapshot::release(std::move(*snapshot));
+
+    LockGuard guard(_iterators_lock);
+    static IteratorId id_counter(0);
+    IteratorId id(++id_counter);
+    _iterators[id] = entry;
+    return CreateIteratorResult(id);
+}
+
+
+IterateResult
+PersistenceEngine::iterate(IteratorId id, uint64_t maxByteSize, Context&) const
+{
+    RWLockReader rguard(getRLock());
+    LockGuard guard(_iterators_lock);
+    Iterators::const_iterator it = _iterators.find(id);
+    if (it == _iterators.end()) {
+        return IterateResult(Result::PERMANENT_ERROR, make_string("Unknown iterator with id %" PRIu64, id.getValue()));
+    }
+    if (it->second->in_use) {
+        return IterateResult(Result::TRANSIENT_ERROR, make_string("Iterator with id %" PRIu64 " is already in use", id.getValue()));
+    }
+    it->second->in_use = true;
+    guard.unlock();
+
+    DocumentIterator &iterator = it->second->it;
+    try {
+        IterateResult result = iterator.iterate(maxByteSize);
+        LockGuard guard2(_iterators_lock);
+        it->second->in_use = false;
+        return result;
+    } catch (const std::exception & e) {
+        IterateResult result(Result::PERMANENT_ERROR, make_string("Caught exception during visitor iterator.iterate() = '%s'", e.what()));
+        LOG(warning, "Caught exception during visitor iterator.iterate() = '%s'", e.what());
+        LockGuard guard2(_iterators_lock);
+        it->second->in_use = false;
+        return result;
+    }
+}
+
+
+Result
+PersistenceEngine::destroyIterator(IteratorId id, Context&)
+{
+    RWLockReader rguard(getRLock());
+    LockGuard guard(_iterators_lock);
+    Iterators::iterator it = _iterators.find(id);
+    if (it == _iterators.end()) {
+        return Result();
+    }
+    if (it->second->in_use) {
+        return Result(Result::TRANSIENT_ERROR, make_string("Iterator with id %" PRIu64 " is currently in use", id.getValue()));
+    }
+    delete it->second;
+    _iterators.erase(it);
+    return Result();
+}
+
+
+Result
+PersistenceEngine::createBucket(const Bucket &b, Context &)
+{
+    RWLockReader rguard(getRLock());
+    LOG(spam, "createBucket(%s)", b.toString().c_str());
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    TransportLatch latch(snap->size());
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        FeedToken token(latch, Reply::UP(new DocumentReply(0)));
+        handler->handleCreateBucket(token, b);
+    }
+    latch.await();
+    return latch.getResult();
+}
+
+
+Result
+PersistenceEngine::deleteBucket(const Bucket& b, Context&)
+{
+    RWLockReader rguard(getRLock());
+    LOG(spam, "deleteBucket(%s)", b.toString().c_str());
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    TransportLatch latch(snap->size());
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        FeedToken token(latch, Reply::UP(new DocumentReply(0)));
+        handler->handleDeleteBucket(token, b);
+    }
+    latch.await();
+    return latch.getResult();
+}
+
+
+BucketIdListResult
+PersistenceEngine::getModifiedBuckets() const
+{
+    RWLockReader rguard(getRLock());
+    typedef BucketIdListResultV MBV;
+    MBV extraModifiedBuckets;
+    {
+        LockGuard guard(_lock);
+        extraModifiedBuckets.swap(_extraModifiedBuckets);
+    }
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    SynchronizedBucketIdListResultHandler resultHandler(snap->size() + extraModifiedBuckets.size());
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        handler->handleGetModifiedBuckets(resultHandler);
+    }
+    for (const auto & item : extraModifiedBuckets) {
+        resultHandler.handle(*item);
+    }
+    resultHandler.await();
+    return resultHandler.getResult();
+}
+
+
+Result
+PersistenceEngine::split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&)
+{
+    RWLockReader rguard(getRLock());
+    LOG(spam, "split(%s, %s, %s)", source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str());
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    TransportLatch latch(snap->size());
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        FeedToken token(latch, Reply::UP(new DocumentReply(0)));
+        handler->handleSplit(token, source, target1, target2);
+    }
+    latch.await();
+    return latch.getResult();
+}
+
+
+Result
+PersistenceEngine::join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&)
+{
+    RWLockReader rguard(getRLock());
+    LOG(spam, "join(%s, %s, %s)", source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str());
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    TransportLatch latch(snap->size());
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        FeedToken token(latch, Reply::UP(new DocumentReply(0)));
+        handler->handleJoin(token, source1, source2, target);
+    }
+    latch.await();
+    return latch.getResult();
+}
+
+
+Result
+PersistenceEngine::maintain(const Bucket& , MaintenanceLevel)
+{
+    return Result();
+}
+
+void
+PersistenceEngine::destroyIterators(void)
+{
+    Context context(storage::spi::LoadType(0, "default"),
+                    storage::spi::Priority(0x80),
+                    storage::spi::Trace::TraceLevel(0));
+    for (;;) {
+        IteratorId id;
+        {
+            LockGuard guard(_iterators_lock);
+            if (_iterators.empty())
+                break;
+            id = _iterators.begin()->first;
+        }
+        Result res(destroyIterator(id, context));
+        if (res.hasError()) {
+            LOG(debug, "%ld iterator left. Can not destroy iterator '%ld'. Reason='%s'", _iterators.size(), id.getValue(), res.toString().c_str());
+            FastOS_Thread::Sleep(100); // Sleep 0.1 seconds
+        }
+    }
+}
+
+
+void
+PersistenceEngine::saveClusterState(const ClusterState &calc)
+{
+    auto clusterState = std::make_shared<ClusterState>(calc);
+    {
+        LockGuard guard(_lock);
+        clusterState.swap(_clusterState);
+    }
+}
+
+ClusterState::SP
+PersistenceEngine::savedClusterState(void) const
+{
+    LockGuard guard(_lock);
+    return _clusterState;
+}
+
+void
+PersistenceEngine::propagateSavedClusterState(IPersistenceHandler &handler)
+{
+    ClusterState::SP clusterState(savedClusterState());
+    if (clusterState.get() == NULL)
+        return;
+    // Propagate saved cluster state.
+    // TODO: Fix race with new cluster state setting.
+    GenericResultHandler resultHandler(1);
+    handler.handleSetClusterState(*clusterState, resultHandler);
+    resultHandler.await();
+}
+
+void
+PersistenceEngine::grabExtraModifiedBuckets(IPersistenceHandler &handler)
+{
+    BucketIdListResultHandler resultHandler;
+    handler.handleListBuckets(resultHandler);
+    auto result = std::make_shared<BucketIdListResult>(resultHandler.getResult());
+    LockGuard guard(_lock);
+    _extraModifiedBuckets.push_back(result);
+}
+
+
+class ActiveBucketIdListResultHandler : public IBucketIdListResultHandler
+{
+private:
+    typedef std::map<document::BucketId, size_t> BucketIdMap;
+    typedef  std::pair<BucketIdMap::iterator, bool> IR;
+    BucketIdMap _bucketMap;
+public:
+    ActiveBucketIdListResultHandler() : _bucketMap() { }
+
+    virtual void handle(const BucketIdListResult &result) {
+        const BucketIdListResult::List &buckets = result.getList();
+        for (size_t i = 0; i < buckets.size(); ++i) {
+            IR ir(_bucketMap.insert(std::make_pair(buckets[i], 1u)));
+            if (!ir.second) {
+                ++(ir.first->second);
+            }
+        }
+    }
+
+    const BucketIdMap & getBucketMap(void) const { return _bucketMap; }
+};
+
+void
+PersistenceEngine::populateInitialBucketDB(IPersistenceHandler &targetHandler)
+{
+    HandlerSnapshot::UP snap = getHandlerSnapshot();
+    
+    size_t snapSize(snap->size());
+    size_t flawed = 0;
+
+    // handleListActiveBuckets() runs in SPI thread.
+    // No handover to write threads in persistence handlers.
+    ActiveBucketIdListResultHandler resultHandler;
+    for (; snap->handlers().valid(); snap->handlers().next()) {
+        IPersistenceHandler *handler = snap->handlers().get();
+        handler->handleListActiveBuckets(resultHandler);
+    }
+    typedef std::map<document::BucketId, size_t> BucketIdMap;
+    document::BucketId::List buckets;
+    const BucketIdMap &bucketMap(resultHandler.getBucketMap());
+
+    for (const auto & item : bucketMap) {
+        if (item.second != snapSize) {
+            ++flawed;
+        }
+        buckets.push_back(item.first);
+    }
+    LOG(info, "Adding %zu active buckets (%zu flawed) to new bucket db", buckets.size(), flawed);
+    GenericResultHandler trHandler(1);
+    targetHandler.handlePopulateActiveBuckets(buckets, trHandler);
+    trHandler.await();
+}
+
+
+RWLockReader
+PersistenceEngine::getRLock(void) const
+{
+    return RWLockReader(_rwLock);
+}
+
+
+RWLockWriter
+PersistenceEngine::getWLock(void) const
+{
+    return RWLockWriter(_rwLock);
+}
+
+
+} // storage
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h
new file mode 100644
index 00000000000..fc7b38a25ff
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h
@@ -0,0 +1,143 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "document_iterator.h"
+#include "i_resource_write_filter.h"
+#include <vespa/persistence/spi/abstractpersistenceprovider.h>
+#include <vespa/searchcore/proton/common/handlermap.hpp>
+#include <vespa/searchcore/proton/persistenceengine/ipersistencehandler.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton {
+
+using document::DocumentUpdate;
+using storage::spi::Bucket;
+using storage::spi::BucketIdListResult;
+using storage::spi::BucketInfo;
+using storage::spi::BucketInfoResult;
+using storage::spi::ClusterState;
+using storage::spi::Context;
+using storage::spi::CreateIteratorResult;
+using storage::spi::GetResult;
+using storage::spi::IncludedVersions;
+using storage::spi::IterateResult;
+using storage::spi::IteratorId;
+using storage::spi::MaintenanceLevel;
+using storage::spi::PartitionId;
+using storage::spi::PartitionStateListResult;
+using storage::spi::RemoveResult;
+using storage::spi::Result;
+using storage::spi::Selection;
+using storage::spi::Timestamp;
+using storage::spi::TimestampList;
+using storage::spi::UpdateResult;
+
+class IPersistenceEngineOwner;
+
+class PersistenceEngine : public storage::spi::AbstractPersistenceProvider {
+private:
+    typedef vespalib::Sequence<IPersistenceHandler *> PersistenceHandlerSequence;
+
+    class HandlerSnapshot {
+    private:
+        PersistenceHandlerSequence::UP _handlers;
+        size_t                         _size;
+    public:
+        typedef std::unique_ptr<HandlerSnapshot> UP;
+        HandlerSnapshot(PersistenceHandlerSequence::UP handlers_, size_t size_) :
+            _handlers(handlers_.release()),
+            _size(size_)
+        {}
+        HandlerSnapshot(const HandlerSnapshot &) = delete;
+        HandlerSnapshot & operator = (const HandlerSnapshot &) = delete;
+
+        size_t size() const { return _size; }
+        PersistenceHandlerSequence &handlers() { return *_handlers; }
+        static PersistenceHandlerSequence::UP release(HandlerSnapshot && rhs) { return std::move(rhs._handlers); }
+    };
+
+    struct IteratorEntry {
+        PersistenceHandlerSequence::UP handler_sequence;
+        DocumentIterator it;
+        bool in_use;
+        std::vector<BucketGuard::UP> bucket_guards;
+        IteratorEntry(storage::spi::ReadConsistency readConsistency,
+                      const Bucket &b,
+                      const document::FieldSet& f,
+                      const Selection &s,
+                      IncludedVersions v,
+                      ssize_t defaultSerializedSize,
+                      bool ignoreMaxBytes)
+            : handler_sequence(),
+              it(b, f, s, v, defaultSerializedSize, ignoreMaxBytes, readConsistency),
+              in_use(false),
+              bucket_guards() {}
+    };
+    typedef std::map<IteratorId, IteratorEntry *> Iterators;
+    typedef std::vector<std::shared_ptr<BucketIdListResult> > BucketIdListResultV;
+
+    const ssize_t                           _defaultSerializedSize;
+    const bool                              _ignoreMaxBytes;
+    mutable HandlerMap<IPersistenceHandler> _handlers;
+    vespalib::Lock                          _lock;
+    Iterators                               _iterators;
+    vespalib::Lock                          _iterators_lock;
+    IPersistenceEngineOwner                &_owner;
+    const IResourceWriteFilter             &_writeFilter;
+    ClusterState::SP                        _clusterState;
+    mutable BucketIdListResultV             _extraModifiedBuckets;
+    mutable vespalib::RWLock                _rwLock;
+
+    HandlerSnapshot::UP getHandlerSnapshot() const;
+    HandlerSnapshot::UP getHandlerSnapshot(const document::DocumentId &) const;
+
+    void saveClusterState(const ClusterState &calc);
+    ClusterState::SP savedClusterState(void) const;
+
+public:
+    typedef std::unique_ptr<PersistenceEngine> UP;
+
+    PersistenceEngine(IPersistenceEngineOwner &owner,
+                      const IResourceWriteFilter &writeFilter,
+                      ssize_t defaultSerializedSize, bool ignoreMaxBytes);
+    ~PersistenceEngine();
+
+    IPersistenceHandler::SP putHandler(const DocTypeName &docType, const IPersistenceHandler::SP &handler);
+    IPersistenceHandler::SP getHandler(const DocTypeName &docType) const;
+    IPersistenceHandler::SP removeHandler(const DocTypeName &docType);
+
+    // Implements PersistenceProvider
+    virtual Result initialize() override;
+    virtual PartitionStateListResult getPartitionStates() const override;
+    virtual BucketIdListResult listBuckets(PartitionId) const override;
+    virtual Result setClusterState(const ClusterState& calc) override;
+    virtual Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override;
+    virtual BucketInfoResult getBucketInfo(const Bucket&) const;
+    virtual Result put(const Bucket&, Timestamp, const document::Document::SP&, Context&) override;
+    virtual RemoveResult remove(const Bucket&, Timestamp, const document::DocumentId&, Context&) override;
+    virtual UpdateResult update(const Bucket&, Timestamp, const document::DocumentUpdate::SP&, Context&) override;
+    virtual GetResult get(const Bucket&, const document::FieldSet&, const document::DocumentId&, Context&) const override;
+    virtual CreateIteratorResult createIterator(const Bucket&, const document::FieldSet&, const Selection&,
+                                                IncludedVersions, Context&) override;
+    virtual IterateResult iterate(IteratorId, uint64_t maxByteSize, Context&) const override;
+    virtual Result destroyIterator(IteratorId, Context&) override;
+
+    virtual Result createBucket(const Bucket &bucketId, Context &) override ;
+    virtual Result deleteBucket(const Bucket&, Context&) override;
+    virtual BucketIdListResult getModifiedBuckets() const override;
+    virtual Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override;
+    virtual Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override;
+
+    virtual Result maintain(const Bucket&, MaintenanceLevel) override;
+
+    void destroyIterators();
+    void propagateSavedClusterState(IPersistenceHandler &handler);
+    void grabExtraModifiedBuckets(IPersistenceHandler &handler);
+    void populateInitialBucketDB(IPersistenceHandler &targetHandler);
+    vespalib::RWLockReader getRLock(void) const;
+    vespalib::RWLockWriter getWLock(void) const;
+};
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h
new file mode 100644
index 00000000000..c939ef1e3ee
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/persistence/spi/abstractpersistenceprovider.h>
+
+namespace proton {
+
+template <typename ResultType>
+class IResultHandler {
+public:
+    virtual ~IResultHandler() { }
+    virtual void handle(const ResultType &result) = 0;
+};
+
+typedef IResultHandler<storage::spi::BucketIdListResult> IBucketIdListResultHandler;
+typedef IResultHandler<storage::spi::BucketInfoResult>   IBucketInfoResultHandler;
+typedef IResultHandler<storage::spi::Result>             IGenericResultHandler;
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
new file mode 100644
index 00000000000..55062b3c201
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.persistenceengine.transportlatch");
+
+#include "transport_latch.h"
+
+using storage::spi::Result;
+
+namespace proton {
+
+TransportLatch::TransportLatch(uint32_t cnt)
+    : _latch(cnt),
+      _lock(),
+      _result()
+{}
+
+void
+TransportLatch::send(mbus::Reply::UP reply,
+                     ResultUP result,
+                     bool documentWasFound,
+                     double latency_ms)
+{
+    (void) reply;
+    (void) latency_ms;
+    {
+        vespalib::LockGuard guard(_lock);
+        if (!_result.get()) {
+            _result = std::move(result);
+        } else if (result->hasError()) {
+            _result.reset(new Result(mergeErrorResults(*_result, *result)));
+        } else if (documentWasFound) {
+            _result = std::move(result);
+        }
+    }
+    _latch.countDown();
+}
+
+Result
+TransportLatch::mergeErrorResults(const Result &lhs, const Result &rhs)
+{
+    Result::ErrorType error = (lhs.getErrorCode() > rhs.getErrorCode() ? lhs : rhs).getErrorCode();
+    return Result(error, vespalib::make_string("%s, %s",
+                                               lhs.getErrorMessage().c_str(),
+                                               rhs.getErrorMessage().c_str()));
+}
+
+} // proton
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h
new file mode 100644
index 00000000000..49b721edd23
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/persistence/spi/result.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/vespalib/util/sequence.h>
+
+namespace proton {
+
+/**
+ * Implementation of FeedToken::ITransport for handling the async reply for an operation.
+ * Uses an internal count down latch to keep track the number of outstanding replies.
+ */
+class TransportLatch : public FeedToken::ITransport {
+private:
+    vespalib::CountDownLatch _latch;
+    vespalib::Lock           _lock;
+    ResultUP                 _result;
+
+public:
+    TransportLatch(uint32_t cnt);
+    virtual void send(mbus::Reply::UP reply,
+                      ResultUP result,
+                      bool documentWasFound,
+                      double latency_ms);
+    void await() {
+        _latch.await();
+    }
+    const storage::spi::UpdateResult &getUpdateResult() const {
+        return dynamic_cast<const storage::spi::UpdateResult &>(*_result);
+    }
+    const storage::spi::Result &getResult() const {
+        return *_result;
+    }
+    const storage::spi::RemoveResult &getRemoveResult() const {
+        return dynamic_cast<const storage::spi::RemoveResult &>(*_result);
+    }
+    static storage::spi::Result mergeErrorResults(const storage::spi::Result &lhs,
+                                                  const storage::spi::Result &rhs);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/reprocessing/CMakeLists.txt
new file mode 100644
index 00000000000..ba78d75f482
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_reprocessing STATIC
+    SOURCES
+    attribute_reprocessing_initializer.cpp
+    document_reprocessing_handler.cpp
+    reprocess_documents_task.cpp
+    reprocessingrunner.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS b/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS
new file mode 100644
index 00000000000..7c03446a5d4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS
@@ -0,0 +1,2 @@
+geirst
+tegge
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp
new file mode 100644
index 00000000000..3d7473d779a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp
@@ -0,0 +1,147 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.reprocessing.attribute_reprocessing_initializer");
+
+#include "attribute_reprocessing_initializer.h"
+#include <vespa/searchcore/proton/attribute/attribute_populator.h>
+#include <vespa/searchcore/proton/attribute/document_field_populator.h>
+#include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
+
+using search::index::Schema;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::SerialNum;
+
+namespace proton {
+
+typedef AttributeReprocessingInitializer::Config ARIConfig;
+
+namespace {
+
+constexpr search::SerialNum ATTRIBUTE_INIT_SERIAL = 1;
+
+const char *
+toStr(bool value)
+{
+    return (value ? "true" : "false");
+}
+
+bool fastPartialUpdateAttribute(const Schema::DataType &attrType) {
+    // Partial update to tensor or predicate attribute must update document
+    return ((attrType != Schema::BOOLEANTREE) && (attrType != Schema::TENSOR));
+}
+
+
+FilterAttributeManager::AttributeSet
+getAttributeSetToPopulate(const ARIConfig &newCfg,
+                          const ARIConfig &oldCfg)
+{
+    FilterAttributeManager::AttributeSet attrsToPopulate;
+    std::vector<AttributeGuard> attrList;
+    newCfg.getAttrMgr()->getAttributeList(attrList);
+    for (const auto &guard : attrList) {
+        const vespalib::string &name = guard->getName();
+        bool inOldAttrMgr = oldCfg.getAttrMgr()->getAttribute(name)->valid();
+        bool inOldSchema = oldCfg.getInspector()->hasField(name);
+        bool populateAttribute = !inOldAttrMgr && inOldSchema;
+        LOG(debug, "getAttributeSetToPopulate(): name='%s', inOldAttrMgr=%s, inOldSchema=%s, populate=%s",
+                name.c_str(), toStr(inOldAttrMgr), toStr(inOldSchema), toStr(populateAttribute));
+        if (populateAttribute) {
+            attrsToPopulate.insert(name);
+        }
+    }
+    return attrsToPopulate;
+}
+
+IReprocessingReader::SP
+getAttributesToPopulate(const ARIConfig &newCfg,
+                        const ARIConfig &oldCfg,
+                        const vespalib::string &subDbName)
+{
+    FilterAttributeManager::AttributeSet attrsToPopulate =
+            getAttributeSetToPopulate(newCfg, oldCfg);
+    if (!attrsToPopulate.empty()) {
+        return IReprocessingReader::SP(new AttributePopulator
+                (IAttributeManager::SP(new FilterAttributeManager
+                        (attrsToPopulate, newCfg.getAttrMgr())),
+                        ATTRIBUTE_INIT_SERIAL, subDbName));
+    }
+    return IReprocessingReader::SP();
+}
+
+Schema::AttributeField
+getAttributeField(const Schema &schema, const vespalib::string &name)
+{
+    uint32_t attrFieldId = schema.getAttributeFieldId(name);
+    assert(attrFieldId != Schema::UNKNOWN_FIELD_ID);
+    return schema.getAttributeField(attrFieldId);
+}
+
+std::vector<IReprocessingRewriter::SP>
+getFieldsToPopulate(const ARIConfig &newCfg,
+                    const ARIConfig &oldCfg,
+                    const vespalib::string &subDbName)
+{
+    std::vector<IReprocessingRewriter::SP> fieldsToPopulate;
+    std::vector<AttributeGuard> attrList;
+    oldCfg.getAttrMgr()->getAttributeList(attrList);
+    for (const auto &guard : attrList) {
+        const vespalib::string &name = guard->getName();
+        Schema::AttributeField attrField = getAttributeField(oldCfg.getSchema(), name);
+        Schema::DataType attrType(attrField.getDataType());
+        bool inNewAttrMgr = newCfg.getAttrMgr()->getAttribute(name)->valid();
+        bool inNewSchema = newCfg.getInspector()->hasField(name);
+        // NOTE: If it is a string and index field we shall
+        // keep the original in order to preserve annotations.
+        bool isStringIndexField = attrField.getDataType() == Schema::STRING &&
+                newCfg.getSchema().isIndexField(name);
+        bool populateField = !inNewAttrMgr && inNewSchema && !isStringIndexField &&
+                             fastPartialUpdateAttribute(attrType);
+        LOG(debug, "getFieldsToPopulate(): name='%s', inNewAttrMgr=%s, inNewSchema=%s, "
+                "isStringIndexField=%s, dataType=%s, populate=%s",
+                name.c_str(), toStr(inNewAttrMgr), toStr(inNewSchema),
+            toStr(isStringIndexField),
+            Schema::getTypeName(attrType).c_str(),
+            toStr(populateField));
+        if (populateField) {
+            fieldsToPopulate.push_back(IReprocessingRewriter::SP
+                    (new DocumentFieldPopulator(attrField,
+                            guard.getSP(), subDbName)));
+        }
+    }
+    return fieldsToPopulate;
+}
+
+}
+
+AttributeReprocessingInitializer::
+AttributeReprocessingInitializer(const Config &newCfg,
+                                 const Config &oldCfg,
+                                 const vespalib::string &subDbName)
+    : _attrsToPopulate(getAttributesToPopulate(newCfg, oldCfg, subDbName)),
+      _fieldsToPopulate(getFieldsToPopulate(newCfg, oldCfg, subDbName))
+{
+}
+
+bool
+AttributeReprocessingInitializer::hasReprocessors() const
+{
+    return _attrsToPopulate.get() != nullptr || !_fieldsToPopulate.empty();
+}
+
+void
+AttributeReprocessingInitializer::initialize(IReprocessingHandler &handler)
+{
+    if (_attrsToPopulate.get() != nullptr) {
+        handler.addReader(_attrsToPopulate);
+    }
+    if (!_fieldsToPopulate.empty()) {
+        for (const auto &rewriter : _fieldsToPopulate) {
+            handler.addRewriter(rewriter);
+        }
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h
new file mode 100644
index 00000000000..f1837a6dbcb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_reprocessing_handler.h"
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include <vespa/searchcore/proton/common/i_document_type_inspector.h>
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_initializer.h>
+#include <vespa/searchlib/common/serialnum.h>
+
+namespace proton {
+
+/**
+ * Class responsible for initialize reprocessing of attribute vectors if needed.
+ */
+class AttributeReprocessingInitializer : public IReprocessingInitializer
+{
+public:
+    typedef std::unique_ptr<AttributeReprocessingInitializer> UP;
+
+    class Config
+    {
+    private:
+        proton::IAttributeManager::SP _attrMgr;
+        const search::index::Schema &_schema;
+        IDocumentTypeInspector::SP _inspector;
+    public:
+        Config(const proton::IAttributeManager::SP &attrMgr,
+               const search::index::Schema &schema,
+               const IDocumentTypeInspector::SP &inspector)
+            : _attrMgr(attrMgr),
+              _schema(schema),
+              _inspector(inspector)
+        {
+        }
+        const proton::IAttributeManager::SP &getAttrMgr() const { return _attrMgr; }
+        const search::index::Schema &getSchema() const { return _schema; }
+        const IDocumentTypeInspector::SP &getInspector() const { return _inspector; }
+    };
+
+private:
+    IReprocessingReader::SP                _attrsToPopulate;
+    std::vector<IReprocessingRewriter::SP> _fieldsToPopulate;
+
+public:
+    AttributeReprocessingInitializer(const Config &newCfg,
+                                     const Config &oldCfg,
+                                     const vespalib::string &subDbName);
+
+    // Implements IReprocessingInitializer
+    virtual bool hasReprocessors() const;
+
+    virtual void initialize(IReprocessingHandler &handler);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.cpp
new file mode 100644
index 00000000000..4da9bdf76d9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.cpp
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.reprocessing.document_reprocessing_handler");
+
+#include "document_reprocessing_handler.h"
+
+namespace proton {
+
+void
+DocumentReprocessingHandler::rewriteVisit(uint32_t lid, document::Document &doc)
+{
+    if (lid == 0 || lid >= _docIdLimit)
+        return;
+    for (const auto &reader : _readers) {
+        reader->handleExisting(lid, doc);
+    }
+    for (const auto &rewriter : _rewriters) {
+        rewriter->handleExisting(lid, doc);
+    }
+}
+
+DocumentReprocessingHandler::DocumentReprocessingHandler(uint32_t docIdLimit)
+    : _readers(),
+      _rewriters(),
+      _rewriteVisitor(*this),
+      _docIdLimit(docIdLimit)
+{
+}
+
+void
+DocumentReprocessingHandler::visit(uint32_t lid, const document::Document &doc)
+{
+    if (lid == 0 || lid >= _docIdLimit)
+        return;
+    for (const auto &reader : _readers) {
+        reader->handleExisting(lid, doc);
+    }
+}
+
+void
+DocumentReprocessingHandler::visit(uint32_t lid)
+{
+    (void) lid;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h b/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h
new file mode 100644
index 00000000000..32e472beaa0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_reprocessing_handler.h"
+#include <vespa/searchlib/docstore/idocumentstore.h>
+
+namespace proton {
+
+/**
+ * Class that is a visitor over a document store and proxies all documents
+ * to the registered readers and rewriters upon visiting.
+ */
+class DocumentReprocessingHandler : public IReprocessingHandler,
+                                    public search::IDocumentStoreReadVisitor
+{
+private:
+    typedef std::vector<IReprocessingReader::SP> ReaderVector;
+    typedef std::vector<IReprocessingRewriter::SP> RewriterVector;
+
+    class RewriteVisitor : public search::IDocumentStoreRewriteVisitor
+    {
+    private:
+        DocumentReprocessingHandler &_handler;
+    public:
+        RewriteVisitor(DocumentReprocessingHandler &handler) : _handler(handler) {}
+        // Implements search::IDocumentStoreRewriteVisitor
+        virtual void visit(uint32_t lid, document::Document &doc) {
+            _handler.rewriteVisit(lid, doc);
+        }
+    };
+
+    ReaderVector _readers;
+    RewriterVector _rewriters;
+    RewriteVisitor _rewriteVisitor;
+    uint32_t       _docIdLimit;
+
+    void rewriteVisit(uint32_t lid, document::Document &doc);
+
+public:
+    DocumentReprocessingHandler(uint32_t docIdLimit);
+
+    bool hasReaders() const {
+        return !_readers.empty();
+    }
+
+    bool hasRewriters() const {
+        return !_rewriters.empty();
+    }
+
+    bool hasProcessors() const {
+        return hasReaders() || hasRewriters();
+    }
+
+    search::IDocumentStoreRewriteVisitor &getRewriteVisitor() {
+        return _rewriteVisitor;
+    }
+
+    // Implements IReprocessingHandler
+    virtual void addReader(const IReprocessingReader::SP &reader) {
+        _readers.push_back(reader);
+    }
+
+    virtual void addRewriter(const IReprocessingRewriter::SP &rewriter) {
+        _rewriters.push_back(rewriter);
+    }
+
+    // Implements search::IDocumentStoreReadVisitor
+    virtual void visit(uint32_t lid, const document::Document &doc);
+
+    virtual void visit(uint32_t lid);
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h
new file mode 100644
index 00000000000..ead82ba1606
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_reprocessing_reader.h"
+#include "i_reprocessing_rewriter.h"
+
+namespace proton {
+
+/**
+ * Interface for a handler of a particular reprocessing job.
+ * Readers and rewriters are registered to this handler to receive all documents being reprocessed.
+ */
+struct IReprocessingHandler
+{
+    virtual ~IReprocessingHandler() {}
+
+    /**
+     * Adds the given processing reader to this handler.
+     */
+    virtual void addReader(const IReprocessingReader::SP &reader) = 0;
+
+    /**
+     * Adds the given processing rewriter to this handler.
+     */
+    virtual void addRewriter(const IReprocessingRewriter::SP &rewriter) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_initializer.h b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_initializer.h
new file mode 100644
index 00000000000..9426e677b3f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_initializer.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_reprocessing_handler.h"
+
+namespace proton {
+
+/**
+ * Interface for an initializer of a reprocessing handler.
+ */
+struct IReprocessingInitializer
+{
+    typedef std::unique_ptr<IReprocessingInitializer> UP;
+
+    virtual ~IReprocessingInitializer() {}
+
+    /**
+     * Returns whether this initializer has any reprocessors to add to the handler.
+     */
+    virtual bool hasReprocessors() const = 0;
+
+    /**
+     * Initialize the given handler by adding processing readers and/or rewriters.
+     */
+    virtual void initialize(IReprocessingHandler &handler) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_reader.h b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_reader.h
new file mode 100644
index 00000000000..e265befe257
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_reader.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+
+namespace proton {
+
+/**
+ * Interface for a reprocessor that handles a set of documents
+ * to update some other components based on the content of those documents.
+ */
+struct IReprocessingReader
+{
+    typedef std::shared_ptr<IReprocessingReader> SP;
+
+    virtual ~IReprocessingReader() {}
+
+    /**
+     * Handle the given existing document.
+     */
+    virtual void handleExisting(uint32_t lid, const document::Document &doc) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_rewriter.h b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_rewriter.h
new file mode 100644
index 00000000000..d558cec7b50
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_rewriter.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+
+namespace proton {
+
+/**
+ * Interface for a reprocessor that rewrites a set of documents
+ * based on the content of other underlying components.
+ */
+struct IReprocessingRewriter
+{
+    typedef std::shared_ptr<IReprocessingRewriter> SP;
+
+    virtual ~IReprocessingRewriter() {}
+
+    /**
+     * Handle and rewrite the given existing document.
+     */
+    virtual void handleExisting(uint32_t lid, document::Document &doc) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_task.h b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_task.h
new file mode 100644
index 00000000000..5f695fa87f1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_task.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton {
+
+/**
+ * Interface class for reprocessing task, subclassed by specific
+ * task.
+ */
+class IReprocessingTask
+{
+public:
+    typedef std::shared_ptr<IReprocessingTask> SP;
+    typedef std::unique_ptr<IReprocessingTask> UP;
+    typedef std::vector<SP> List;
+
+    struct Progress
+    {
+        double _progress;
+        double _weight;
+
+        Progress()
+            : _progress(0.0),
+              _weight(0.0)
+        {
+        }
+
+        Progress(double progress, double weight)
+            : _progress(progress),
+              _weight(weight)
+        {
+        }
+    };
+
+    virtual
+    ~IReprocessingTask()
+    {
+    }
+
+    /**
+     * Run reprocessing task.
+     */
+    virtual void
+    run() = 0;
+
+    virtual Progress
+    getProgress() const = 0;
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp
new file mode 100644
index 00000000000..9084c3d8e63
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "reprocess_documents_task.h"
+#include "attribute_reprocessing_initializer.h"
+#include "document_reprocessing_handler.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+
+namespace proton
+{
+
+
+ReprocessDocumentsTask::
+ReprocessDocumentsTask(IReprocessingInitializer &initializer,
+                       const proton::ISummaryManager::SP &sm,
+                       const document::DocumentTypeRepo::SP &docTypeRepo,
+                       const vespalib::string &subDbName,
+                       uint32_t docIdLimit)
+    : _sm(sm),
+      _docTypeRepo(docTypeRepo),
+      _subDbName(subDbName),
+      _visitorProgress(0.0),
+      _visitorCost(0.0),
+      _handler(docIdLimit),
+      _startTime(0),
+      _loggedProgress(0.0),
+      _loggedTime(0)
+{
+    initializer.initialize(_handler);
+    if (_handler.hasProcessors()) {
+        search::IDocumentStore &docstore = _sm->getBackingStore();
+        _visitorCost = docstore.getVisitCost();
+    }
+}
+
+
+void
+ReprocessDocumentsTask::run()
+{
+    if (_handler.hasProcessors()) {
+        EventLogger::reprocessDocumentsStart(_subDbName,
+                                             _visitorCost);
+        fastos::TimeStamp ts(fastos::ClockSystem::now());
+        _startTime = ts.ms();
+        _loggedTime = _startTime;
+        search::IDocumentStore &docstore = _sm->getBackingStore();
+        if (_handler.hasRewriters()) {
+            docstore.accept(_handler.getRewriteVisitor(),
+                    *this,
+                    *_docTypeRepo);
+        } else {
+            docstore.accept(_handler,
+                    *this,
+                    *_docTypeRepo);
+        }
+        ts = fastos::ClockSystem::now();
+        int64_t elapsedTime = ts.ms() - _startTime;
+        EventLogger::reprocessDocumentsComplete(_subDbName,
+                                                _visitorCost,
+                                                elapsedTime);
+    }
+}
+
+
+void
+ReprocessDocumentsTask::updateProgress(double progress)
+{
+    _visitorProgress = progress;
+    double deltaProgress = progress - _loggedProgress;
+    if (deltaProgress >= 0.01) {
+        fastos::TimeStamp ts = fastos::ClockSystem::now();
+        int64_t logDelayTime = ts.ms() - _loggedTime;
+        if (logDelayTime >= 60000 || deltaProgress >= 0.10) {
+            EventLogger::reprocessDocumentsProgress(_subDbName,
+                                                    progress,
+                                                    _visitorCost);
+            _loggedTime = ts.ms();
+            _loggedProgress = progress;
+        }
+    }
+}
+
+
+IReprocessingTask::Progress
+ReprocessDocumentsTask::getProgress() const
+{
+    return Progress(_visitorProgress, _visitorCost);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.h b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.h
new file mode 100644
index 00000000000..013f9e9f004
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_reprocessing_task.h"
+#include <vespa/searchcore/proton/docsummary/isummarymanager.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include "attribute_reprocessing_initializer.h"
+#include "document_reprocessing_handler.h"
+#include "i_reprocessing_initializer.h"
+
+namespace proton
+{
+
+/**
+ * The reprocessing documents task handles reprocessing of documents,
+ * e.g. populate attributes from document store when adding attribute
+ * aspect on existing field and populating documents in document store
+ * when removing attribute aspect on existing field.
+ */
+class ReprocessDocumentsTask : public IReprocessingTask,
+                               public search::IDocumentStoreVisitorProgress
+{
+    proton::ISummaryManager::SP          _sm;
+    document::DocumentTypeRepo::SP       _docTypeRepo;
+    vespalib::string                     _subDbName;
+    double                               _visitorProgress;
+    double                               _visitorCost;
+    DocumentReprocessingHandler          _handler;
+    int64_t                              _startTime;
+    double                               _loggedProgress;
+    int64_t                              _loggedTime;
+
+public:
+    ReprocessDocumentsTask(IReprocessingInitializer &initializer,
+                           const proton::ISummaryManager::SP &sm,
+                           const document::DocumentTypeRepo::SP &docTypeRepo,
+                           const vespalib::string &subDbName,
+                           uint32_t docIdLimit);
+
+    virtual void
+    run();
+
+    virtual void
+    updateProgress(double progress);
+
+    virtual Progress
+    getProgress() const; 
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.cpp
new file mode 100644
index 00000000000..4891829bbea
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.cpp
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <mutex>
+#include "reprocessingrunner.h"
+#include "i_reprocessing_task.h"
+
+namespace proton
+{
+
+
+ReprocessingRunner::ReprocessingRunner()
+    : _lock(),
+      _tasks(),
+      _state(NOT_STARTED)
+{
+}
+
+
+void
+ReprocessingRunner::addTasks(const ReprocessingTasks &tasks)
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    for (auto task : tasks) {
+        _tasks.push_back(task);
+    }
+}
+
+
+void
+ReprocessingRunner::run()
+{
+    {
+        std::lock_guard<std::mutex> guard(_lock);
+        _state = RUNNING;
+    }
+    for (auto &task : _tasks) {
+        task->run();
+    }
+    std::lock_guard<std::mutex> guard(_lock);
+    _tasks.clear();
+    _state = DONE;
+    
+}
+
+
+void
+ReprocessingRunner::reset()
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    _tasks.clear();
+    _state = NOT_STARTED;
+}
+
+
+bool
+ReprocessingRunner::empty() const
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    return _tasks.empty();
+}
+
+
+double
+ReprocessingRunner::getProgress() const
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    switch (_state) {
+    case State::NOT_STARTED:
+        return 0.0;
+    case State::DONE:
+        return 1.0;
+    default:
+        ;
+    }
+    double weightedProgress = 0.0;
+    double weight = 0.0;
+    for (auto task : _tasks) {
+        IReprocessingTask::Progress progress = task->getProgress();
+        weightedProgress += progress._progress * progress._weight;
+        weight += progress._weight;
+    }
+    if (weight == 0.0)
+        return 1.0;
+    return weightedProgress / weight;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.h b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.h
new file mode 100644
index 00000000000..e9f4a3f31b7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.h
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <mutex>
+
+namespace proton
+{
+
+class IReprocessingTask;
+
+/**
+ * Class for running reprocessing task.
+ */
+class ReprocessingRunner
+{
+public:
+    typedef std::vector<std::shared_ptr<IReprocessingTask>> ReprocessingTasks;
+private:
+    mutable std::mutex _lock;
+    ReprocessingTasks _tasks; // Protected by _lock
+    enum State
+    {
+        NOT_STARTED,
+        RUNNING,
+        DONE
+    };
+    State _state;
+public:
+    ReprocessingRunner();
+
+    void
+    addTasks(const ReprocessingTasks &tasks);
+
+    void
+    run();
+
+    void
+    reset();
+
+    bool
+    empty() const;
+
+    double
+    getProgress() const;
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/.gitignore b/searchcore/src/vespa/searchcore/proton/server/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
new file mode 100644
index 00000000000..e0f885a8c6f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
@@ -0,0 +1,116 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_server STATIC
+    SOURCES
+    attribute_config_validator.cpp
+    bootstrapconfig.cpp
+    bootstrapconfigmanager.cpp
+    buckethandler.cpp
+    bucketmovejob.cpp
+    clusterstatehandler.cpp
+    combiningfeedview.cpp
+    configvalidator.cpp
+    data_directory_upgrader.cpp
+    ddbstate.cpp
+    disk_mem_usage_filter.cpp
+    disk_mem_usage_sampler.cpp
+    docstorevalidator.cpp
+    document_db_explorer.cpp
+    document_db_maintenance_config.cpp
+    document_scan_iterator.cpp
+    document_subdb_collection_explorer.cpp
+    document_subdb_collection_initializer.cpp
+    document_subdb_explorer.cpp
+    document_subdb_initializer.cpp
+    document_subdb_initializer_result.cpp
+    documentbucketmover.cpp
+    documentdb.cpp
+    documentdb_commit_job.cpp
+    documentdbconfig.cpp
+    documentdbconfigscout.cpp
+    documentdbconfigmanager.cpp
+    documentretriever.cpp
+    documentretrieverbase.cpp
+    documentsubdbcollection.cpp
+    emptysearchview.cpp
+    executor_thread_service.cpp
+    executorthreadingservice.cpp
+    fast_access_doc_subdb.cpp
+    fast_access_doc_subdb_configurer.cpp
+    fast_access_feed_view.cpp
+    feedhandler.cpp
+    feedstate.cpp
+    feedstates.cpp
+    fileconfigmanager.cpp
+    flushhandlerproxy.cpp
+    forcecommitcontext.cpp
+    forcecommitdonetask.cpp
+    frozenbuckets.cpp
+    health_adapter.cpp
+    heart_beat_job.cpp
+    idocumentdbowner.cpp
+    ireplayconfig.cpp
+    job_tracked_maintenance_job.cpp
+    lid_space_compaction_handler.cpp
+    lid_space_compaction_job.cpp
+    maintenance_controller_explorer.cpp
+    maintenance_jobs_injector.cpp
+    maintenancecontroller.cpp
+    maintenancejobrunner.cpp
+    matchers.cpp
+    matchhandlerproxy.cpp
+    matchview.cpp
+    memoryflush.cpp
+    minimal_document_retriever.cpp
+    ooscli.cpp
+    operationdonecontext.cpp
+    persistencehandlerproxy.cpp
+    persistenceproviderproxy.cpp
+    proton.cpp
+    protonconfigurer.cpp
+    prune_session_cache_job.cpp
+    pruneremoveddocumentsjob.cpp
+    putdonecontext.cpp
+    reconfig_params.cpp
+    removedonecontext.cpp
+    removedonetask.cpp
+    replaypacketdispatcher.cpp
+    resource_usage_explorer.cpp
+    rpc_hooks.cpp
+    sample_attribute_usage_job.cpp
+    schema_config_validator.cpp
+    searchable_doc_subdb_configurer.cpp
+    searchable_document_retriever.cpp
+    searchable_feed_view.cpp
+    searchabledocsubdb.cpp
+    searchcontext.cpp
+    searchhandlerproxy.cpp
+    searchview.cpp
+    simpleflush.cpp
+    storeonlydocsubdb.cpp
+    storeonlyfeedview.cpp
+    summaryadapter.cpp
+    tlcproxy.cpp
+    tlssyncer.cpp
+    transactionlogmanager.cpp
+    transactionlogmanagerbase.cpp
+    updatedonecontext.cpp
+    visibilityhandler.cpp
+    wipe_old_removed_fields_job.cpp
+    DEPENDS
+    searchcore_attribute
+    searchcore_bucketdb
+    searchcore_docsummary
+    searchcore_documentmetastore
+    searchcore_flushengine
+    searchcore_index
+    searchcore_initializer
+    searchcore_matchengine
+    searchcore_matching
+    searchcore_persistenceengine
+    searchcore_reprocessing
+    searchcore_summaryengine
+    searchcore_fconfig
+    configdefinitions
+)
+vespa_add_target_system_dependency(searchcore_server boost boost_system-mt-d)
+vespa_add_target_system_dependency(searchcore_server boost boost_filesystem-mt-d)
diff --git a/searchcore/src/vespa/searchcore/proton/server/OWNERS b/searchcore/src/vespa/searchcore/proton/server/OWNERS
new file mode 100644
index 00000000000..7c03446a5d4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/OWNERS
@@ -0,0 +1,2 @@
+geirst
+tegge
diff --git a/searchcore/src/vespa/searchcore/proton/server/attribute_config_validator.cpp b/searchcore/src/vespa/searchcore/proton/server/attribute_config_validator.cpp
new file mode 100644
index 00000000000..666a40cac9c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/attribute_config_validator.cpp
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.attribute_config_validator");
+#include "attribute_config_validator.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using vespa::config::search::AttributesConfig;
+using vespalib::make_string;
+
+namespace proton {
+
+typedef ConfigValidator CV;
+
+namespace {
+
+CV::Result
+checkFastAccess(const AttributesConfig &cfg1,
+                const AttributesConfig &cfg2,
+                CV::ResultType type,
+                const vespalib::string &typeStr)
+{
+    for (const auto &attr1 : cfg1.attribute) {
+        if (attr1.fastaccess) {
+            for (const auto &attr2 : cfg2.attribute) {
+                if (attr1.name == attr2.name && !attr2.fastaccess) {
+                    return CV::Result(type,
+                            make_string("Trying to %s 'fast-access' to attribute '%s'",
+                                    typeStr.c_str(), attr1.name.c_str()));
+                }
+            }
+        }
+    }
+    return CV::Result();
+}
+
+CV::Result
+checkFastAccessAdded(const AttributesConfig &newCfg,
+                     const AttributesConfig &oldCfg)
+{
+    return checkFastAccess(newCfg, oldCfg, CV::ATTRIBUTE_FAST_ACCESS_ADDED, "add");
+}
+
+CV::Result
+checkFastAccessRemoved(const AttributesConfig &newCfg,
+                       const AttributesConfig &oldCfg)
+{
+    return checkFastAccess(oldCfg, newCfg, CV::ATTRIBUTE_FAST_ACCESS_REMOVED, "remove");
+}
+
+}
+
+CV::Result
+AttributeConfigValidator::validate(const AttributesConfig &newCfg,
+                                   const AttributesConfig &oldCfg)
+{
+    CV::Result res;
+    if (!(res = checkFastAccessAdded(newCfg, oldCfg)).ok()) return res;
+    if (!(res = checkFastAccessRemoved(newCfg, oldCfg)).ok()) return res;
+    return CV::Result();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/attribute_config_validator.h b/searchcore/src/vespa/searchcore/proton/server/attribute_config_validator.h
new file mode 100644
index 00000000000..dd4ccbc51e8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/attribute_config_validator.h
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "configvalidator.h"
+
+namespace proton {
+
+/**
+ * Class used to validate new attribute config before starting using it.
+ **/
+struct AttributeConfigValidator
+{
+    static ConfigValidator::Result
+    validate(const vespa::config::search::AttributesConfig &newCfg,
+             const vespa::config::search::AttributesConfig &oldCfg);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/attributeadapterfactory.h b/searchcore/src/vespa/searchcore/proton/server/attributeadapterfactory.h
new file mode 100644
index 00000000000..6ba0ec5344f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/attributeadapterfactory.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "iattributeadapterfactory.h"
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+
+namespace proton {
+
+struct AttributeAdapterFactory : public IAttributeAdapterFactory
+{
+    AttributeAdapterFactory()
+    { }
+    virtual IAttributeWriter::SP create(const IAttributeWriter::SP &old,
+                                        const AttributeCollectionSpec &attrSpec) const
+    {
+        const AttributeWriter &oldAdapter = dynamic_cast<const AttributeWriter &>(*old.get());
+        const proton::IAttributeManager::SP &oldMgr = oldAdapter.getAttributeManager();
+        proton::IAttributeManager::SP newMgr = oldMgr->create(attrSpec);
+        return IAttributeWriter::SP(new AttributeWriter(newMgr));
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp
new file mode 100644
index 00000000000..f4b5757f558
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "bootstrapconfig.h"
+
+using namespace vespa::config::search;
+using namespace config;
+using document::DocumentTypeRepo;
+using search::TuneFileDocumentDB;
+using vespa::config::search::core::ProtonConfig;
+using document::DocumenttypesConfig;
+
+namespace {
+    template <typename T>
+    bool equals(const T * lhs, const T * rhs)
+    {
+        if (lhs == NULL)
+            return rhs == NULL;
+        return rhs != NULL && *lhs == *rhs;
+    }
+}
+
+namespace proton {
+
+BootstrapConfig::BootstrapConfig(
+               int64_t generation,
+               const DocumenttypesConfigSP &documenttypes,
+               const DocumentTypeRepo::SP &repo,
+               const ProtonConfigSP &protonConfig,
+               const search::TuneFileDocumentDB::SP &tuneFileDocumentDB)
+    : _documenttypes(documenttypes),
+      _repo(repo),
+      _proton(protonConfig),
+      _tuneFileDocumentDB(tuneFileDocumentDB),
+      _generation(generation)
+{
+}
+
+bool
+BootstrapConfig::operator==(const BootstrapConfig &rhs) const
+{
+    return equals<DocumenttypesConfig>(_documenttypes.get(),
+                                       rhs._documenttypes.get()) &&
+        _repo.get() == rhs._repo.get() &&
+        equals<ProtonConfig>(_proton.get(), rhs._proton.get()) &&
+        equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(),
+                                   rhs._tuneFileDocumentDB.get());
+}
+
+
+bool
+BootstrapConfig::valid(void) const
+{
+    return _documenttypes.get() != NULL &&
+                    _repo.get() != NULL &&
+                  _proton.get() != NULL &&
+      _tuneFileDocumentDB.get() != NULL;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
new file mode 100644
index 00000000000..a222c51920a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentdbconfig.h"
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/config/config-proton.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/config/retriever/configkeyset.h>
+#include <vespa/config/retriever/configsnapshot.h>
+
+namespace proton {
+
+/**
+ * This class represents all config classes required by proton to bootstrap.
+ */
+class BootstrapConfig
+{
+public:
+    typedef std::shared_ptr<BootstrapConfig> SP;
+    typedef std::shared_ptr<vespa::config::search::core::ProtonConfig> ProtonConfigSP;
+    typedef DocumentDBConfig::DocumenttypesConfigSP DocumenttypesConfigSP;
+
+private:
+    DocumenttypesConfigSP _documenttypes;
+    document::DocumentTypeRepo::SP _repo;
+    ProtonConfigSP _proton;
+    search::TuneFileDocumentDB::SP _tuneFileDocumentDB;
+    int64_t _generation;
+
+public:
+    BootstrapConfig(int64_t generation,
+                    const DocumenttypesConfigSP & documenttypes,
+                    const document::DocumentTypeRepo::SP &repo,
+                    const ProtonConfigSP &protonConfig,
+                    const search::TuneFileDocumentDB::SP &
+                    _tuneFileDocumentDB);
+
+    const document::DocumenttypesConfig &
+    getDocumenttypesConfig(void) const
+    {
+        return *_documenttypes;
+    }
+
+    const DocumenttypesConfigSP &
+    getDocumenttypesConfigSP(void) const
+    {
+        return _documenttypes;
+    }
+
+    const document::DocumentTypeRepo::SP &
+    getDocumentTypeRepoSP() const
+    {
+        return _repo;
+    }
+
+    const vespa::config::search::core::ProtonConfig &
+    getProtonConfig(void) const
+    {
+        return *_proton;
+    }
+
+    const ProtonConfigSP &
+    getProtonConfigSP(void) const
+    {
+        return _proton;
+    }
+
+    const search::TuneFileDocumentDB::SP &
+    getTuneFileDocumentDBSP(void) const
+    {
+        return _tuneFileDocumentDB;
+    }
+
+    int64_t
+    getGeneration() const
+    {
+        return _generation;
+    }
+
+    void
+    setGeneration(int64_t generation)
+    {
+        _generation = generation;
+    }
+
+    /**
+     * Shared pointers are checked for identity, not equality.
+     */
+    bool
+    operator==(const BootstrapConfig &rhs) const;
+
+    bool
+    valid(void) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
new file mode 100644
index 00000000000..555e2f2797f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "bootstrapconfigmanager.h"
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.bootstrapconfigmanager");
+
+using namespace vespa::config::search;
+using namespace config;
+using document::DocumentTypeRepo;
+using search::TuneFileDocumentDB;
+using vespa::config::search::core::ProtonConfig;
+using document::DocumenttypesConfig;
+using config::SourceSpec;
+
+namespace proton
+{
+
+BootstrapConfigManager::
+BootstrapConfigManager(const vespalib::string & configId)
+    : _pendingConfigSnapshot(),
+      _configId(configId),
+      _pendingConfigLock()
+{
+}
+
+
+const ConfigKeySet
+BootstrapConfigManager::createConfigKeySet() const
+{
+    return ConfigKeySet().add<ProtonConfig, DocumenttypesConfig>(_configId);
+}
+
+void
+BootstrapConfigManager::update(const ConfigSnapshot & snapshot)
+{
+    typedef BootstrapConfig::ProtonConfigSP ProtonConfigSP;
+    typedef BootstrapConfig::DocumenttypesConfigSP DocumenttypesConfigSP;
+
+    ProtonConfigSP newProtonConfig;
+    TuneFileDocumentDB::SP newTuneFileDocumentDB;
+    DocumenttypesConfigSP newDocumenttypesConfig;
+    DocumentTypeRepo::SP newRepo;
+    int64_t currentGen = -1;
+
+    BootstrapConfig::SP current = _pendingConfigSnapshot;
+    if (current.get() != NULL) {
+        newProtonConfig = current->getProtonConfigSP();
+        newTuneFileDocumentDB = current->getTuneFileDocumentDBSP();
+        newDocumenttypesConfig = current->getDocumenttypesConfigSP();
+        newRepo = current->getDocumentTypeRepoSP();
+        currentGen = current->getGeneration();
+    }
+
+    if (snapshot.isChanged<ProtonConfig>(_configId, currentGen)) {
+        LOG(spam, "Proton config is changed");
+        std::unique_ptr<ProtonConfig> protonConfig =
+            snapshot.getConfig<ProtonConfig>(_configId);
+        TuneFileDocumentDB::SP tuneFileDocumentDB(new TuneFileDocumentDB);
+        TuneFileDocumentDB &tune = *tuneFileDocumentDB;
+        ProtonConfig &conf = *protonConfig;
+        tune._index._indexing._write.
+            setFromConfig<ProtonConfig::Indexing::Write>(
+                    conf.indexing.write.io);
+        tune._index._indexing._read.
+            setFromConfig<ProtonConfig::Indexing::Read>(conf.indexing.read.io);
+        tune._attr._write.
+            setFromConfig<ProtonConfig::Attribute::Write>(
+                    conf.attribute.write.io);
+        tune._index._search._read.
+            setFromConfig<ProtonConfig::Search, ProtonConfig::Search::Mmap>(conf.search.io, conf.search.mmap);
+        tune._summary._write.
+            setFromConfig<ProtonConfig::Summary::Write>(conf.summary.write.io);
+        tune._summary._seqRead.
+            setFromConfig<ProtonConfig::Summary::Read>(conf.summary.read.io);
+        tune._summary._randRead.
+            setFromConfig<ProtonConfig::Summary::Read, ProtonConfig::Summary::Read::Mmap>(conf.summary.read.io, conf.summary.read.mmap);
+
+        newProtonConfig = ProtonConfigSP(protonConfig.release());
+        newTuneFileDocumentDB = tuneFileDocumentDB;
+    }
+
+    if (snapshot.isChanged<DocumenttypesConfig>(_configId, currentGen)) {
+        LOG(spam, "Documenttypes config is changed");
+        std::unique_ptr<DocumenttypesConfig> documenttypesConfig =
+            snapshot.getConfig<DocumenttypesConfig>(_configId);
+        DocumentTypeRepo::SP repo(new DocumentTypeRepo(*documenttypesConfig));
+        newDocumenttypesConfig =
+            DocumenttypesConfigSP(documenttypesConfig.release());
+        newRepo = repo;
+    }
+    assert(newProtonConfig.get() != NULL);
+    assert(newTuneFileDocumentDB.get() != NULL);
+    assert(newDocumenttypesConfig.get() != NULL);
+    assert(newRepo.get() != NULL);
+
+    BootstrapConfig::SP newSnapshot(
+            new BootstrapConfig(snapshot.getGeneration(),
+                                newDocumenttypesConfig,
+                                newRepo,
+                                newProtonConfig,
+                                newTuneFileDocumentDB));
+
+    assert(newSnapshot->valid());
+    {
+        vespalib::LockGuard lock(_pendingConfigLock);
+        _pendingConfigSnapshot = newSnapshot;
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h
new file mode 100644
index 00000000000..e9e014d8e38
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/config/config.h>
+#include <vespa/config/retriever/configsnapshot.h>
+#include <vespa/config/retriever/configkeyset.h>
+#include "bootstrapconfig.h"
+
+namespace proton {
+
+/**
+ * This class handles subscribing to the proton bootstrap config.
+ */
+class BootstrapConfigManager
+{
+public:
+    BootstrapConfigManager(const vespalib::string & configId);
+    const config::ConfigKeySet createConfigKeySet() const;
+
+    BootstrapConfig::SP
+    getConfig(void) const
+    {
+        vespalib::LockGuard lock(_pendingConfigLock);
+        return _pendingConfigSnapshot;
+    }
+    void update(const config::ConfigSnapshot & snapshot);
+
+private:
+    BootstrapConfig::SP _pendingConfigSnapshot;
+    vespalib::string _configId;
+    vespalib::Lock _pendingConfigLock;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp
new file mode 100644
index 00000000000..9382c9aa0df
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp
@@ -0,0 +1,198 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.buckethandler");
+#include "buckethandler.h"
+#include <vespa/vespalib/util/closuretask.h>
+#include "ibucketstatechangedhandler.h"
+
+using document::BucketId;
+using storage::spi::Bucket;
+using storage::spi::BucketChecksum;
+using storage::spi::BucketIdListResult;
+using storage::spi::BucketInfo;
+using storage::spi::BucketInfoResult;
+using storage::spi::Result;
+using vespalib::Executor;
+using vespalib::makeTask;
+using vespalib::makeClosure;
+
+namespace proton {
+
+namespace {
+
+BucketInfo
+fakeSizeData(const BucketInfo &info)
+{
+    // TODO(geirst): consider if we should store document size and used size in bucket db.
+    return BucketInfo(info.getChecksum(),
+                      info.getDocumentCount(),
+                      info.getDocumentCount(),
+                      info.getEntryCount(),
+                      info.getEntryCount(),
+                      info.getReady(),
+                      info.getActive());
+}
+
+}
+
+void
+BucketHandler::performSetCurrentState(BucketId bucketId,
+                                      storage::spi::BucketInfo::ActiveState newState,
+                                      IGenericResultHandler *resultHandler)
+{
+    if (!_nodeUp) {
+        Result result(Result::TRANSIENT_ERROR,
+                      "Cannot set bucket active state when node is down");
+        resultHandler->handle(result);
+        return;
+    }
+    bool active = (newState == storage::spi::BucketInfo::ACTIVE);
+    LOG(debug, "performSetCurrentState(%s, %s)",
+        bucketId.toString().c_str(), (active ? "ACTIVE" : "NOT_ACTIVE"));
+    _ready->setBucketState(bucketId, active);
+    if (!_changedHandlers.empty()) {
+        typedef std::vector<IBucketStateChangedHandler *> Chv;
+        Chv &chs(_changedHandlers);
+        for (Chv::const_iterator itr = chs.begin(); itr != chs.end(); ++itr) {
+            (*itr)->notifyBucketStateChanged(bucketId, newState);
+        }
+    }
+    resultHandler->handle(Result());
+}
+
+void
+BucketHandler::performPopulateActiveBuckets(document::BucketId::List buckets,
+                                            IGenericResultHandler *resultHandler)
+{
+    _ready->populateActiveBuckets(buckets);
+    resultHandler->handle(Result());
+}
+
+void
+BucketHandler::deactivateAllActiveBuckets()
+{
+    BucketId::List buckets;
+    _ready->getBucketDB().takeGuard()->getActiveBuckets(buckets);
+    for (auto bucketId : buckets) {
+        _ready->setBucketState(bucketId,
+                               storage::spi::BucketInfo::NOT_ACTIVE);
+        // Don't notify bucket state changed, node is marked down so
+        // noone is listening.
+    }
+}
+
+BucketHandler::BucketHandler(vespalib::Executor &executor)
+    : IClusterStateChangedHandler(),
+      IBucketStateChangedNotifier(),
+      _executor(executor),
+      _ready(NULL),
+      _changedHandlers(),
+      _nodeUp(false)
+{
+    LOG(spam, "BucketHandler::BucketHandler");
+}
+
+BucketHandler::~BucketHandler()
+{
+    assert(_changedHandlers.empty());
+}
+
+void
+BucketHandler::setReadyBucketHandler(documentmetastore::IBucketHandler &ready)
+{
+    _ready = &ready;
+}
+
+void
+BucketHandler::handleListBuckets(IBucketIdListResultHandler &resultHandler)
+{
+    // Called by SPI thread.
+    // BucketDBOwner ensures synchronization between SPI thread and
+    // master write thread in document database.
+    BucketIdListResult::List buckets;
+    _ready->getBucketDB().takeGuard()->getBuckets(buckets);
+    resultHandler.handle(BucketIdListResult(buckets));
+}
+
+void
+BucketHandler::handleSetCurrentState(const BucketId &bucketId,
+                                     storage::spi::BucketInfo::ActiveState newState,
+                                     IGenericResultHandler &resultHandler)
+{
+    _executor.execute(makeTask(makeClosure(this,
+                                           &proton::BucketHandler::performSetCurrentState,
+                                           bucketId, newState, &resultHandler)));
+}
+
+void
+BucketHandler::handleGetBucketInfo(const Bucket &bucket,
+                                   IBucketInfoResultHandler &resultHandler)
+{
+    // Called by SPI thread.
+    // BucketDBOwner ensures synchronization between SPI thread and
+    // master write thread in document database.
+    BucketInfo bucketInfo = fakeSizeData(_ready->getBucketDB().takeGuard()->cachedGet(bucket));
+    LOG(spam, "handleGetBucketInfo(%s): %s",
+        bucket.toString().c_str(), bucketInfo.toString().c_str());
+    resultHandler.handle(BucketInfoResult(bucketInfo));
+}
+
+void
+BucketHandler::handleListActiveBuckets(IBucketIdListResultHandler &resultHandler)
+{
+    // Called by SPI thread.
+    // BucketDBOwner ensures synchronization between SPI thread and
+    // master write thread in document database.
+    BucketIdListResult::List buckets;
+    _ready->getBucketDB().takeGuard()->getActiveBuckets(buckets);
+    resultHandler.handle(BucketIdListResult(buckets));
+}
+
+void
+BucketHandler::handlePopulateActiveBuckets(document::BucketId::List &buckets,
+                                           IGenericResultHandler &resultHandler)
+{
+    _executor.execute(makeTask(makeClosure(this,
+            &proton::BucketHandler::
+            performPopulateActiveBuckets,
+            buckets,
+            &resultHandler)));
+}
+
+void
+BucketHandler::notifyClusterStateChanged(const IBucketStateCalculator::SP &
+                                         newCalc)
+{
+    bool oldNodeUp = _nodeUp;
+    _nodeUp = newCalc->nodeUp();
+    LOG(spam,
+        "notifyClusterStateChanged: %s -> %s",
+        oldNodeUp ? "up" : "down",
+        _nodeUp ? "up" : "down");
+    if (oldNodeUp && !_nodeUp) {
+        deactivateAllActiveBuckets();
+    }
+}
+
+void
+BucketHandler::addBucketStateChangedHandler(IBucketStateChangedHandler *handler)
+{
+    _changedHandlers.push_back(handler);
+}
+
+void
+BucketHandler::
+removeBucketStateChangedHandler(IBucketStateChangedHandler *handler)
+{
+    // Called by executor thread
+    auto it = std::find(_changedHandlers.begin(), _changedHandlers.end(),
+                        handler);
+    if (it != _changedHandlers.end()) {
+        _changedHandlers.erase(it);
+    }
+
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h
new file mode 100644
index 00000000000..d1188c022f7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/documentmetastore/i_bucket_handler.h>
+#include <vespa/searchcore/proton/persistenceengine/resulthandler.h>
+#include <vespa/vespalib/util/threadstackexecutorbase.h>
+#include "iclusterstatechangedhandler.h"
+#include "ibucketstatechangednotifier.h"
+
+namespace proton {
+
+class IBucketStateChangedhandler;
+
+
+/**
+ * Class handling bucket operations in IPersistenceHandler that are not persisted
+ * in the transaction log for a document database.
+ */
+class BucketHandler : public IClusterStateChangedHandler,
+                      public IBucketStateChangedNotifier
+{
+private:
+    vespalib::Executor                       &_executor;
+    documentmetastore::IBucketHandler        *_ready;
+    std::vector<IBucketStateChangedHandler *> _changedHandlers;
+    bool                                      _nodeUp;
+
+    void performSetCurrentState(document::BucketId bucketId,
+                                storage::spi::BucketInfo::ActiveState newState,
+                                IGenericResultHandler *resultHandler);
+
+    void
+    performPopulateActiveBuckets(document::BucketId::List buckets,
+                                 IGenericResultHandler *resultHandler);
+    /**
+     * Deactivate all active buckets when this node transitions from
+     * up to down in cluster state.  Called by document db executor thread.
+     */
+    void
+    deactivateAllActiveBuckets(void);
+
+public:
+    /**
+     * Create a new bucket handler.
+     *
+     * @param executor The executor in which to run all tasks.
+     */
+    BucketHandler(vespalib::Executor &executor);
+
+    virtual
+    ~BucketHandler();
+
+    void setReadyBucketHandler(documentmetastore::IBucketHandler &ready);
+
+    /**
+     * Implements the bucket aspect of IPersistenceHandler.
+     */
+    void handleListBuckets(IBucketIdListResultHandler &resultHandler);
+    void handleSetCurrentState(const document::BucketId &bucketId,
+                               storage::spi::BucketInfo::ActiveState newState,
+                               IGenericResultHandler &resultHandler);
+    void handleGetBucketInfo(const storage::spi::Bucket &bucket,
+                             IBucketInfoResultHandler &resultHandler);
+    void handleListActiveBuckets(IBucketIdListResultHandler &resultHandler);
+    void handlePopulateActiveBuckets(document::BucketId::List &buckets,
+                                     IGenericResultHandler &resultHandler);
+
+    // Implements IClusterStateChangedHandler
+    virtual void
+    notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override;
+
+    // Implement IBucketStateChangedNotifier
+    virtual void
+    addBucketStateChangedHandler(IBucketStateChangedHandler *handler) override;
+    virtual void
+    removeBucketStateChangedHandler(IBucketStateChangedHandler *handler) override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp
new file mode 100644
index 00000000000..b000e8b3543
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp
@@ -0,0 +1,364 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.bucketmovejob");
+#include "bucketmovejob.h"
+#include "imaintenancejobrunner.h"
+#include "ibucketstatechangednotifier.h"
+#include "iclusterstatechangednotifier.h"
+#include "maintenancedocumentsubdb.h"
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+
+using document::BucketId;
+using storage::spi::BucketInfo;
+
+namespace proton {
+
+namespace {
+
+const uint32_t FIRST_SCAN_PASS = 1;
+const uint32_t SECOND_SCAN_PASS = 2;
+
+const char * bool2str(bool v) { return (v ? "T" : "F"); }
+
+}
+
+
+BucketMoveJob::ScanIterator::
+ScanIterator(BucketDBOwner::Guard db, uint32_t pass, BucketId lastBucket, BucketId endBucket)
+    : _db(std::move(db)),
+      _itr(lastBucket.isSet() ? _db->upperBound(lastBucket) : _db->begin()),
+      _end(pass == SECOND_SCAN_PASS && endBucket.isSet() ?
+           _db->upperBound(endBucket) : _db->end())
+{
+}
+
+
+BucketMoveJob::ScanIterator::
+ScanIterator(BucketDBOwner::Guard db, BucketId bucket)
+    : _db(std::move(db)),
+      _itr(_db->lowerBound(bucket)),
+      _end(_db->end())
+{
+}
+
+
+BucketMoveJob::ScanIterator::ScanIterator(ScanIterator &&rhs)
+    : _db(std::move(rhs._db)),
+      _itr(rhs._itr),
+      _end(rhs._end)
+{
+}
+
+void
+BucketMoveJob::checkBucket(const BucketId &bucket,
+                           ScanIterator &itr,
+                           DocumentBucketMover &mover,
+                           IFrozenBucketHandler::ExclusiveBucketGuard::UP & bucketGuard)
+{
+    bool hasReadyDocs = itr.hasReadyBucketDocs();
+    bool hasNotReadyDocs = itr.hasNotReadyBucketDocs();
+    if (!hasReadyDocs && !hasNotReadyDocs) {
+        return; // No documents for bucket in ready or notready subdbs
+    }
+    bool shouldBeReady = _calc->shouldBeReady(bucket);
+    bool isActive = itr.isActive();
+    bool wantReady = shouldBeReady || isActive;
+    LOG(spam, "checkBucket(): bucket(%s), shouldBeReady(%s), active(%s)",
+              bucket.toString().c_str(), bool2str(shouldBeReady), bool2str(isActive));
+    if (wantReady) {
+        if (!hasNotReadyDocs)
+            return; // No notready bucket to make ready
+    } else {
+        if (!hasReadyDocs)
+            return; // No ready bucket to make notready
+    }
+    bucketGuard = _frozenBuckets.acquireExclusiveBucket(bucket);
+    if ( ! bucketGuard ) {
+        LOG(debug, "checkBucket(): delay frozen bucket: (%s)", bucket.toString().c_str());
+        _delayedBucketsFrozen.insert(bucket);
+        _delayedBuckets.erase(bucket);
+        return;
+    }
+    const MaintenanceDocumentSubDB &source(wantReady ? _notReady : _ready);
+    const MaintenanceDocumentSubDB &target(wantReady ? _ready : _notReady);
+    LOG(debug, "checkBucket(): mover.setupForBucket(%s, source:%u, target:%u)",
+        bucket.toString().c_str(), source._subDbId, target._subDbId);
+    mover.setupForBucket(bucket, &source, target._subDbId,
+                         _moveHandler, _ready._metaStore->getBucketDB());
+}
+
+
+BucketMoveJob::ScanResult
+BucketMoveJob::scanBuckets(size_t maxBucketsToScan, IFrozenBucketHandler::ExclusiveBucketGuard::UP & bucketGuard)
+{
+    size_t bucketsScanned = 0;
+    bool passDone = false;
+    ScanIterator itr(_ready._metaStore->getBucketDB().takeGuard(),
+            _scanPass, _scanPos._lastBucket, _endPos._lastBucket);
+    BucketId bucket;
+    for (; itr.valid() &&
+             bucketsScanned < maxBucketsToScan && _mover.bucketDone();
+         ++itr, ++bucketsScanned)
+    {
+        bucket = itr.getBucket();
+        _scanPos._lastBucket = bucket;
+        checkBucket(bucket, itr, _mover, bucketGuard);
+    }
+    if (!itr.valid()) {
+        passDone = true;
+        _scanPos._lastBucket = BucketId();
+    }
+    return ScanResult(bucketsScanned, passDone);
+}
+
+
+void
+BucketMoveJob::moveDocuments(DocumentBucketMover &mover,
+                             size_t maxDocsToMove,
+                             IFrozenBucketHandler::ExclusiveBucketGuard::UP & bucketGuard)
+{
+    if ( ! bucketGuard ) {
+        bucketGuard = _frozenBuckets.acquireExclusiveBucket(mover.getBucket());
+        if (! bucketGuard) {
+            maybeDelayMover(mover, mover.getBucket());
+            return;
+        }
+    }
+    assert(mover.getBucket() == bucketGuard->getBucket());
+    mover.moveDocuments(maxDocsToMove);
+    if (mover.bucketDone()) {
+        _modifiedHandler.notifyBucketModified(mover.getBucket());
+    }
+}
+
+
+BucketMoveJob::
+BucketMoveJob(const IBucketStateCalculator::SP &calc,
+              IDocumentMoveHandler &moveHandler,
+              IBucketModifiedHandler &modifiedHandler,
+              const MaintenanceDocumentSubDB &ready,
+              const MaintenanceDocumentSubDB &notReady,
+              IFrozenBucketHandler &frozenBuckets,
+              IClusterStateChangedNotifier &clusterStateChangedNotifier,
+              IBucketStateChangedNotifier &bucketStateChangedNotifier,
+              const vespalib::string &docTypeName)
+    : IMaintenanceJob("move_buckets." + docTypeName, 0.0, 0.0),
+      IClusterStateChangedHandler(),
+      IBucketFreezeListener(),
+      _calc(calc),
+      _moveHandler(moveHandler),
+      _modifiedHandler(modifiedHandler),
+      _ready(ready),
+      _notReady(notReady),
+      _mover(),
+      _doneScan(false),
+      _scanPos(),
+      _scanPass(FIRST_SCAN_PASS),
+      _endPos(),
+      _delayedBuckets(),
+      _delayedBucketsFrozen(),
+      _frozenBuckets(frozenBuckets),
+      _delayedMover(),
+      _runner(nullptr),
+      _clusterUp(false),
+      _nodeUp(false),
+      _nodeInitializing(false),
+      _runnable(false),
+      _clusterStateChangedNotifier(clusterStateChangedNotifier),
+      _bucketStateChangedNotifier(bucketStateChangedNotifier)
+{
+    refreshDerivedClusterState();
+    
+    _frozenBuckets.addListener(this);
+    _clusterStateChangedNotifier.addClusterStateChangedHandler(this);
+    _bucketStateChangedNotifier.addBucketStateChangedHandler(this);
+}
+
+
+BucketMoveJob::~BucketMoveJob()
+{
+    _frozenBuckets.removeListener(this);
+    _clusterStateChangedNotifier.removeClusterStateChangedHandler(this);
+    _bucketStateChangedNotifier.removeBucketStateChangedHandler(this);
+}
+
+
+void
+BucketMoveJob::maybeCancelMover(DocumentBucketMover &mover)
+{
+    // Cancel bucket if moving in wrong direction
+    if (!mover.bucketDone()) {
+        bool ready = mover.getSource() == &_ready;
+        if (!_runnable ||
+            _calc->shouldBeReady(mover.getBucket()) == ready) {
+            mover.cancel();
+        }
+    }
+}
+
+
+void
+BucketMoveJob::maybeDelayMover(DocumentBucketMover &mover, BucketId bucket)
+{
+    // Delay bucket if being frozen.
+    if (!mover.bucketDone() && bucket == mover.getBucket()) {
+        mover.cancel();
+        _delayedBucketsFrozen.insert(bucket);
+        _delayedBuckets.erase(bucket);
+    }
+}
+
+void
+BucketMoveJob::notifyThawedBucket(const BucketId &bucket)
+{
+    if (_delayedBucketsFrozen.erase(bucket) != 0u) {
+        _delayedBuckets.insert(bucket);
+        if (_runner && _runnable) {
+            _runner->run();
+        }
+    }
+}
+
+
+void
+BucketMoveJob::deactivateBucket(BucketId bucket)
+{
+    _delayedBuckets.insert(bucket);
+}
+
+
+void
+BucketMoveJob::activateBucket(BucketId bucket)
+{
+    BucketDBOwner::Guard notReadyBdb(_notReady._metaStore->getBucketDB().takeGuard());
+    if (notReadyBdb->get(bucket).getDocumentCount() == 0) {
+        return; // notready bucket already empty. This is the normal case.
+    }
+    _delayedBuckets.insert(bucket);
+}
+
+
+void
+BucketMoveJob::changedCalculator()
+{
+    if (done()) {
+        _scanPos = ScanPosition();
+        _endPos = ScanPosition();
+    } else {
+        _endPos = _scanPos;
+    }
+    _doneScan = false;
+    _scanPass = FIRST_SCAN_PASS;
+    maybeCancelMover(_mover);
+    maybeCancelMover(_delayedMover);
+}
+
+
+void
+BucketMoveJob::scanAndMove(size_t maxBucketsToScan,
+                           size_t maxDocsToMove)
+{
+    if (done()) {
+        return;
+    }
+    IFrozenBucketHandler::ExclusiveBucketGuard::UP bucketGuard;
+    // Look for delayed bucket to be processed now
+    while (!_delayedBuckets.empty() && _delayedMover.bucketDone()) {
+        const BucketId bucket = *_delayedBuckets.begin();
+        _delayedBuckets.erase(_delayedBuckets.begin());
+        ScanIterator itr(_ready._metaStore->getBucketDB().takeGuard(), bucket);
+        if (itr.getBucket() == bucket) {
+            checkBucket(bucket, itr, _delayedMover, bucketGuard);
+        }
+    }
+    if (!_delayedMover.bucketDone()) {
+        moveDocuments(_delayedMover, maxDocsToMove, bucketGuard);
+        return;
+    }
+    if (_mover.bucketDone()) {
+        size_t bucketsScanned = 0;
+        for (;;) {
+            if (_mover.bucketDone()) {
+                ScanResult res = scanBuckets(maxBucketsToScan -
+                                             bucketsScanned, bucketGuard);
+                bucketsScanned += res.first;
+                if (res.second) {
+                    if (_scanPass == FIRST_SCAN_PASS &&
+                        _endPos.validBucket()) {
+                        _scanPos = ScanPosition();
+                        _scanPass = SECOND_SCAN_PASS;
+                    } else {
+                        _doneScan = true;
+                        break;
+                    }
+                }
+            }
+            if (!_mover.bucketDone() || bucketsScanned >= maxBucketsToScan) {
+                break;
+            }
+        }
+    }
+    if (!_mover.bucketDone()) {
+        moveDocuments(_mover, maxDocsToMove, bucketGuard);
+    }
+}
+
+void
+BucketMoveJob::registerRunner(IMaintenanceJobRunner *runner)
+{
+    _runner = runner;
+}
+
+
+bool
+BucketMoveJob::run()
+{
+    if (!_runnable)
+        return true; // indicate work is done, since node state is bad
+    scanAndMove(200, 1);
+    return done();
+}
+
+
+void
+BucketMoveJob::refreshDerivedClusterState()
+{
+    _clusterUp = _calc.get() != NULL && _calc->clusterUp();
+    _nodeUp = _calc.get() != NULL && _calc->nodeUp();
+    _nodeInitializing = _calc.get() != NULL && _calc->nodeInitializing();
+    _runnable = _clusterUp && _nodeUp && !_nodeInitializing;
+}
+
+void
+BucketMoveJob::notifyClusterStateChanged(const IBucketStateCalculator::SP &
+                                         newCalc)
+{
+    // Called by master write thread
+    _calc = newCalc;
+    refreshDerivedClusterState();
+    changedCalculator();
+    if (_runner && _runnable) {
+        _runner->run();
+    }
+}
+
+void
+BucketMoveJob::notifyBucketStateChanged(const BucketId &bucketId,
+                                        BucketInfo::ActiveState newState)
+{
+    // Called by master write thread
+    if (newState == BucketInfo::NOT_ACTIVE) {
+        deactivateBucket(bucketId);
+    } else {
+        activateBucket(bucketId);
+    }
+    if (!done() && _runner && _runnable) {
+        _runner->run();
+    }
+}
+
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h
new file mode 100644
index 00000000000..d0adc4cefb0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h
@@ -0,0 +1,186 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "ibucketstatecalculator.h"
+#include "ibucketmodifiedhandler.h"
+#include "ifrozenbuckethandler.h"
+#include "documentbucketmover.h"
+#include "i_maintenance_job.h"
+#include "iclusterstatechangedhandler.h"
+#include "ibucketfreezelistener.h"
+#include "ibucketstatechangedhandler.h"
+#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h>
+
+namespace proton
+{
+
+
+class IBucketStateChangedNotifier;
+class IClusterStateChangedNotifier;
+
+/**
+ * Class used to control the moving of buckets between the ready and
+ * not ready sub databases.
+ */
+class BucketMoveJob : public IMaintenanceJob,
+                      public IClusterStateChangedHandler,
+                      public IBucketFreezeListener,
+                      public IBucketStateChangedHandler
+                      
+{
+public:
+    struct ScanPosition
+    {
+        document::BucketId _lastBucket;
+
+        ScanPosition() : _lastBucket() { }
+        ScanPosition(document::BucketId lastBucket) : _lastBucket(lastBucket) { }
+        bool validBucket() const { return _lastBucket.isSet(); }
+    };
+
+    typedef BucketDB::ConstMapIterator BucketIterator;
+
+    class ScanIterator
+    {
+    private:
+        BucketDBOwner::Guard _db;
+        BucketIterator       _itr;
+        BucketIterator       _end;
+
+    public:
+        ScanIterator(BucketDBOwner::Guard db,
+                     uint32_t pass,
+                     document::BucketId lastBucket,
+                     document::BucketId endBucket);
+
+        ScanIterator(BucketDBOwner::Guard db, document::BucketId bucket);
+
+        ScanIterator(const ScanIterator &) = delete;
+        ScanIterator(ScanIterator &&rhs);
+        ScanIterator &operator=(const ScanIterator &) = delete;
+        ScanIterator &operator=(ScanIterator &&rhs) = delete;
+
+        bool                   valid() const { return _itr != _end; }
+        bool                isActive() const { return _itr->second.isActive(); }
+        document::BucketId getBucket() const { return _itr->first; }
+        bool      hasReadyBucketDocs() const { return _itr->second.getReadyCount() != 0; }
+        bool   hasNotReadyBucketDocs() const { return _itr->second.getNotReadyCount() != 0; }
+
+        ScanIterator & operator++(void) {
+            ++_itr;
+            return *this;
+        }
+    };
+
+private:
+    typedef std::pair<size_t, bool> ScanResult;
+    IBucketStateCalculator::SP         _calc;
+    IDocumentMoveHandler              &_moveHandler;
+    IBucketModifiedHandler            &_modifiedHandler;
+    const MaintenanceDocumentSubDB    &_ready;
+    const MaintenanceDocumentSubDB    &_notReady;
+    DocumentBucketMover                _mover;
+    bool                               _doneScan;
+    ScanPosition                       _scanPos;
+    uint32_t                           _scanPass;
+    ScanPosition                       _endPos;
+
+    typedef std::set<document::BucketId>    DelayedBucketSet;
+
+    // Delayed buckets that are no longer frozen or active that can be considered for moving.
+    DelayedBucketSet                   _delayedBuckets;
+    // Frozen buckets that cannot be moved at all.
+    DelayedBucketSet                   _delayedBucketsFrozen;
+    IFrozenBucketHandler              &_frozenBuckets;
+    DocumentBucketMover                _delayedMover;
+    IMaintenanceJobRunner             *_runner;
+    bool                               _clusterUp;
+    bool                               _nodeUp;
+    bool                               _nodeInitializing;
+    bool                               _runnable;  // can try to perform work
+    IClusterStateChangedNotifier      &_clusterStateChangedNotifier;
+    IBucketStateChangedNotifier       &_bucketStateChangedNotifier;
+
+    ScanResult
+    scanBuckets(size_t maxBucketsToScan,
+                IFrozenBucketHandler::ExclusiveBucketGuard::UP & bucketGuard);
+
+    void maybeCancelMover(DocumentBucketMover &mover);
+    void maybeDelayMover(DocumentBucketMover &mover, document::BucketId bucket);
+
+    void
+    moveDocuments(DocumentBucketMover &mover,
+                  size_t maxDocsToMove,
+                  IFrozenBucketHandler::ExclusiveBucketGuard::UP & bucketGuard);
+
+    void
+    checkBucket(const document::BucketId &bucket,
+                ScanIterator &itr,
+                DocumentBucketMover &mover,
+                IFrozenBucketHandler::ExclusiveBucketGuard::UP & bucketGuard);
+
+    void refreshDerivedClusterState();
+
+    /**
+     * Signal that the given bucket should be de-activated.
+     * An active bucket is not considered for moving from ready to not ready sub database.
+     * A de-activated bucket can be considered for moving.
+     **/
+    void deactivateBucket(document::BucketId bucket);
+
+    /**
+     * Signal that the given bucket should be activated.
+     */
+    void activateBucket(document::BucketId bucket);
+
+public:
+    BucketMoveJob(const IBucketStateCalculator::SP &calc,
+                  IDocumentMoveHandler &moveHandler,
+                  IBucketModifiedHandler &modifiedHandler,
+                  const MaintenanceDocumentSubDB &ready,
+                  const MaintenanceDocumentSubDB &notReady,
+                  IFrozenBucketHandler &frozenBuckets,
+                  IClusterStateChangedNotifier &clusterStateChangedNotifier,
+                  IBucketStateChangedNotifier &bucketStateChangedNotifier,
+                  const vespalib::string &docTypeName);
+
+    virtual ~BucketMoveJob();
+
+    void changedCalculator();
+    void scanAndMove(size_t maxBucketsToScan, size_t maxDocsToMove);
+
+    bool done() const {
+        // Ignores _delayedBucketsFrozen, since no work can be done there yet
+        return
+            _doneScan &&
+            _mover.bucketDone() &&
+            _delayedMover.bucketDone() &&
+            _delayedBuckets.empty();
+    }
+
+    // IMaintenanceJob API
+    virtual void registerRunner(IMaintenanceJobRunner *runner) override;
+
+    // IMaintenanceJob API
+    virtual bool run() override;
+
+    // IClusterStateChangedHandler API
+    virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override;
+
+
+    // IBucketFreezeListener API
+    /**
+     * Signal that the given bucket has been thawed.
+     * A thawed bucket can be considered for moving.
+     */
+    virtual void notifyThawedBucket(const document::BucketId &bucket) override;
+
+    // IBucketStateChangedHandler API
+    void notifyBucketStateChanged(const document::BucketId &bucketId,
+                             storage::spi::BucketInfo::ActiveState newState) override;
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp
new file mode 100644
index 00000000000..aa2cb4e982d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp
@@ -0,0 +1,179 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.clusterstatehandler");
+#include "clusterstatehandler.h"
+#include "iclusterstatechangedhandler.h"
+#include <vespa/vespalib/util/closuretask.h>
+#include <algorithm>
+
+using storage::spi::Bucket;
+using storage::spi::BucketIdListResult;
+using storage::spi::ClusterState;
+using storage::spi::PartitionId;
+using storage::spi::Result;
+using vespalib::Executor;
+using vespalib::makeTask;
+using vespalib::makeClosure;
+
+namespace proton {
+
+namespace {
+
+class ClusterStateAdapter : public IBucketStateCalculator
+{
+private:
+    ClusterState _calc;
+
+public:
+    ClusterStateAdapter(const ClusterState &calc)
+        : _calc(calc)
+    {
+    }
+
+    virtual bool
+    shouldBeReady(const document::BucketId &bucket) const
+    {
+        return _calc.shouldBeReady(Bucket(bucket, PartitionId(0)));
+    }
+
+    virtual bool
+    clusterUp(void) const
+    {
+        return _calc.clusterUp();
+    }
+
+    virtual bool
+    nodeUp(void) const
+    {
+        return _calc.nodeUp();
+    }
+
+    virtual bool
+    nodeInitializing() const
+    {
+        return _calc.nodeInitializing();
+    }
+};
+
+}
+
+
+void
+ClusterStateHandler::performSetClusterState(const ClusterState *calc,
+        IGenericResultHandler *resultHandler)
+{
+    LOG(debug,
+        "performSetClusterState(): "
+        "clusterUp(%s), nodeUp(%s), nodeInitializing(%s)"
+        "changedHandlers.size() = %zu",
+        (calc->clusterUp() ? "true" : "false"),
+        (calc->nodeUp() ? "true" : "false"),
+        (calc->nodeInitializing() ? "true" : "false"),
+        _changedHandlers.size());
+    if (!_changedHandlers.empty()) {
+        IBucketStateCalculator::SP newCalc(new ClusterStateAdapter(*calc));
+        typedef std::vector<IClusterStateChangedHandler *> Chv;
+        Chv &chs(_changedHandlers);
+        for (Chv::const_iterator it = chs.begin(), ite = chs.end(); it != ite;
+             ++it) {
+            (*it)->notifyClusterStateChanged(newCalc);
+        }
+    }
+    resultHandler->handle(Result());
+}
+
+
+void
+ClusterStateHandler::performGetModifiedBuckets(
+        IBucketIdListResultHandler *resultHandler)
+{
+    storage::spi::BucketIdListResult::List modifiedBuckets;
+    modifiedBuckets.resize(_modifiedBuckets.size());
+    std::copy(_modifiedBuckets.begin(), _modifiedBuckets.end(),
+              modifiedBuckets.begin());
+
+    if (LOG_WOULD_LOG(debug) && !modifiedBuckets.empty()) {
+        std::ostringstream oss;
+        for (size_t i = 0; i < modifiedBuckets.size(); ++i) {
+            if (i != 0) {
+                oss << ",";
+            }
+            oss << modifiedBuckets[i];
+        }
+        LOG(debug, "performGetModifiedBuckets(): modifiedBuckets(%zu): %s",
+            modifiedBuckets.size(), oss.str().c_str());
+    }
+    resultHandler->handle(BucketIdListResult(modifiedBuckets));
+    _modifiedBuckets.clear();
+}
+
+
+void
+ClusterStateHandler::notifyBucketModified(const document::BucketId &bucket)
+{
+    _modifiedBuckets.insert(bucket);
+}
+
+
+ClusterStateHandler::ClusterStateHandler(Executor &executor)
+    : IBucketModifiedHandler(),
+      IClusterStateChangedNotifier(),
+      _executor(executor),
+      _changedHandlers(),
+      _modifiedBuckets()
+{
+}
+
+
+ClusterStateHandler::~ClusterStateHandler()
+{
+    assert(_changedHandlers.empty());
+}
+
+
+void
+ClusterStateHandler::
+addClusterStateChangedHandler(IClusterStateChangedHandler *handler)
+{
+    _changedHandlers.push_back(handler);
+}
+
+
+void
+ClusterStateHandler::
+removeClusterStateChangedHandler(IClusterStateChangedHandler *handler)
+{
+    auto it = std::find(_changedHandlers.begin(), _changedHandlers.end(),
+                        handler);
+    if (it != _changedHandlers.end()) {
+        _changedHandlers.erase(it);
+    }
+}
+
+
+void
+ClusterStateHandler::handleSetClusterState(const ClusterState &calc,
+        IGenericResultHandler &resultHandler)
+{
+    _executor.execute(makeTask(makeClosure(this,
+                                       &proton::ClusterStateHandler::
+                                       performSetClusterState,
+                                       &calc,
+                                       &resultHandler)));
+}
+
+
+void
+ClusterStateHandler::handleGetModifiedBuckets(
+        IBucketIdListResultHandler &resultHandler)
+{
+    _executor.execute(makeTask(makeClosure(this,
+                                       &proton::ClusterStateHandler::
+                                       performGetModifiedBuckets,
+                                       &resultHandler)));
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.h b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.h
new file mode 100644
index 00000000000..f4569a9114b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.h
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include "ibucketmodifiedhandler.h"
+#include "ibucketstatecalculator.h"
+#include "iclusterstatechangednotifier.h"
+#include <vespa/persistence/spi/clusterstate.h>
+#include <vespa/searchcore/proton/persistenceengine/resulthandler.h>
+#include <vespa/vespalib/util/threadstackexecutorbase.h>
+
+namespace proton {
+
+/**
+ * Class handling operations in IPersistenceHandler that are dealing with
+ * that the cluster state is changing.
+ */
+class ClusterStateHandler : public IBucketModifiedHandler,
+                            public IClusterStateChangedNotifier
+{
+private:
+    vespalib::Executor &_executor;
+    std::vector<IClusterStateChangedHandler *> _changedHandlers;
+    // storage::spi::BucketIdListResult::List _modifiedBuckets;
+    std::set<document::BucketId> _modifiedBuckets;
+
+    void
+    performSetClusterState(const storage::spi::ClusterState *calc,
+                           IGenericResultHandler *resultHandler);
+
+    void
+    performGetModifiedBuckets(IBucketIdListResultHandler *resultHandler);
+
+    // Implements IBucketModifiedHandler
+    virtual void
+    notifyBucketModified(const document::BucketId &bucket);
+
+public:
+    ClusterStateHandler(vespalib::Executor &executor);
+
+    virtual
+    ~ClusterStateHandler();
+
+    virtual void
+    addClusterStateChangedHandler(IClusterStateChangedHandler *handler);
+
+    virtual void
+    removeClusterStateChangedHandler(IClusterStateChangedHandler *handler);
+
+    /**
+     * Implements the cluster state aspect of IPersistenceHandler.
+     */
+    void
+    handleSetClusterState(const storage::spi::ClusterState &calc,
+                               IGenericResultHandler &resultHandler);
+
+    void
+    handleGetModifiedBuckets(IBucketIdListResultHandler &resultHandler);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
new file mode 100644
index 00000000000..39025c78b31
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
@@ -0,0 +1,306 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.combiningfeedview");
+#include "combiningfeedview.h"
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+
+using document::DocumentTypeRepo;
+using document::DocumentId;
+
+namespace proton
+{
+
+namespace
+{
+
+DocumentTypeRepo::SP
+getRepo(const std::vector<IFeedView::SP> &views)
+{
+    for (const auto &view : views) {
+        if (view.get() == NULL)
+            continue;
+        return view->getDocumentTypeRepo();
+    }
+    abort();
+    return DocumentTypeRepo::SP();
+}
+
+};
+
+
+CombiningFeedView::CombiningFeedView(const std::vector<IFeedView::SP> &views,
+                                     const IBucketStateCalculator::SP &calc)
+    : _repo(getRepo(views)),
+      _views(views),
+      _metaStores(),
+      _calc(calc),
+      _clusterUp(calc.get() != NULL && calc->clusterUp()),
+      _forceReady(!_clusterUp || !hasNotReadyFeedView())
+{
+    _metaStores.reserve(views.size());
+    for (const auto &view : views) {
+        _metaStores.push_back(view->getDocumentMetaStorePtr());
+    }
+    assert(getReadyFeedView() != NULL);
+    assert(getRemFeedView() != NULL);
+    if (hasNotReadyFeedView()) {
+        assert(getNotReadyFeedView() != NULL);
+    }
+}
+
+
+CombiningFeedView::~CombiningFeedView(void)
+{
+}
+
+
+const ISimpleDocumentMetaStore *
+CombiningFeedView::getDocumentMetaStorePtr(void) const
+{
+    return NULL;
+}
+
+
+void
+CombiningFeedView::findPrevDbdId(const document::GlobalId &gid,
+                                 DocumentOperation &op)
+{
+    uint32_t subDbIdLim = _metaStores.size();
+    uint32_t skipSubDbId = std::numeric_limits<uint32_t>::max();
+    DbDocumentId newId(op.getDbDocumentId());
+    if (newId.valid()) {
+        skipSubDbId = newId.getSubDbId();
+    }
+    for (uint32_t subDbId = 0; subDbId < subDbIdLim; ++subDbId) {
+        if (subDbId == skipSubDbId)
+            continue;
+        const documentmetastore::IStore *metaStore = _metaStores[subDbId];
+        if (metaStore == NULL)
+            continue;
+        documentmetastore::IStore::Result inspectRes(metaStore->inspectExisting(gid));
+        if (inspectRes._found) {
+            op.setPrevDbDocumentId(DbDocumentId(subDbId,
+                                           inspectRes._lid));
+            op.setPrevMarkedAsRemoved(subDbId == getRemFeedViewId());
+            op.setPrevTimestamp(inspectRes._timestamp);
+            break;
+        }
+    }
+}
+
+
+const DocumentTypeRepo::SP &
+CombiningFeedView::getDocumentTypeRepo(void) const
+{
+    return _repo;
+}
+
+
+/**
+ * Similar to IFeedHandler and IPersistenceHandler functions.
+ */
+
+void
+CombiningFeedView::preparePut(PutOperation &putOp)
+{
+    if (shouldBeReady(putOp.getBucketId())) {
+        getReadyFeedView()->preparePut(putOp);
+    } else {
+        getNotReadyFeedView()->preparePut(putOp);
+    }
+    if (!putOp.getPrevDbDocumentId().valid()) {
+        const DocumentId &docId = putOp.getDocument()->getId();
+        const document::GlobalId &gid = docId.getGlobalId();
+        findPrevDbdId(gid, putOp);
+    }
+}
+
+
+void
+CombiningFeedView::handlePut(FeedToken *token,
+                             const PutOperation &putOp)
+{
+    assert(putOp.getValidDbdId());
+    uint32_t subDbId = putOp.getSubDbId();
+    uint32_t prevSubDbId = putOp.getPrevSubDbId();
+    if (putOp.getValidPrevDbdId() && prevSubDbId != subDbId) {
+        if (token != NULL)
+            token->incNeededAcks();
+        _views[subDbId]->handlePut(token, putOp);
+        _views[prevSubDbId]->handlePut(token, putOp);
+    } else {
+        _views[subDbId]->handlePut(token, putOp);
+    }
+}
+
+
+void
+CombiningFeedView::prepareUpdate(UpdateOperation &updOp)
+{
+    getReadyFeedView()->prepareUpdate(updOp);
+    if (!updOp.getPrevDbDocumentId().valid() && hasNotReadyFeedView()) {
+        getNotReadyFeedView()->prepareUpdate(updOp);
+    }
+}
+
+
+void
+CombiningFeedView::handleUpdate(FeedToken *token,
+                                const UpdateOperation &updOp)
+{
+    assert(updOp.getValidDbdId());
+    assert(updOp.getValidPrevDbdId());
+    assert(!updOp.changedDbdId());
+    uint32_t subDbId(updOp.getSubDbId());
+    _views[subDbId]->handleUpdate(token, updOp);
+}
+
+
+void
+CombiningFeedView::prepareRemove(RemoveOperation &rmOp)
+{
+    getRemFeedView()->prepareRemove(rmOp);
+    if (!rmOp.getPrevDbDocumentId().valid()) {
+        const DocumentId &docId = rmOp.getDocumentId();
+        const document::GlobalId &gid = docId.getGlobalId();
+        findPrevDbdId(gid, rmOp);
+    }
+}
+
+
+void
+CombiningFeedView::handleRemove(FeedToken *token,
+                                const RemoveOperation &rmOp)
+{
+    if (rmOp.getValidDbdId()) {
+        uint32_t subDbId = rmOp.getSubDbId();
+        uint32_t prevSubDbId = rmOp.getPrevSubDbId();
+        if (rmOp.getValidPrevDbdId() && prevSubDbId != subDbId) {
+            if (token != NULL)
+                token->incNeededAcks();
+            _views[subDbId]->handleRemove(token, rmOp);
+            _views[prevSubDbId]->handleRemove(token, rmOp);
+        } else {
+            _views[subDbId]->handleRemove(token, rmOp);
+        }
+    } else {
+        assert(rmOp.getValidPrevDbdId());
+        uint32_t prevSubDbId = rmOp.getPrevSubDbId();
+        _views[prevSubDbId]->handleRemove(token, rmOp);
+    }
+}
+
+void
+CombiningFeedView::prepareDeleteBucket(DeleteBucketOperation &delOp)
+{
+    for (const auto &view : _views) {
+        view->prepareDeleteBucket(delOp);
+    }
+}
+
+
+void
+CombiningFeedView::handleDeleteBucket(const DeleteBucketOperation &delOp)
+{
+    for (const auto &view : _views) {
+        view->handleDeleteBucket(delOp);
+    }
+}
+
+
+void
+CombiningFeedView::prepareMove(MoveOperation &moveOp)
+{
+    uint32_t subDbId = moveOp.getSubDbId();
+    assert(subDbId < _views.size());
+    _views[subDbId]->prepareMove(moveOp);
+}
+
+
+void
+CombiningFeedView::handleMove(const MoveOperation &moveOp)
+{
+    assert(moveOp.getValidDbdId());
+    uint32_t subDbId = moveOp.getSubDbId();
+    uint32_t prevSubDbId = moveOp.getPrevSubDbId();
+    if (moveOp.getValidPrevDbdId() && prevSubDbId != subDbId) {
+        _views[subDbId]->handleMove(moveOp);
+        // XXX: index executor not synced.
+        _views[prevSubDbId]->handleMove(moveOp);
+    } else {
+        _views[subDbId]->handleMove(moveOp);
+    }
+}
+
+
+void
+CombiningFeedView::heartBeat(search::SerialNum serialNum)
+{
+    for (const auto &view : _views) {
+        view->heartBeat(serialNum);
+    }
+}
+
+
+void
+CombiningFeedView::sync(void)
+{
+    getReadyFeedView()->sync();
+    // Assume this synced all feed views due to sharing of threads.
+}
+
+void
+CombiningFeedView::forceCommit(search::SerialNum serialNum)
+{
+    for (const auto &view : _views) {
+        view->forceCommit(serialNum);
+    }
+}
+
+
+void
+CombiningFeedView::
+handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp)
+{
+    getRemFeedView()->handlePruneRemovedDocuments(pruneOp);
+}
+
+void
+CombiningFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+{
+    uint32_t subDbId = op.getSubDbId();
+    assert(subDbId < _views.size());
+    _views[subDbId]->handleCompactLidSpace(op);
+}
+
+void
+CombiningFeedView::setCalculator(const IBucketStateCalculator::SP &newCalc)
+{
+    // Called by document db executor
+    _calc = newCalc;
+    _clusterUp = _calc.get() != NULL && _calc->clusterUp();
+    _forceReady = !_clusterUp || !hasNotReadyFeedView();
+}
+
+
+bool
+CombiningFeedView::shouldBeReady(const document::BucketId &bucket) const
+{
+    LOG(debug,
+        "shouldBeReady(%s): forceReady(%s), clusterUp(%s), calcReady(%s)",
+        bucket.toString().c_str(),
+        (_forceReady ? "true" : "false"),
+        (_clusterUp ? "true" : "false"),
+        (_calc.get() != NULL ?
+         (_calc->shouldBeReady(bucket) ? "true" : "false") : "null"));
+    const documentmetastore::IBucketHandler *readyMetaStore =
+        _metaStores[getReadyFeedViewId()];
+    bool isActive = readyMetaStore->getBucketDB().takeGuard()->isActiveBucket(bucket);
+    return _forceReady || isActive || _calc->shouldBeReady(bucket);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h
new file mode 100644
index 00000000000..879fc75a802
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "ifeedview.h"
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/feedoperation/deletebucketoperation.h>
+#include <vespa/searchcore/proton/feedoperation/joinbucketsoperation.h>
+#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h>
+#include <vespa/searchcore/proton/feedoperation/putoperation.h>
+#include <vespa/searchcore/proton/feedoperation/removeoperation.h>
+#include <vespa/searchcore/proton/feedoperation/splitbucketoperation.h>
+#include <vespa/searchcore/proton/feedoperation/updateoperation.h>
+#include <vespa/searchcore/proton/feedoperation/createbucketoperation.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include "replaypacketdispatcher.h"
+#include "ibucketstatecalculator.h"
+
+namespace proton
+{
+
+
+class CombiningFeedView : public IFeedView
+{
+private:
+    const document::DocumentTypeRepo::SP          _repo;
+    std::vector<IFeedView::SP>                    _views;
+    std::vector<const ISimpleDocumentMetaStore *> _metaStores;
+    IBucketStateCalculator::SP                    _calc;
+    bool                                          _clusterUp;
+    bool                                          _forceReady;
+
+    const ISimpleDocumentMetaStore * getDocumentMetaStorePtr(void) const override;
+
+    void findPrevDbdId(const document::GlobalId &gid, DocumentOperation &op);
+    uint32_t getReadyFeedViewId() const { return 0u; }
+    uint32_t getRemFeedViewId() const { return 1u; }
+    uint32_t getNotReadyFeedViewId() const { return 2u; }
+
+    IFeedView * getReadyFeedView() {
+        return _views[getReadyFeedViewId()].get();
+    }
+
+    IFeedView * getRemFeedView() {
+        return _views[getRemFeedViewId()].get();
+    }
+
+    IFeedView * getNotReadyFeedView() {
+        return _views[getNotReadyFeedViewId()].get();
+    }
+
+    bool hasNotReadyFeedView() const {
+        return _views.size() > getNotReadyFeedViewId();
+    }
+
+    bool shouldBeReady(const document::BucketId &bucket) const;
+    void forceCommit(search::SerialNum serialNum) override;
+public:
+    typedef std::shared_ptr<CombiningFeedView> SP;
+
+    CombiningFeedView(const std::vector<IFeedView::SP> &views,
+                      const IBucketStateCalculator::SP &calc);
+
+    virtual ~CombiningFeedView();
+
+    const document::DocumentTypeRepo::SP & getDocumentTypeRepo(void) const override;
+
+    /**
+     * Similar to IPersistenceHandler functions.
+     */
+
+    void preparePut(PutOperation &putOp) override;
+    void handlePut(FeedToken *token, const PutOperation &putOp) override;
+    void prepareUpdate(UpdateOperation &updOp) override;
+    void handleUpdate(FeedToken *token, const UpdateOperation &updOp) override;
+    void prepareRemove(RemoveOperation &rmOp) override;
+    void handleRemove(FeedToken *token, const RemoveOperation &rmOp) override;
+    void prepareDeleteBucket(DeleteBucketOperation &delOp) override;
+    void handleDeleteBucket(const DeleteBucketOperation &delOp) override;
+    void prepareMove(MoveOperation &putOp) override;
+    void handleMove(const MoveOperation &moveOp) override;
+    void heartBeat(search::SerialNum serialNum) override;
+    void sync() override;
+    void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp) override;
+    void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
+
+    // Called by document db executor
+    void setCalculator(const IBucketStateCalculator::SP &newCalc);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/commit_and_wait_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/commit_and_wait_document_retriever.h
new file mode 100644
index 00000000000..3735240d55d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/commit_and_wait_document_retriever.h
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
+#include <vespa/searchcore/proton/server/icommitable.h>
+
+namespace proton {
+
+/*
+ * Wrapper class for document retriever that performs commit and waits for
+ * it to complete before retrieving document. This is used to ensure that
+ * attribute vectors are committed before we read from them.
+ */
+class CommitAndWaitDocumentRetriever : public IDocumentRetriever
+{
+    IDocumentRetriever::SP _retriever;
+    ICommitable           &_commit;
+    using Bucket = storage::spi::Bucket;
+public:
+    CommitAndWaitDocumentRetriever(const IDocumentRetriever::SP &retriever, ICommitable &commit)
+        : _retriever(retriever),
+          _commit(commit)
+    { }
+
+    ~CommitAndWaitDocumentRetriever() {}
+
+    const document::DocumentTypeRepo &getDocumentTypeRepo() const override {
+        return _retriever->getDocumentTypeRepo();
+    }
+
+    void getBucketMetaData(const Bucket &bucket, search::DocumentMetaData::Vector &result) const override {
+        return _retriever->getBucketMetaData(bucket, result);
+    }
+
+    search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const override {
+        return _retriever->getDocumentMetaData(id);
+    }
+    document::Document::UP getDocument(search::DocumentIdT lid) const override {
+        // Ensure that attribute vectors are committed
+        _commit.commitAndWait();
+        return _retriever->getDocument(lid);
+    }
+    void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor,
+                        ReadConsistency readConsistency) const override
+    {
+        if (readConsistency == ReadConsistency::STRONG) {
+            _commit.commitAndWait();
+        }
+        _retriever->visitDocuments(lids, visitor, readConsistency);
+    }
+
+    CachedSelect::SP parseSelect(const vespalib::string &selection) const override {
+        return _retriever->parseSelect(selection);
+    }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/configstore.h b/searchcore/src/vespa/searchcore/proton/server/configstore.h
new file mode 100644
index 00000000000..ee6352d61d5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/configstore.h
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentdbconfig.h"
+#include "feedconfigstore.h"
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/config/config-proton.h>
+
+namespace proton {
+
+struct ConfigStore : FeedConfigStore {
+    typedef std::unique_ptr<ConfigStore> UP;
+    typedef search::SerialNum SerialNum;
+    using ProtonConfig = vespa::config::search::core::ProtonConfig;
+    using ProtonConfigSP = std::shared_ptr<ProtonConfig>;
+
+    /**
+     * @param currentSnapshot the current snapshot, for reusing
+     *                        unchanged parts .
+     * @param serialNum the serial number of the config snapshot to load.
+     * @param loadedSnapshot the shared pointer in which to store the
+     *                       resulting config snapshot.
+     * @param historySchema the shared pointer in which to store the
+     *                      resulting history schema.
+     */
+    virtual void loadConfig(const DocumentDBConfig &currentSnapshot,
+                            SerialNum serialNum,
+                            DocumentDBConfig::SP &loadedSnapshot,
+                            search::index::Schema::SP &historySchema) = 0;
+    virtual void saveConfig(const DocumentDBConfig &snapshot,
+                            const search::index::Schema &historySchema,
+                            SerialNum serialNum) = 0;
+
+    virtual void removeInvalid() = 0;
+    virtual void prune(SerialNum serialNum) = 0;
+
+    virtual SerialNum getBestSerialNum() const = 0;
+    virtual SerialNum getOldestSerialNum() const = 0;
+    virtual bool hasValidSerial(SerialNum serialNum) const = 0;
+    virtual SerialNum getPrevValidSerial(SerialNum serialNum) const = 0;
+    virtual void setProtonConfig(const ProtonConfigSP &protonConfig) = 0;
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/configvalidator.cpp b/searchcore/src/vespa/searchcore/proton/server/configvalidator.cpp
new file mode 100644
index 00000000000..430b11c18b8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/configvalidator.cpp
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.configvalidator");
+#include "configvalidator.h"
+
+#include "schema_config_validator.h"
+#include "attribute_config_validator.h"
+
+namespace proton {
+
+ConfigValidator::Result
+ConfigValidator::validate(const ConfigValidator::Config &newCfg,
+                          const ConfigValidator::Config &oldCfg,
+                          const search::index::Schema &oldHistory)
+{
+    Result res;
+    if (!(res = SchemaConfigValidator::validate(newCfg.getSchema(),
+            oldCfg.getSchema(), oldHistory)).ok()) return res;
+    if (!(res = AttributeConfigValidator::validate(newCfg.getAttributeConfig(),
+            oldCfg.getAttributeConfig())).ok()) return res;
+    return Result();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/configvalidator.h b/searchcore/src/vespa/searchcore/proton/server/configvalidator.h
new file mode 100644
index 00000000000..5e700234041
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/configvalidator.h
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/config-attributes.h>
+
+namespace proton {
+
+/**
+ * Class used to validate new document db config before starting using it.
+ **/
+class ConfigValidator {
+public:
+    /**
+     * The various results of a schema check.
+     * All but OK means that the new schema should be rejected.
+     */
+    enum ResultType
+    {
+        OK,
+        DATA_TYPE_CHANGED,
+        COLLECTION_TYPE_CHANGED,
+        INDEX_ASPECT_ADDED,
+        INDEX_ASPECT_REMOVED,
+        ATTRIBUTE_ASPECT_ADDED,
+        ATTRIBUTE_ASPECT_REMOVED,
+        ATTRIBUTE_FAST_ACCESS_ADDED,
+        ATTRIBUTE_FAST_ACCESS_REMOVED
+    };
+
+    class Result
+    {
+    private:
+        ResultType _type;
+        vespalib::string _what;
+    public:
+        Result()
+            : _type(OK),
+              _what("")
+        {}
+        Result(ResultType type_, const vespalib::string &what_)
+            : _type(type_),
+              _what(what_)
+        {}
+        ResultType type() const { return _type; }
+        const vespalib::string &what() const { return _what; }
+        bool ok() const { return type() == OK; }
+    };
+
+    class Config
+    {
+    private:
+        const search::index::Schema &_schema;
+        const vespa::config::search::AttributesConfig &_attributeCfg;
+    public:
+        Config(const search::index::Schema &schema,
+               const vespa::config::search::AttributesConfig &attributeCfg)
+            : _schema(schema),
+              _attributeCfg(attributeCfg)
+        {}
+        const search::index::Schema &getSchema() const {
+            return _schema;
+        }
+        const vespa::config::search::AttributesConfig &getAttributeConfig() const {
+            return _attributeCfg;
+        }
+    };
+
+    /**
+     * Check if new schema can be applied or not.
+     */
+    static Result
+    validate(const Config &newCfg,
+             const Config &oldCfg,
+             const search::index::Schema &oldHistory);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp b/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp
new file mode 100644
index 00000000000..626e8e664bd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp
@@ -0,0 +1,196 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.data_directory_upgrader");
+#include "data_directory_upgrader.h"
+
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <algorithm>
+#include <iostream>
+
+namespace proton {
+
+namespace {
+
+vespalib::string UPGRADE_SOURCE_FILE = "data-directory-upgrade-source.txt";
+vespalib::string DOWNGRADE_SCRIPT_FILE = "data-directory-downgrade.sh";
+
+bool
+isValidDir(const vespalib::string &dir, char prefix)
+{
+    if (dir.empty() || dir[0] != prefix) {
+        return false;
+    }
+    vespalib::asciistream stream(dir.substr(1));
+    uint32_t number = 0;
+    try {
+        stream >> number;
+    } catch (const vespalib::IllegalArgumentException &) {
+        return false;
+    }
+    return true;
+}
+
+bool
+isRowDir(const vespalib::string &dir)
+{
+    return isValidDir(dir, 'r');
+}
+
+bool
+isColumnDir(const vespalib::string &dir)
+{
+    return isValidDir(dir, 'c');
+}
+
+vespalib::string
+createDirString(const DataDirectoryUpgrader::RowColDirs &dirs)
+{
+    vespalib::asciistream result;
+    bool first = true;
+    for (const auto &dir : dirs) {
+        if (!first) {
+            result << ", ";
+        }
+        result << "'" << dir.dir() << "'";
+        first = false;
+    }
+    return result.str();
+}
+
+void
+writeUpgradeFile(const vespalib::string &srcDir,
+                 const vespalib::string &dstDir)
+{
+    vespalib::File file(dstDir + "/" + UPGRADE_SOURCE_FILE);
+    file.open(vespalib::File::CREATE);
+    file.write(&srcDir[0], srcDir.size(), 0);
+    file.close();
+}
+
+void
+writeDowngradeScript(const vespalib::string &scanDir,
+                     const vespalib::string &dstDir,
+                     const DataDirectoryUpgrader::RowColDir &rowColDir)
+{
+    vespalib::asciistream script;
+    vespalib::string fullRowDir = scanDir + "/" + rowColDir.row();
+    vespalib::string fullRowColDir = scanDir + "/" + rowColDir.dir();
+    script << "#!/bin/sh\n\n";
+    script << "mkdir " << fullRowDir << " || exit 1\n";
+    script << "chown yahoo " << fullRowDir << "\n";
+    script << "mv " << dstDir << " " << fullRowColDir << "\n";
+    script << "rm " << fullRowColDir << "/" << UPGRADE_SOURCE_FILE << "\n";
+    script << "rm " << fullRowColDir << "/" << DOWNGRADE_SCRIPT_FILE << "\n";
+    vespalib::string fileName = dstDir + "/" + DOWNGRADE_SCRIPT_FILE;
+    vespalib::File file(fileName);
+    file.open(vespalib::File::CREATE);
+    file.write(script.c_str(), script.size(), 0);
+    file.close();
+    chmod(fileName.c_str(), 0755);
+}
+
+}
+
+
+DataDirectoryUpgrader::RowColDir::RowColDir(const vespalib::string &row_,
+                                            const vespalib::string &col_)
+    : _row(row_),
+      _col(col_)
+{
+}
+
+DataDirectoryUpgrader::ScanResult::ScanResult()
+    : _rowColDirs(),
+      _destDirExisting(false)
+{
+}
+
+DataDirectoryUpgrader::UpgradeResult::UpgradeResult(const Status status,
+                                                    const vespalib::string &desc)
+    : _status(status),
+      _desc(desc)
+{
+}
+
+DataDirectoryUpgrader::DataDirectoryUpgrader(const vespalib::string &scanDir,
+                                             const vespalib::string &destDir)
+    : _scanDir(scanDir),
+      _destDir(destDir)
+{
+}
+
+DataDirectoryUpgrader::ScanResult
+DataDirectoryUpgrader::scan() const
+{
+    ScanResult result;
+    try {
+        vespalib::DirectoryList dirs = listDirectory(_scanDir);
+        for (const auto &dir : dirs) {
+            if (isRowDir(dir)) {
+                vespalib::DirectoryList subDirs = listDirectory(_scanDir + "/" + dir);
+                for (const auto &subDir : subDirs) {
+                    if (isColumnDir(subDir)) {
+                        result.addDir(RowColDir(dir, subDir));
+                    }
+                }
+            }
+        }
+    } catch (const vespalib::IoException &) {
+        // Scan dir does not exists
+    }
+    try {
+        if (vespalib::stat(_destDir).get() != NULL) {
+            result.setDestDirExisting(true);
+        }
+    } catch (const vespalib::IoException &) {}
+    std::sort(result.getRowColDirs().begin(), result.getRowColDirs().end());
+    return result;
+}
+
+DataDirectoryUpgrader::UpgradeResult
+DataDirectoryUpgrader::upgrade(const ScanResult &scanResult) const
+{
+    if (scanResult.isDestDirExisting()) {
+        return UpgradeResult(IGNORE,
+                             vespalib::make_string("Destination directory '%s' is already existing",
+                                                   _destDir.c_str()));
+    }
+    const RowColDirs &rowColDirs = scanResult.getRowColDirs();
+    if (rowColDirs.empty()) {
+        return UpgradeResult(IGNORE, "No directory to upgrade");
+    }
+    if (rowColDirs.size() > 1) {
+        return UpgradeResult(ERROR,
+                             vespalib::make_string("Can only upgrade a single directory, was asked to upgrade %zu (%s)",
+                                                   rowColDirs.size(), createDirString(rowColDirs).c_str()));
+    }
+    const vespalib::string src = _scanDir + "/" + rowColDirs[0].dir();
+    const vespalib::string &dst = _destDir;
+    try {
+        if (!vespalib::rename(src, dst)) {
+            return UpgradeResult(ERROR,
+                                 vespalib::make_string("Failed to rename directory '%s' to '%s'",
+                                                       src.c_str(), dst.c_str()));
+        }
+        const vespalib::string rmDir = _scanDir + "/" + rowColDirs[0].row();
+        if (!vespalib::rmdir(rmDir, false)) {
+            return UpgradeResult(ERROR,
+                                 vespalib::make_string("Failed to remove empty directory '%s'",
+                                                       rmDir.c_str()));
+        }
+        writeUpgradeFile(src, dst);
+        writeDowngradeScript(_scanDir, dst, rowColDirs[0]);
+    } catch (const vespalib::IoException &ex) {
+        return UpgradeResult(ERROR,
+                             vespalib::make_string("Got exception during data directory upgrade from '%s' to '%s': %s",
+                                                   src.c_str(), dst.c_str(), ex.what()));
+    }
+    return UpgradeResult(COMPLETE,
+                         vespalib::make_string("Moved data from '%s' to '%s'",
+                                               src.c_str(), dst.c_str()));
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h b/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h
new file mode 100644
index 00000000000..97329c9b66e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+/**
+ * Class used to upgrade a row column directory /rX/cY to an elastic directory /nZ
+ * where Z is the distribution key for that search node.
+ */
+class DataDirectoryUpgrader
+{
+public:
+    class RowColDir
+    {
+    private:
+        vespalib::string _row;
+        vespalib::string _col;
+
+    public:
+        RowColDir(const vespalib::string &row_, const vespalib::string &col_);
+        const vespalib::string &row() const { return _row; }
+        const vespalib::string &col() const { return _col; }
+        vespalib::string dir() const { return row() + "/" + col(); }
+        bool operator< (const RowColDir &rhs) const { return dir() < rhs.dir(); }
+    };
+    typedef std::vector<RowColDir> RowColDirs;
+
+    class ScanResult
+    {
+    private:
+        RowColDirs  _rowColDirs;
+        bool        _destDirExisting;
+
+    public:
+        ScanResult();
+        void addDir(const RowColDir &dir) {
+            _rowColDirs.push_back(dir);
+        }
+        RowColDirs &getRowColDirs() { return _rowColDirs; }
+        const RowColDirs &getRowColDirs() const { return _rowColDirs; }
+        void setDestDirExisting(bool val) { _destDirExisting = val; }
+        bool isDestDirExisting() const { return _destDirExisting; }
+    };
+
+    enum Status
+    {
+        IGNORE,
+        COMPLETE,
+        ERROR
+    };
+
+    class UpgradeResult
+    {
+    private:
+        const Status _status;
+        const vespalib::string _desc;
+
+    public:
+        UpgradeResult(const Status status, const vespalib::string &desc);
+        Status getStatus() const { return _status; }
+        const vespalib::string &getDesc() const { return _desc; }
+    };
+
+private:
+    const vespalib::string _scanDir;
+    const vespalib::string _destDir;
+
+public:
+    DataDirectoryUpgrader(const vespalib::string &scanDir, const vespalib::string &destDir);
+    ScanResult scan() const;
+    UpgradeResult upgrade(const ScanResult &scanResult) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ddbstate.cpp b/searchcore/src/vespa/searchcore/proton/server/ddbstate.cpp
new file mode 100644
index 00000000000..8ffa2c88bab
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ddbstate.cpp
@@ -0,0 +1,214 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.ddbstate");
+
+#include "ddbstate.h"
+
+
+namespace proton {
+
+std::vector<vespalib::string> DDBState::_stateNames =
+{
+    "CONSTRUCT",
+    "LOAD",
+    "REPLAY_TRANSACTION_LOG",
+    "REDO_REPROCESS",
+    "APPLY_LIVE_CONFIG",
+    "REPROCESS",
+    "ONLINE",
+    "SHUTDOWN",
+    "DEAD",
+};
+
+std::vector<vespalib::string> DDBState::_configStateNames =
+{
+    "OK",
+    "NEED_RESTART",
+    "REJECT"
+};
+
+DDBState::DDBState()
+    : _state(State::CONSTRUCT),
+      _configState(ConfigState::OK),
+      _lock(),
+      _cond()
+{
+}
+
+
+DDBState::~DDBState()
+{
+
+}
+
+
+bool
+DDBState::enterLoadState()
+{
+    Guard guard(_lock);
+    if (getClosed()) {
+        return false;
+    }
+    assert(_state == State::CONSTRUCT);
+    _state = State::LOAD;
+    return true;
+}
+
+    
+bool
+DDBState::enterReplayTransactionLogState()
+{
+    Guard guard(_lock);
+    if (getClosed()) {
+        return false;
+    }
+    assert(_state == State::LOAD);
+    _state = State::REPLAY_TRANSACTION_LOG;
+    return true;
+}
+
+
+bool
+DDBState::enterRedoReprocessState()
+{
+    Guard guard(_lock);
+    if (getClosed()) {
+        return false;
+    }
+    assert(_state == State::REPLAY_TRANSACTION_LOG);
+    _state = State::REDO_REPROCESS;
+    return true;
+}
+
+
+bool
+DDBState::enterApplyLiveConfigState()
+{
+    Guard guard(_lock);
+    if (getClosed()) {
+        return false;
+    }
+    assert(_state == State::REPLAY_TRANSACTION_LOG ||
+           _state == State::REDO_REPROCESS);
+    _state = State::APPLY_LIVE_CONFIG;
+    return true;
+}
+
+
+bool
+DDBState::enterReprocessState()
+{
+    Guard guard(_lock);
+    if (getClosed()) {
+        return false;
+    }
+    assert(_state == State::APPLY_LIVE_CONFIG);
+    _state = State::REPROCESS;
+    return true;
+}
+
+bool
+DDBState::enterOnlineState()
+{
+    Guard guard(_lock);
+    if (getClosed()) {
+        return false;
+    }
+    assert(_state == State::REPROCESS);
+    _state = State::ONLINE;
+    _cond.notify_all();
+    return true;
+}
+
+
+void
+DDBState::enterShutdownState()
+{
+    Guard guard(_lock);
+    // Shutdown can be initiated before online state was reached
+    if (getClosed()) {
+        return;
+    }
+    _state = State::SHUTDOWN;
+    _cond.notify_all();
+}
+
+void
+DDBState::enterDeadState()
+{
+    Guard guard(_lock);
+    if (_state == State::DEAD) {
+        return;
+    }
+    assert(_state == State::SHUTDOWN);
+    _state = State::DEAD;
+    _cond.notify_all();
+}
+
+
+void
+DDBState::setConfigState(ConfigState newConfigState)
+{
+    Guard guard(_lock);
+    _configState = newConfigState;
+}
+
+
+void
+DDBState::clearRejectedConfig()
+{
+    setConfigState(ConfigState::OK);
+}
+
+
+DDBState::ConfigState
+DDBState::calcConfigState(const ConfigValidator::ResultType &cvr)
+{
+    if (_state < State::APPLY_LIVE_CONFIG) {
+        // Config has been accepted, placed in transaction log and
+        // activated by earlier instance. Rejecting config would cause
+        // a divergent state.
+        return ConfigState::OK;
+    }
+    switch (cvr) {
+    case ConfigValidator::ResultType::OK:
+        return ConfigState::OK;
+    case ConfigValidator::ResultType::ATTRIBUTE_ASPECT_ADDED:
+    case ConfigValidator::ResultType::ATTRIBUTE_FAST_ACCESS_ADDED:
+    case ConfigValidator::ResultType::ATTRIBUTE_ASPECT_REMOVED:
+    case ConfigValidator::ResultType::ATTRIBUTE_FAST_ACCESS_REMOVED:
+        if (_state == State::APPLY_LIVE_CONFIG) {
+            return ConfigState::OK;
+        }
+        return ConfigState::NEED_RESTART;
+    default:
+        return ConfigState::REJECT;
+    }
+}
+
+
+vespalib::string
+DDBState::getStateString(State state)
+{
+    return _stateNames[static_cast<unsigned int>(state)];
+}
+
+
+vespalib::string
+DDBState::getConfigStateString(ConfigState configState)
+{
+    return _configStateNames[static_cast<unsigned int>(configState)];
+}
+
+
+void
+DDBState::waitForOnlineState()
+{
+    GuardLock lk(_lock);
+    _cond.wait(lk, [this] { return this->_state >= State::ONLINE; } );
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/ddbstate.h b/searchcore/src/vespa/searchcore/proton/server/ddbstate.h
new file mode 100644
index 00000000000..26cb85f3b39
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ddbstate.h
@@ -0,0 +1,171 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "configvalidator.h"
+#include <mutex>
+#include <condition_variable>
+
+namespace proton
+{
+
+/**
+ * Track document db main state and validate that state transitions follow
+ * legal edges.
+ *
+ * Note that SHUTDOWN state can be entered from almost any state.
+ */
+
+class DDBState
+{
+public:
+    enum class State
+    {
+        CONSTRUCT,
+        LOAD,
+        REPLAY_TRANSACTION_LOG,
+        REDO_REPROCESS,
+        APPLY_LIVE_CONFIG,
+        REPROCESS,
+        ONLINE,
+        SHUTDOWN,
+        DEAD
+    };
+
+    enum class ConfigState
+    {
+        OK,
+        NEED_RESTART,
+        REJECT
+    };
+private:
+
+    State          _state;
+    ConfigState    _configState;
+
+    using Mutex = std::mutex;
+    using Guard = std::lock_guard<Mutex>;
+    using GuardLock = std::unique_lock<Mutex>;
+    Mutex     _lock;  // protects state transition
+    std::condition_variable       _cond;
+
+    static std::vector<vespalib::string> _stateNames;
+    static std::vector<vespalib::string> _configStateNames;
+
+public:
+    DDBState();
+
+    ~DDBState();
+
+    /**
+     * Try to enter LOAD state.  Fail and return false if document db is
+     * being shut down.
+     */ 
+    bool
+    enterLoadState();
+    
+    bool
+    enterReplayTransactionLogState();
+
+    bool
+    enterReplaySpoolerState();
+
+    bool
+    enterRedoReprocessState();
+
+    bool
+    enterApplyLiveConfigState();
+
+    bool
+    enterReprocessState();
+
+    bool
+    enterOnlineState();
+
+    void
+    enterShutdownState();
+
+    void
+    enterDeadState();
+
+    State
+    getState(void) const
+    {
+        return _state;
+    }
+
+    static vespalib::string
+    getStateString(State state);
+    
+    bool
+    getClosed(void) const
+    {
+        State state(_state);
+        return state >= State::SHUTDOWN;
+    }
+
+    bool
+    getAllowReconfig(void) const
+    {
+        State state(_state);
+        return state >= State::APPLY_LIVE_CONFIG && state < State::SHUTDOWN;
+    }
+
+    bool
+    getAllowPrune(void) const
+    {
+        State state(_state);
+        return state == State::ONLINE;
+    }
+    
+    static bool
+    getRejectedConfig(ConfigState state)
+    {
+        return state != ConfigState::OK;
+    }
+    
+    bool
+    getRejectedConfig() const
+    {
+        ConfigState state(_configState);
+        return getRejectedConfig(state);
+    }
+
+    static bool
+    isFeedBlockedByRejectedConfig(ConfigState state)
+    {
+        return state == ConfigState::REJECT;
+    }
+    
+    bool
+    isFeedBlockedByRejectedConfig() const
+    {
+        ConfigState state(_configState);
+        return isFeedBlockedByRejectedConfig(state);
+    }
+    
+    void
+    clearRejectedConfig();
+
+    ConfigState
+    getConfigState() const
+    {
+        return _configState;
+    }
+
+    static vespalib::string
+    getConfigStateString(ConfigState configState);
+
+    void
+    setConfigState(ConfigState newConfigState);
+
+    ConfigState
+    calcConfigState(const ConfigValidator::ResultType &cvr);
+
+    void
+    waitForOnlineState();
+};
+
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
new file mode 100644
index 00000000000..ddcd038077e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
@@ -0,0 +1,182 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "disk_mem_usage_filter.h"
+#include <sstream>
+
+namespace proton {
+
+namespace {
+
+void
+makeMemoryLimitMessage(std::ostream &os,
+                       double memoryUsed,
+                       double memoryLimit,
+                       const vespalib::ProcessMemoryStats &memoryStats,
+                       uint64_t physicalMemory)
+{
+    os << "memoryLimitReached: { "
+            "action: \"add more content nodes\", "
+            "reason: \""
+            "memory used (" << memoryUsed << ") > "
+            "memory limit (" << memoryLimit << ")"
+            "\", mapped: { virt: " <<
+            memoryStats.getMappedVirt() << ", rss: " <<
+            memoryStats.getMappedRss() << "}, anonymous: { virt: " <<
+            memoryStats.getAnonymousVirt() << ", rss: " <<
+            memoryStats.getAnonymousRss() << "}, physicalMemory: " <<
+            physicalMemory << ", memoryLimit : " <<
+            memoryLimit << "}";
+}
+
+void
+makeDiskLimitMessage(std::ostream &os,
+                     double diskUsed,
+                     double diskLimit,
+                     const DiskMemUsageFilter::space_info &diskStats)
+{
+    os << "diskLimitReached: { "
+            "action: \"add more content nodes\", "
+            "reason: \""
+            "disk used (" << diskUsed << ") > "
+            "disk limit (" << diskLimit << ")"
+            "\", capacity: " <<
+            diskStats.capacity << ", free: " <<
+            diskStats.free << ", available: " <<
+            diskStats.available << ", diskLimit: " <<
+            diskLimit << "}";
+}
+
+}
+
+void
+DiskMemUsageFilter::recalcState(const Guard &guard)
+{
+    (void) guard;
+    bool hasMessage = false;
+    std::ostringstream message;
+    double memoryUsed = getMemoryUsedRatio(guard);
+    if (memoryUsed > _config._memoryLimit) {
+        hasMessage = true;
+        makeMemoryLimitMessage(message, memoryUsed,
+                _config._memoryLimit, _memoryStats, _physicalMemory);
+    }
+    double diskUsed = getDiskUsedRatio(guard);
+    if (diskUsed > _config._diskLimit) {
+        if (hasMessage) {
+            message << ", ";
+        }
+        hasMessage = true;
+        makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _diskStats);
+    }
+    if (hasMessage) {
+        _state = State(false, message.str());
+        _acceptWrite = false;
+    } else {
+        _state = State();
+        _acceptWrite = true;
+    }
+}
+
+double
+DiskMemUsageFilter::getMemoryUsedRatio(const Guard &guard) const
+{
+    (void) guard;
+    uint64_t unscaledMemoryUsed = _memoryStats.getAnonymousRss();
+    return static_cast<double>(unscaledMemoryUsed) / _physicalMemory;
+}
+
+double
+DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const
+{
+    (void) guard;
+    double availableDiskSpaceRatio = static_cast<double>(_diskStats.available) /
+                                     static_cast<double>(_diskStats.capacity);
+    return 1.0 - availableDiskSpaceRatio;
+}
+
+DiskMemUsageFilter::DiskMemUsageFilter(uint64_t physicalMemory_in)
+    : _lock(),
+      _memoryStats(),
+      _physicalMemory(physicalMemory_in),
+      _diskStats(),
+      _config(),
+      _state(),
+      _acceptWrite(true)
+{
+}
+
+
+void
+DiskMemUsageFilter::setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in)
+{
+    Guard guard(_lock);
+    _memoryStats = memoryStats_in;
+    recalcState(guard);
+}
+
+void
+DiskMemUsageFilter::setDiskStats(space_info diskStats_in)
+{
+    Guard guard(_lock);
+    _diskStats = diskStats_in;
+    recalcState(guard);
+}
+
+void
+DiskMemUsageFilter::setConfig(Config config_in)
+{
+    Guard guard(_lock);
+    _config = config_in;
+    recalcState(guard);
+}
+
+vespalib::ProcessMemoryStats
+DiskMemUsageFilter::getMemoryStats() const
+{
+    Guard guard(_lock);
+    return _memoryStats;
+}
+
+DiskMemUsageFilter::space_info
+DiskMemUsageFilter::getDiskStats() const
+{
+    Guard guard(_lock);
+    return _diskStats;
+}
+
+DiskMemUsageFilter::Config
+DiskMemUsageFilter::getConfig() const
+{
+    Guard guard(_lock);
+    return _config;
+}
+
+double
+DiskMemUsageFilter::getMemoryUsedRatio() const
+{
+    Guard guard(_lock);
+    return getMemoryUsedRatio(guard);
+}
+
+double
+DiskMemUsageFilter::getDiskUsedRatio() const
+{
+    Guard guard(_lock);
+    return getDiskUsedRatio(guard);
+}
+
+bool
+DiskMemUsageFilter::acceptWriteOperation() const
+{
+    return _acceptWrite;
+}
+
+DiskMemUsageFilter::State
+DiskMemUsageFilter::getAcceptState() const
+{
+    Guard guard(_lock);
+    return _state;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
new file mode 100644
index 00000000000..1879e7b9385
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <boost/filesystem.hpp>
+#include <vespa/vespalib/util/process_memory_stats.h>
+#include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
+#include <mutex>
+#include <atomic>
+
+namespace proton {
+
+/*
+ * Class to filter write operations based on sampled disk and memory
+ * usage. If resource limit is reached then further writes are denied
+ * in order to prevent entering an unrecoverable state.
+ */
+class DiskMemUsageFilter : public IResourceWriteFilter {
+public:
+    using space_info = boost::filesystem::space_info;
+    using Mutex = std::mutex;
+    using Guard = std::lock_guard<Mutex>;
+
+    struct Config
+    {
+        double _memoryLimit;
+        double _diskLimit;
+
+        Config()
+            : _memoryLimit(1.0),
+              _diskLimit(1.0)
+        {
+        }
+
+        Config(double memoryLimit_in, double diskLimit_in)
+            : _memoryLimit(memoryLimit_in),
+              _diskLimit(diskLimit_in)
+        {
+        }
+    };
+
+private:
+    mutable Mutex _lock; // protect _memoryStats, _diskStats, _config, _state
+    vespalib::ProcessMemoryStats _memoryStats;
+    uint64_t _physicalMemory;
+    space_info _diskStats;
+    Config _config;
+    State _state;
+    std::atomic<bool> _acceptWrite;
+
+    void recalcState(const Guard &guard); // called with _lock held
+    double getMemoryUsedRatio(const Guard &guard) const;
+    double getDiskUsedRatio(const Guard &guard) const;
+
+public:
+    DiskMemUsageFilter(uint64_t physicalMememory_in);
+    void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in);
+    void setDiskStats(space_info diskStats_in);
+    void setConfig(Config config);
+    vespalib::ProcessMemoryStats getMemoryStats() const;
+    space_info getDiskStats() const;
+    Config getConfig() const;
+    uint64_t getPhysicalMemory() const { return _physicalMemory; }
+    double getMemoryUsedRatio() const;
+    double getDiskUsedRatio() const;
+    virtual bool acceptWriteOperation() const override;
+    virtual State getAcceptState() const override;
+};
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
new file mode 100644
index 00000000000..b66ec26bd05
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "disk_mem_usage_sampler.h"
+#include <vespa/vespalib/util/timer.h>
+#include <vespa/searchlib/common/lambdatask.h>
+
+using search::makeLambdaTask;
+
+namespace proton {
+
+namespace {
+
+uint64_t getPhysicalMemory() {
+    return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
+}
+
+} // namespace proton:<anonymous>
+
+DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in,
+                                         const Config &config)
+    : _filter(getPhysicalMemory()),
+      _path(path_in),
+      _sampleInterval(60.0),
+      _periodicTimer()
+{
+    setConfig(config);
+}
+
+DiskMemUsageSampler::~DiskMemUsageSampler()
+{
+    _periodicTimer.reset();
+}
+
+void
+DiskMemUsageSampler::setConfig(const Config &config)
+{
+    _periodicTimer.reset();
+    _filter.setConfig(config._filterConfig);
+    _sampleInterval = config._sampleInterval;
+    sampleUsage();
+    _periodicTimer = std::make_unique<vespalib::Timer>();
+    _periodicTimer->scheduleAtFixedRate(makeLambdaTask([this]()
+                                                       { sampleUsage(); }),
+                                        _sampleInterval, _sampleInterval);
+}
+
+void
+DiskMemUsageSampler::sampleUsage()
+{
+    sampleMemoryUsage();
+    sampleDiskUsage();
+}
+
+void
+DiskMemUsageSampler::sampleDiskUsage()
+{
+    _filter.setDiskStats(boost::filesystem::space(_path));
+}
+
+void
+DiskMemUsageSampler::sampleMemoryUsage()
+{
+    _filter.setMemoryStats(vespalib::ProcessMemoryStats::create());
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
new file mode 100644
index 00000000000..89dab8c4ff8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "disk_mem_usage_filter.h"
+#include <boost/filesystem.hpp>
+
+namespace vespalib { class Timer; }
+
+namespace proton {
+
+/*
+ * Class to sample disk and memory usage used for filtering write operations.
+ */
+class DiskMemUsageSampler {
+    DiskMemUsageFilter _filter;
+    boost::filesystem::path _path;
+    double _sampleInterval;
+    std::unique_ptr<vespalib::Timer> _periodicTimer;
+
+    void sampleUsage();
+    void sampleDiskUsage();
+    void sampleMemoryUsage();
+public:
+    struct Config {
+        DiskMemUsageFilter::Config _filterConfig;
+        double _sampleInterval;
+    public:
+        Config()
+            : _filterConfig(),
+              _sampleInterval(60.0)
+        {
+        }
+
+        Config(double memoryLimit_in, double diskLimit_in,
+               double sampleInterval_in)
+            : _filterConfig(memoryLimit_in, diskLimit_in),
+              _sampleInterval(sampleInterval_in)
+        {
+        }
+    };
+
+    DiskMemUsageSampler(const std::string &path_in,
+                        const Config &config);
+
+    ~DiskMemUsageSampler();
+
+    void setConfig(const Config &config);
+
+    const DiskMemUsageFilter &writeFilter() const { return _filter; }
+};
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
new file mode 100644
index 00000000000..8e402c7e3b9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
@@ -0,0 +1,101 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.docstorevalidator");
+#include "docstorevalidator.h"
+
+namespace proton
+{
+
+DocStoreValidator::DocStoreValidator(IDocumentMetaStore &dms)
+    : _dms(dms),
+      _docIdLimit(dms.getCommittedDocIdLimit()),
+      _invalid(search::BitVector::create(_docIdLimit)),
+      _orphans(search::BitVector::create(_docIdLimit)),
+      _visitCount(0u),
+      _visitEmptyCount(0u)
+{
+    for (uint32_t lid = 1; lid < _docIdLimit; ++lid) {
+        if (_dms.validLid(lid)) {
+            _invalid->setBit(lid);
+        }
+    }
+}
+
+
+void
+DocStoreValidator::visit(uint32_t lid, const document::Document &doc)
+{
+    if (lid == 0 || lid >= _docIdLimit)
+        return;
+    ++_visitCount;
+    if (!_dms.validLid(lid)) {
+        _orphans->setBit(lid);
+        return;
+    }
+    const document::DocumentId &docId(doc.getId());
+    const document::GlobalId &gid = docId.getGlobalId();
+    const RawDocumentMetaData &meta = _dms.getRawMetaData(lid);
+    const document::GlobalId &dmsGid = meta.getGid();
+    if (gid == dmsGid) {
+        _invalid->clearBit(lid);
+    } else {
+        _invalid->setBit(lid);
+    }
+}
+
+
+void
+DocStoreValidator::visit(uint32_t lid)
+{
+    if (lid == 0 || lid >= _docIdLimit)
+        return;
+    ++_visitEmptyCount;
+    if (!_dms.validLid(lid)) {
+        _orphans->clearBit(lid);
+        return;
+    }
+    _invalid->setBit(lid);
+}
+
+
+void
+DocStoreValidator::visitDone(void)
+{
+    _invalid->invalidateCachedCount();
+    _orphans->invalidateCachedCount();
+    (void) _invalid->countTrueBits();
+    (void) _orphans->countTrueBits();
+}
+
+
+void
+DocStoreValidator::killOrphans(search::IDocumentStore &store,
+                               search::SerialNum serialNum)
+{
+    for (uint32_t lid = 1; lid < _docIdLimit; ++lid) {
+        if (_orphans->testBit(lid)) {
+            assert(!_dms.validLid(lid));
+            store.remove(serialNum, lid);
+        }
+    }
+}
+
+
+LidVectorContext::LP
+DocStoreValidator::getInvalidLids(void) const
+{
+    LidVectorContext::LP res(new LidVectorContext(_docIdLimit));
+    assert(_invalid->size() == _docIdLimit);
+    for (search::DocumentIdT lid(_invalid->getFirstTrueBit(1));
+         lid < _docIdLimit;
+         lid = _invalid->getNextTrueBit(lid + 1)) {
+
+        res->addLid(lid);
+    }
+    return res;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h
new file mode 100644
index 00000000000..6fbaefb9088
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
+
+namespace proton
+{
+
+class DocStoreValidator : public search::IDocumentStoreReadVisitor
+{
+    IDocumentMetaStore         &_dms;
+    uint32_t                   _docIdLimit;
+    search::BitVector::UP  _invalid;
+    search::BitVector::UP  _orphans;
+    uint32_t                   _visitCount;
+    uint32_t                   _visitEmptyCount;
+    
+public:
+    DocStoreValidator(IDocumentMetaStore &dms);
+
+    virtual void
+    visit(uint32_t lid, const document::Document &doc);
+
+    virtual void
+    visit(uint32_t lid);
+
+    void
+    visitDone(void);
+
+    void
+    killOrphans(search::IDocumentStore &store,
+                search::SerialNum serialNum);
+
+    uint32_t
+    getInvalidCount(void) const
+    {
+        return _invalid->countTrueBits();
+    }
+
+    uint32_t
+    getOrphanCount(void) const
+    {
+        return _orphans->countTrueBits();
+    }
+
+    uint32_t
+    getVisitCount(void) const
+    {
+        return _visitCount;
+    }
+
+    uint32_t
+    getVisitEmptyCount(void) const
+    {
+        return _visitEmptyCount;
+    }
+
+    LidVectorContext::LP
+    getInvalidLids(void) const;
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.cpp
new file mode 100644
index 00000000000..0c03604ae72
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.cpp
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.document_db_explorer");
+#include "document_db_explorer.h"
+
+#include "document_meta_store_read_guards.h"
+#include "document_subdb_collection_explorer.h"
+#include "maintenance_controller_explorer.h"
+#include <vespa/searchcore/proton/common/state_reporter_utils.h>
+#include <vespa/searchcore/proton/bucketdb/bucket_db_explorer.h>
+#include <vespa/searchcore/proton/matching/session_manager_explorer.h>
+
+using vespalib::StateExplorer;
+using namespace vespalib::slime;
+
+namespace proton {
+
+DocumentDBExplorer::DocumentDBExplorer(const DocumentDB::SP &docDb)
+    : _docDb(docDb)
+{
+}
+
+void
+DocumentDBExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    (void) full;
+    Cursor &object = inserter.insertObject();
+    object.setString("documentType", _docDb->getDocTypeName().toString());
+    {
+        StateReporterUtils::convertToSlime(*_docDb->reportStatus(), ObjectInserter(object, "status"));
+    }
+    {
+        // TODO(geirst): Avoid const cast by adding const interface to
+        // IDocumentMetaStoreContext as seen from IDocumentSubDB.
+        DocumentMetaStoreReadGuards dmss
+            (const_cast<DocumentSubDBCollection &>(_docDb->getDocumentSubDBs()));
+        Cursor &documents = object.setObject("documents");
+        documents.setLong("active", dmss.numActiveDocs());
+        documents.setLong("indexed", dmss.numIndexedDocs());
+        documents.setLong("stored", dmss.numStoredDocs());
+        documents.setLong("removed", dmss.numRemovedDocs());
+    }
+}
+
+const vespalib::string SUB_DB = "subdb";
+const vespalib::string BUCKET_DB = "bucketdb";
+const vespalib::string MAINTENANCE_CONTROLLER = "maintenancecontroller";
+const vespalib::string SESSION = "session";
+
+std::vector<vespalib::string>
+DocumentDBExplorer::get_children_names() const
+{
+    return {SUB_DB, BUCKET_DB, MAINTENANCE_CONTROLLER, SESSION};
+}
+
+std::unique_ptr<StateExplorer>
+DocumentDBExplorer::get_child(vespalib::stringref name) const
+{
+    if (name == SUB_DB) {
+        return std::unique_ptr<StateExplorer>
+            (new DocumentSubDBCollectionExplorer(_docDb->getDocumentSubDBs()));
+    } else if (name == BUCKET_DB) {
+        // TODO(geirst): const_cast can be avoided if we add const guard to BucketDBOwner.
+        return std::unique_ptr<StateExplorer>(new BucketDBExplorer(
+            (const_cast<DocumentSubDBCollection &>(_docDb->getDocumentSubDBs())).getBucketDB().takeGuard()));
+    } else if (name == MAINTENANCE_CONTROLLER) {
+        return std::unique_ptr<StateExplorer>
+            (new MaintenanceControllerExplorer(_docDb->getMaintenanceController().getJobList()));
+    } else if (name == SESSION) {
+        return std::unique_ptr<StateExplorer>
+            (new matching::SessionManagerExplorer(_docDb->session_manager()));
+    }
+    return std::unique_ptr<StateExplorer>(nullptr);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.h b/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.h
new file mode 100644
index 00000000000..a888c6df8c8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.h
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentdb.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of a document database and its components.
+ */
+class DocumentDBExplorer : public vespalib::StateExplorer
+{
+private:
+    DocumentDB::SP _docDb;
+
+public:
+    DocumentDBExplorer(const DocumentDB::SP &docDb);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+    virtual std::vector<vespalib::string> get_children_names() const override;
+    virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp
new file mode 100644
index 00000000000..d821753d63e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp
@@ -0,0 +1,121 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "document_db_maintenance_config.h"
+
+namespace proton {
+
+DocumentDBPruneConfig::
+DocumentDBPruneConfig(void)
+    : _interval(21600.0),
+      _age(1209600.0)
+{
+}
+
+DocumentDBPruneConfig::
+DocumentDBPruneConfig(double interval,
+                      double age)
+    : _interval(interval),
+      _age(age)
+{
+}
+
+bool
+DocumentDBPruneConfig::
+operator==(const DocumentDBPruneConfig &rhs) const
+{
+    return _interval == rhs._interval &&
+                _age == rhs._age;
+}
+
+DocumentDBHeartBeatConfig::DocumentDBHeartBeatConfig(void)
+    : _interval(60.0)
+{
+}
+
+DocumentDBHeartBeatConfig::DocumentDBHeartBeatConfig(double interval)
+    : _interval(interval)
+{
+}
+
+bool
+DocumentDBHeartBeatConfig::
+operator==(const DocumentDBHeartBeatConfig &rhs) const
+{
+    return _interval == rhs._interval;
+}
+
+DocumentDBLidSpaceCompactionConfig::DocumentDBLidSpaceCompactionConfig()
+    : _interval(3600),
+      _allowedLidBloat(1000000000),
+      _allowedLidBloatFactor(1.0),
+      _maxDocsToScan(10000)
+{
+}
+
+DocumentDBLidSpaceCompactionConfig::DocumentDBLidSpaceCompactionConfig(double interval,
+                                                                       uint32_t allowedLidBloat,
+                                                                       double allowedLidBloatFactor,
+                                                                       uint32_t maxDocsToScan)
+    : _interval(interval),
+      _allowedLidBloat(allowedLidBloat),
+      _allowedLidBloatFactor(allowedLidBloatFactor),
+      _maxDocsToScan(maxDocsToScan)
+{
+}
+
+bool
+DocumentDBLidSpaceCompactionConfig::operator==(const DocumentDBLidSpaceCompactionConfig &rhs) const
+{
+   return _interval == rhs._interval &&
+           _allowedLidBloat == rhs._allowedLidBloat &&
+           _allowedLidBloatFactor == rhs._allowedLidBloatFactor;
+}
+
+DocumentDBMaintenanceConfig::DocumentDBMaintenanceConfig(void)
+    : _pruneRemovedDocuments(),
+      _heartBeat(),
+      _sessionCachePruneInterval(900.0),
+      _visibilityDelay(0),
+      _lidSpaceCompaction(),
+      _attributeUsageFilterConfig(),
+      _attributeUsageSampleInterval(60.0)
+{
+}
+
+DocumentDBMaintenanceConfig::
+DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig &
+                            pruneRemovedDocuments,
+                            const DocumentDBHeartBeatConfig &heartBeat,
+                            const DocumentDBWipeOldRemovedFieldsConfig &
+                            wipeOldRemovedFields,
+                            double groupingSessionPruneInterval,
+                            fastos::TimeStamp visibilityDelay,
+                            const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction,
+                            const AttributeUsageFilterConfig &attributeUsageFilterConfig,
+                            double attributeUsageSampleInterval)
+    : _pruneRemovedDocuments(pruneRemovedDocuments),
+      _heartBeat(heartBeat),
+      _wipeOldRemovedFields(wipeOldRemovedFields),
+      _sessionCachePruneInterval(groupingSessionPruneInterval),
+      _visibilityDelay(visibilityDelay),
+      _lidSpaceCompaction(lidSpaceCompaction),
+      _attributeUsageFilterConfig(attributeUsageFilterConfig),
+      _attributeUsageSampleInterval(attributeUsageSampleInterval)
+{
+}
+
+bool
+DocumentDBMaintenanceConfig::
+operator==(const DocumentDBMaintenanceConfig &rhs) const
+{
+    return _pruneRemovedDocuments == rhs._pruneRemovedDocuments &&
+        _heartBeat == rhs._heartBeat &&
+        _wipeOldRemovedFields == rhs._wipeOldRemovedFields &&
+        _sessionCachePruneInterval == rhs._sessionCachePruneInterval &&
+        _visibilityDelay == rhs._visibilityDelay &&
+        _lidSpaceCompaction == rhs._lidSpaceCompaction &&
+        _attributeUsageFilterConfig == rhs._attributeUsageFilterConfig &&
+        _attributeUsageSampleInterval == rhs._attributeUsageSampleInterval;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h
new file mode 100644
index 00000000000..b7deb914a00
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h
@@ -0,0 +1,133 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <vector>
+#include <vespa/searchcore/proton/attribute/attribute_usage_filter_config.h>
+
+namespace proton {
+
+class DocumentDBPruneConfig
+{
+private:
+    double _interval;
+    double _age;
+
+public:
+    DocumentDBPruneConfig(void);
+    DocumentDBPruneConfig(double interval, double age);
+
+    bool operator==(const DocumentDBPruneConfig &rhs) const;
+    double getInterval(void) const { return _interval; }
+    double getAge(void) const { return _age; }
+};
+
+typedef DocumentDBPruneConfig DocumentDBPruneRemovedDocumentsConfig;
+typedef DocumentDBPruneConfig DocumentDBWipeOldRemovedFieldsConfig;
+
+class DocumentDBHeartBeatConfig
+{
+private:
+    double _interval;
+
+public:
+    DocumentDBHeartBeatConfig(void);
+    DocumentDBHeartBeatConfig(double interval);
+
+    bool operator==(const DocumentDBHeartBeatConfig &rhs) const;
+    double getInterval(void) const { return _interval; }
+};
+
+class DocumentDBLidSpaceCompactionConfig
+{
+private:
+    double   _interval;
+    uint32_t _allowedLidBloat;
+    double   _allowedLidBloatFactor;
+    uint32_t _maxDocsToScan;
+
+public:
+    DocumentDBLidSpaceCompactionConfig();
+    DocumentDBLidSpaceCompactionConfig(double interval,
+                                       uint32_t allowedLidBloat,
+                                       double allowwedLidBloatFactor,
+                                       uint32_t maxDocsToScan = 10000);
+
+    bool operator==(const DocumentDBLidSpaceCompactionConfig &rhs) const;
+    double getInterval() const { return _interval; }
+    uint32_t getAllowedLidBloat() const { return _allowedLidBloat; }
+    double getAllowedLidBloatFactor() const { return _allowedLidBloatFactor; }
+    uint32_t getMaxDocsToScan() const { return _maxDocsToScan; }
+};
+
+class DocumentDBMaintenanceConfig
+{
+public:
+    typedef std::shared_ptr<DocumentDBMaintenanceConfig> SP;
+
+private:
+    DocumentDBPruneRemovedDocumentsConfig _pruneRemovedDocuments;
+    DocumentDBHeartBeatConfig             _heartBeat;
+    DocumentDBWipeOldRemovedFieldsConfig  _wipeOldRemovedFields;
+    double                                _sessionCachePruneInterval;
+    fastos::TimeStamp                     _visibilityDelay;
+    DocumentDBLidSpaceCompactionConfig    _lidSpaceCompaction;
+    AttributeUsageFilterConfig            _attributeUsageFilterConfig;
+    double                                _attributeUsageSampleInterval;
+
+public:
+    DocumentDBMaintenanceConfig(void);
+
+    DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig &pruneRemovedDocuments,
+                                const DocumentDBHeartBeatConfig &heartBeat,
+                                const DocumentDBWipeOldRemovedFieldsConfig &wipeOldRemovedFields,
+                                double sessionCachePruneInterval,
+                                fastos::TimeStamp visibilityDelay,
+                                const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction,
+                                const AttributeUsageFilterConfig &attributeUsageFilterConfig,
+                                double attributeUsageSampleInterval);
+
+    bool
+    operator==(const DocumentDBMaintenanceConfig &rhs) const;
+
+    const DocumentDBPruneRemovedDocumentsConfig &
+    getPruneRemovedDocumentsConfig(void) const
+    {
+        return _pruneRemovedDocuments;
+    }
+
+    const DocumentDBHeartBeatConfig &
+    getHeartBeatConfig(void) const
+    {
+        return _heartBeat;
+    }
+
+    const DocumentDBWipeOldRemovedFieldsConfig &
+    getWipeOldRemovedFieldsConfig() const
+    {
+        return _wipeOldRemovedFields;
+    }
+
+    double
+    getSessionCachePruneInterval() const
+    {
+        return _sessionCachePruneInterval;
+    }
+
+    fastos::TimeStamp getVisibilityDelay() const { return _visibilityDelay; }
+
+    const DocumentDBLidSpaceCompactionConfig &getLidSpaceCompactionConfig() const {
+        return _lidSpaceCompaction;
+    }
+
+    const AttributeUsageFilterConfig &getAttributeUsageFilterConfig() const {
+        return _attributeUsageFilterConfig;
+    }
+
+    double getAttributeUsageSampleInterval() const {
+        return _attributeUsageSampleInterval;
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.h b/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.h
new file mode 100644
index 00000000000..b2e845d221a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_meta_store_read_guards.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentsubdbcollection.h"
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h>
+
+namespace proton {
+
+/**
+ * Class that takes and owns read guards of the document meta stores of the 3 sub databases.
+ * Provides stats regarding the number of documents in the sub databases.
+ */
+struct DocumentMetaStoreReadGuards
+{
+    IDocumentMetaStoreContext::IReadGuard::UP readydms;
+    IDocumentMetaStoreContext::IReadGuard::UP notreadydms;
+    IDocumentMetaStoreContext::IReadGuard::UP remdms;
+    DocumentMetaStoreReadGuards(DocumentSubDBCollection &subDBs)
+        : readydms(subDBs.getReadySubDB()->getDocumentMetaStoreContext().getReadGuard()),
+          notreadydms(subDBs.getNotReadySubDB()->getDocumentMetaStoreContext().getReadGuard()),
+          remdms(subDBs.getRemSubDB()->getDocumentMetaStoreContext().getReadGuard())
+    {
+    }
+    uint32_t numActiveDocs() const {
+        return readydms->get().getNumActiveLids();
+    }
+    uint32_t numIndexedDocs() const {
+        return readydms->get().getNumUsedLids();
+    }
+    uint32_t numStoredDocs() const {
+        return numIndexedDocs() + notreadydms->get().getNumUsedLids();
+    }
+    uint32_t numRemovedDocs() const {
+        return remdms->get().getNumUsedLids();
+    }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp
new file mode 100644
index 00000000000..d5777d3c723
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.document_scan_iterator");
+
+#include "document_scan_iterator.h"
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+
+using search::DocumentMetaData;
+
+namespace proton {
+
+typedef IDocumentMetaStore::Iterator Iterator;
+
+DocumentScanIterator::DocumentScanIterator(const IDocumentMetaStore &metaStore)
+    : _metaStore(metaStore),
+      _lastGid(),
+      _lastGidValid(false),
+      _itrValid(true)
+{
+}
+
+bool
+DocumentScanIterator::valid() const
+{
+    return _itrValid;
+}
+
+DocumentMetaData
+DocumentScanIterator::next(uint32_t compactLidLimit,
+                           uint32_t maxDocsToScan,
+                           bool retry)
+{
+    Iterator itr = (_lastGidValid ?
+            (retry ? _metaStore.lowerBound(_lastGid) : _metaStore.upperBound(_lastGid))
+                    : _metaStore.begin());
+    uint32_t i = 1; // We have already 'scanned' a document when creating the iterator
+    for (; i < maxDocsToScan && itr.valid() && itr.getKey() <= compactLidLimit; ++i, ++itr) {}
+    if (itr.valid()) {
+        uint32_t lid = itr.getKey();
+        const RawDocumentMetaData &metaData = _metaStore.getRawMetaData(lid);
+        _lastGid = metaData.getGid();
+        _lastGidValid = true;
+        if (lid > compactLidLimit) {
+            return DocumentMetaData(lid, metaData.getTimestamp(),
+                    metaData.getBucketId(), metaData.getGid());
+        }
+    } else {
+        _itrValid = false;
+    }
+    return DocumentMetaData();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h
new file mode 100644
index 00000000000..eed523ed058
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_document_scan_iterator.h"
+
+namespace proton {
+
+class IDocumentMetaStore;
+
+/**
+ * Iterator for scanning all documents in a document meta store.
+ */
+class DocumentScanIterator : public IDocumentScanIterator
+{
+private:
+    const IDocumentMetaStore &_metaStore;
+    document::GlobalId        _lastGid;
+    bool                      _lastGidValid;
+    bool                      _itrValid;
+
+public:
+    DocumentScanIterator(const IDocumentMetaStore &_metaStore);
+
+    // Implements IDocumentScanIterator
+    virtual bool valid() const;
+
+    virtual search::DocumentMetaData next(uint32_t compactLidLimit,
+                                          uint32_t maxDocsToScan,
+                                          bool retry);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.cpp
new file mode 100644
index 00000000000..fffb40d1cdf
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.cpp
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.document_subdb_collection_explorer");
+#include "document_subdb_collection_explorer.h"
+
+#include "document_subdb_explorer.h"
+
+using vespalib::slime::Inserter;
+
+namespace proton {
+
+const vespalib::string READY = "ready";
+const vespalib::string REMOVED = "removed";
+const vespalib::string NOT_READY = "notready";
+
+namespace {
+
+std::unique_ptr<vespalib::StateExplorer>
+createExplorer(const IDocumentSubDB &subDb)
+{
+    return std::unique_ptr<vespalib::StateExplorer>(new DocumentSubDBExplorer(subDb));
+}
+
+}
+
+DocumentSubDBCollectionExplorer::DocumentSubDBCollectionExplorer(const DocumentSubDBCollection &subDbs)
+    : _subDbs(subDbs)
+{
+}
+
+void
+DocumentSubDBCollectionExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    // This is a transparent state where the short state of all children is rendered instead.
+    (void) inserter;
+    (void) full;
+}
+
+std::vector<vespalib::string>
+DocumentSubDBCollectionExplorer::get_children_names() const
+{
+    return {READY, REMOVED, NOT_READY};
+}
+
+std::unique_ptr<vespalib::StateExplorer>
+DocumentSubDBCollectionExplorer::get_child(vespalib::stringref name) const
+{
+    if (name == READY) {
+        return createExplorer(*_subDbs.getReadySubDB());
+    } else if (name == REMOVED) {
+        return createExplorer(*_subDbs.getRemSubDB());
+    } else if (name == NOT_READY) {
+        return createExplorer(*_subDbs.getNotReadySubDB());
+    }
+    return std::unique_ptr<vespalib::StateExplorer>();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.h
new file mode 100644
index 00000000000..b42f9e8c650
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentsubdbcollection.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of a collection of document sub databases.
+ */
+class DocumentSubDBCollectionExplorer : public vespalib::StateExplorer
+{
+private:
+    const DocumentSubDBCollection &_subDbs;
+
+public:
+    DocumentSubDBCollectionExplorer(const DocumentSubDBCollection &subDbs);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+    virtual std::vector<vespalib::string> get_children_names() const override;
+    virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.cpp
new file mode 100644
index 00000000000..d370a735a37
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.cpp
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "document_subdb_collection_initializer.h"
+
+namespace proton {
+
+DocumentSubDbCollectionInitializer::DocumentSubDbCollectionInitializer()
+    : _subDbInitializers()
+{
+}
+
+void
+DocumentSubDbCollectionInitializer::run()
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.h
new file mode 100644
index 00000000000..104c6902bf6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.h
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "document_subdb_initializer.h"
+
+namespace proton {
+
+/**
+ * Class used to initialize a collection of document sub databases.
+ */
+class DocumentSubDbCollectionInitializer : public initializer::InitializerTask
+{
+private:
+    std::vector<DocumentSubDbInitializer::SP> _subDbInitializers;
+
+public:
+    using SP = std::shared_ptr<DocumentSubDbCollectionInitializer>;
+
+    DocumentSubDbCollectionInitializer();
+    void add(const DocumentSubDbInitializer::SP subDbInitializer) {
+        _subDbInitializers.push_back(subDbInitializer);
+        addDependency(subDbInitializer);
+    }
+    virtual void run() override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp
new file mode 100644
index 00000000000..68d51e13f2c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.document_subdb_explorer");
+#include "document_subdb_explorer.h"
+
+#include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h>
+#include <vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h>
+#include <vespa/searchcore/proton/docsummary/document_store_explorer.h>
+#include <vespa/searchcorespi/index/index_manager_explorer.h>
+
+using searchcorespi::IndexManagerExplorer;
+using vespalib::slime::Inserter;
+using vespalib::StateExplorer;
+
+namespace proton {
+
+namespace {
+
+const vespalib::string DOCUMENT_META_STORE = "documentmetastore";
+const vespalib::string DOCUMENT_STORE = "documentstore";
+const vespalib::string ATTRIBUTE = "attribute";
+const vespalib::string INDEX = "index";
+
+}
+
+DocumentSubDBExplorer::DocumentSubDBExplorer(const IDocumentSubDB &subDb)
+    : _subDb(subDb)
+{
+}
+
+void
+DocumentSubDBExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    (void) full;
+    inserter.insertObject();
+}
+
+std::vector<vespalib::string>
+DocumentSubDBExplorer::get_children_names() const
+{
+    std::vector<vespalib::string> children = {DOCUMENT_META_STORE, DOCUMENT_STORE};
+    if (_subDb.getAttributeManager().get() != nullptr) {
+        children.push_back(ATTRIBUTE);
+    }
+    if (_subDb.getIndexManager().get() != nullptr) {
+        children.push_back(INDEX);
+    }
+    return children;
+}
+
+std::unique_ptr<StateExplorer>
+DocumentSubDBExplorer::get_child(vespalib::stringref name) const
+{
+    if (name == DOCUMENT_META_STORE) {
+        // TODO(geirst): Avoid const cast by adding const interface to
+        // IDocumentMetaStoreContext as seen from IDocumentSubDB.
+        return std::unique_ptr<StateExplorer>(new DocumentMetaStoreExplorer(
+                (const_cast<IDocumentSubDB &>(_subDb)).getDocumentMetaStoreContext().getReadGuard()));
+    } else if (name == DOCUMENT_STORE) {
+        return std::unique_ptr<StateExplorer>(new DocumentStoreExplorer(_subDb.getSummaryManager()));
+    } else if (name == ATTRIBUTE) {
+        proton::IAttributeManager::SP attrMgr = _subDb.getAttributeManager();
+        if (attrMgr.get() != nullptr) {
+            return std::unique_ptr<StateExplorer>(new AttributeManagerExplorer(attrMgr));
+        }
+    } else if (name == INDEX) {
+        searchcorespi::IIndexManager::SP idxMgr = _subDb.getIndexManager();
+        if (idxMgr.get() != nullptr) {
+            return std::unique_ptr<StateExplorer>(new IndexManagerExplorer(std::move(idxMgr)));
+        }
+    }
+    return std::unique_ptr<StateExplorer>();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.h
new file mode 100644
index 00000000000..4e8fc7e8176
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "idocumentsubdb.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of a document sub database.
+ */
+class DocumentSubDBExplorer : public vespalib::StateExplorer
+{
+private:
+    const IDocumentSubDB &_subDb;
+
+public:
+    DocumentSubDBExplorer(const IDocumentSubDB &subDb);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+    virtual std::vector<vespalib::string> get_children_names() const override;
+    virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp
new file mode 100644
index 00000000000..5f7e97cd5d5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "document_subdb_initializer.h"
+#include "idocumentsubdb.h"
+#include <future>
+#include <vespa/searchlib/common/lambdatask.h>
+
+using search::makeLambdaTask;
+
+namespace proton {
+
+DocumentSubDbInitializer::DocumentSubDbInitializer(IDocumentSubDB &subDB,
+                                                   searchcorespi::index::IThreadService &master)
+    : InitTask(),
+      _result(),
+      _documentMetaStoreInitTask(),
+      _subDB(subDB),
+      _master(master)
+{
+}
+
+void
+DocumentSubDbInitializer::
+addDocumentMetaStoreInitTask(InitTask::SP documentMetaStoreInitTask)
+{
+    assert(!_documentMetaStoreInitTask);
+    _documentMetaStoreInitTask = documentMetaStoreInitTask;
+    addDependency(documentMetaStoreInitTask);
+}
+
+void
+DocumentSubDbInitializer::run()
+{
+    std::promise<bool> promise;
+    std::future<bool> future = promise.get_future();
+    _master.execute(makeLambdaTask([&]() { _subDB.setup(_result); promise.set_value(true); }));
+    (void) future.get();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.h
new file mode 100644
index 00000000000..b417a0680df
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.h
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "document_subdb_initializer_result.h"
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+
+namespace searchcorespi { namespace index { class IThreadService; } }
+
+namespace proton {
+
+class IDocumentSubDB;
+
+/**
+ * Class used to initialize a set of components that is used by a document sub database.
+ *
+ * The initialization of components will typically happen in parallel to reduce startup times.
+ */
+class DocumentSubDbInitializer : public initializer::InitializerTask
+{
+private:
+    DocumentSubDbInitializerResult _result;
+    initializer::InitializerTask::SP _documentMetaStoreInitTask;
+    IDocumentSubDB                  &_subDB;
+    searchcorespi::index::IThreadService &_master;
+
+public:
+    using SP = std::shared_ptr<DocumentSubDbInitializer>;
+    using UP = std::unique_ptr<DocumentSubDbInitializer>;
+    using InitTask = initializer::InitializerTask;
+
+    DocumentSubDbInitializer(IDocumentSubDB &subDB,
+                             searchcorespi::index::IThreadService &master);
+    const DocumentSubDbInitializerResult &result() const {
+        return _result;
+    }
+
+    DocumentSubDbInitializerResult &writableResult() {
+        return _result;
+    }
+
+    void addDocumentMetaStoreInitTask(InitTask::SP documentMetaStoreInitTask);
+
+    InitTask::SP getDocumentMetaStoreInitTask() const {
+        return _documentMetaStoreInitTask;
+    }
+
+    virtual void run() override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp
new file mode 100644
index 00000000000..77e71169f09
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "document_subdb_initializer_result.h"
+
+using searchcorespi::IIndexManager;
+
+namespace proton {
+
+DocumentSubDbInitializerResult::DocumentSubDbInitializerResult()
+    : _documentMetaStore(std::make_shared<DocumentMetaStoreInitializerResult::SP>
+                         ()),
+      _summaryManager(std::make_shared<SummaryManager::SP>()),
+      _attributeManager(std::make_shared<AttributeManager::SP>()),
+      _indexManager(std::make_shared<IIndexManager::SP>()),
+      _lidReuseDelayerConfig()
+{
+}
+
+void
+DocumentSubDbInitializerResult::
+setLidReuseDelayerConfig(LidReuseDelayerConfig lidReuseDelayerConfig_in)
+{
+    _lidReuseDelayerConfig = lidReuseDelayerConfig_in;
+}
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h
new file mode 100644
index 00000000000..5d0a50a7f88
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include <vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h>
+#include <vespa/searchcorespi/index/iindexmanager.h>
+#include <vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h>
+
+namespace proton {
+
+/**
+ * The result after initializing components used by a document sub database.
+ *
+ * The document sub database takes ownership of these (initialized) components.
+ */
+class DocumentSubDbInitializerResult
+{
+private:
+    std::shared_ptr<DocumentMetaStoreInitializerResult::SP> _documentMetaStore;
+    std::shared_ptr<SummaryManager::SP> _summaryManager;
+    std::shared_ptr<AttributeManager::SP> _attributeManager;
+    std::shared_ptr<searchcorespi::IIndexManager::SP> _indexManager;
+    using LidReuseDelayerConfig = documentmetastore::LidReuseDelayerConfig;
+    LidReuseDelayerConfig _lidReuseDelayerConfig;
+
+public:
+    DocumentSubDbInitializerResult();
+
+    std::shared_ptr<DocumentMetaStoreInitializerResult::SP>
+    writableDocumentMetaStore() { return _documentMetaStore; }
+    DocumentMetaStoreInitializerResult::SP documentMetaStore() const {
+        return *_documentMetaStore;
+    }
+    std::shared_ptr<SummaryManager::SP> writableSummaryManager() {
+        return _summaryManager;
+    }
+    SummaryManager::SP summaryManager() const {
+        return *_summaryManager;
+    }
+    std::shared_ptr<AttributeManager::SP> writableAttributeManager() {
+        return _attributeManager;
+    }
+    AttributeManager::SP attributeManager() const {
+        return *_attributeManager;
+    }
+    std::shared_ptr<searchcorespi::IIndexManager::SP> writableIndexManager() {
+        return _indexManager;
+    }
+    searchcorespi::IIndexManager::SP indexManager() const {
+        return *_indexManager;
+    }
+    void setLidReuseDelayerConfig(LidReuseDelayerConfig
+                                  lidReuseDelayerConfig_in);
+    const LidReuseDelayerConfig &lidReuseDelayerConfig() const {
+        return _lidReuseDelayerConfig;
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp
new file mode 100644
index 00000000000..815bfc75a70
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp
@@ -0,0 +1,139 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.documentbucketmover");
+#include "documentbucketmover.h"
+#include "idocumentmovehandler.h"
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
+#include "maintenancedocumentsubdb.h"
+
+using document::BucketId;
+using document::Document;
+using document::GlobalId;
+using search::DocumentIdT;
+using storage::spi::Timestamp;
+
+namespace proton {
+
+typedef IDocumentMetaStore::Iterator Iterator;
+
+void
+DocumentBucketMover::moveDocument(DocumentIdT lid,
+                                  const document::GlobalId &gid,
+                                  Timestamp timestamp)
+{
+    Document::SP doc(_source->_retriever->getDocument(lid).release());
+    if (!doc || doc->getId().getGlobalId() != gid)
+        return; // Failed to retrieve document, removed or changed identity
+    // TODO(geirst): what if doc is NULL?
+    BucketId bucketId = _bucket.stripUnused();
+    MoveOperation op(bucketId, timestamp, doc, DbDocumentId(_source->_subDbId, lid), _targetSubDbId);
+
+    // We cache the bucket for the document we are going to move to avoid getting
+    // inconsistent bucket info (getBucketInfo()) while moving between ready and not-ready
+    // sub dbs as the bucket info is not updated atomically in this case.
+    _bucketDb->takeGuard()->cacheBucket(bucketId);
+    _handler->handleMove(op);
+    _bucketDb->takeGuard()->uncacheBucket();
+}
+
+
+DocumentBucketMover::DocumentBucketMover()
+    : _bucket(),
+      _source(nullptr),
+      _targetSubDbId(0),
+      _handler(nullptr),
+      _bucketDb(nullptr),
+      _bucketDone(true),
+      _lastGid(),
+      _lastGidValid(false)
+{ }
+
+
+void
+DocumentBucketMover::setupForBucket(const BucketId &bucket,
+                                    const MaintenanceDocumentSubDB *source,
+                                    uint32_t targetSubDbId,
+                                    IDocumentMoveHandler &handler,
+                                    BucketDBOwner &bucketDb)
+{
+    _bucket = bucket;
+    _source = source;
+    _targetSubDbId = targetSubDbId;
+    _handler = &handler;
+    _bucketDb = &bucketDb;
+    _bucketDone = false;
+    _lastGid = GlobalId();
+    _lastGidValid = false;
+}
+
+
+namespace
+{
+
+class MoveKey
+{
+public:
+    DocumentIdT _lid;
+    document::GlobalId _gid;
+    Timestamp _timestamp;
+
+    MoveKey(DocumentIdT lid,
+            const document::GlobalId &gid,
+            Timestamp timestamp)
+        : _lid(lid),
+          _gid(gid),
+          _timestamp(timestamp)
+    {
+    }
+};
+
+}
+
+void DocumentBucketMover::setBucketDone() {
+    _bucketDone = true;
+}
+
+void
+DocumentBucketMover::moveDocuments(size_t maxDocsToMove)
+{
+    if (_bucketDone) {
+        return;
+    }
+    Iterator itr = (_lastGidValid ? _source->_metaStore->upperBound(_lastGid)
+                    : _source->_metaStore->lowerBound(_bucket));
+    const Iterator end = _source->_metaStore->upperBound(_bucket);
+    size_t docsMoved = 0;
+    size_t docsSkipped = 0;	// In absence of a proper cost metric
+    typedef std::vector<MoveKey> MoveVec;
+    MoveVec toMove;
+    for (; itr != end && docsMoved < maxDocsToMove; ++itr) {
+        DocumentIdT lid = itr.getKey();
+        const RawDocumentMetaData &metaData = _source->_metaStore->getRawMetaData(lid);
+        if (metaData.getBucketUsedBits() != _bucket.getUsedBits()) {
+            ++docsSkipped;
+            if (docsSkipped >= 50) {
+                ++docsMoved; // In absence of a proper cost metric
+                docsSkipped = 0;
+            }
+        } else {
+            // moveDocument(lid, metaData.getTimestamp());
+            toMove.push_back(MoveKey(lid, metaData.getGid(), metaData.getTimestamp()));
+            ++docsMoved;
+        }
+        _lastGid = metaData.getGid();
+        _lastGidValid = true;
+    }
+    if (itr == end) {
+        setBucketDone();
+    }
+    for (const MoveKey & key : toMove) {
+        moveDocument(key._lid, key._gid, key._timestamp);
+    }
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h
new file mode 100644
index 00000000000..5f8b5c8fe45
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/base/globalid.h>
+#include <vespa/searchlib/query/base.h>
+#include <persistence/spi/types.h>
+#include "ifrozenbuckethandler.h"
+
+namespace proton {
+
+class BucketDBOwner;
+class IDocumentMoveHandler;
+class MaintenanceDocumentSubDB;
+
+/**
+ * Class used to move all documents in a bucket from a source sub database
+ * to a target sub database. The actual moving is handled by a given instance
+ * of IDocumentMoveHandler.
+ */
+class DocumentBucketMover
+{
+private:
+    document::BucketId        _bucket;
+    const MaintenanceDocumentSubDB *_source;
+    uint32_t                  _targetSubDbId;
+    IDocumentMoveHandler     *_handler;
+    BucketDBOwner            *_bucketDb;
+    bool                      _bucketDone;
+    document::GlobalId        _lastGid;
+    bool                      _lastGidValid;
+
+    void moveDocument(search::DocumentIdT lid,
+                      const document::GlobalId &gid,
+                      storage::spi::Timestamp timestamp);
+
+    void setBucketDone();
+public:
+    DocumentBucketMover();
+    void setupForBucket(const document::BucketId &bucket,
+                        const MaintenanceDocumentSubDB *source,
+                        uint32_t targetSubDbId,
+                        IDocumentMoveHandler &handler,
+                        BucketDBOwner &bucketDb);
+    const document::BucketId &getBucket() const { return _bucket; }
+    void moveDocuments(size_t maxDocsToMove);
+    void cancel() { setBucketDone(); }
+    bool bucketDone() const { return _bucketDone; }
+    const MaintenanceDocumentSubDB * getSource(void) const { return _source; }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
new file mode 100644
index 00000000000..eaa166afe9c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -0,0 +1,1498 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.documentdb");
+
+#include "documentdb.h"
+
+#include "combiningfeedview.h"
+#include "configvalidator.h"
+#include "document_meta_store_read_guards.h"
+#include "document_subdb_collection_explorer.h"
+#include "idocumentdbowner.h"
+#include "lid_space_compaction_handler.h"
+#include "maintenance_jobs_injector.h"
+#include <vespa/searchcore/proton/metrics/metricswireservice.h>
+#include "searchcontext.h"
+#include "summaryadapter.h"
+#include "tlcproxy.h"
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/searchcore/proton/common/schemautil.h>
+#include <vespa/searchcore/proton/feedoperation/newconfigoperation.h>
+#include <vespa/searchcore/proton/feedoperation/noopoperation.h>
+#include <vespa/searchcore/proton/feedoperation/wipehistoryoperation.h>
+#include <vespa/searchcore/proton/index/index_writer.h>
+#include <vespa/searchcore/proton/initializer/task_runner.h>
+#include <vespa/searchcore/proton/matching/matching_stats.h>
+#include <vespa/searchcore/proton/persistenceengine/bucket_guard.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/configconverter.h>
+#include <vespa/searchcommon/common/schemaconfigurer.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/searchlib/common/lambdatask.h>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/objects/objectvisitor.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/util/jsonwriter.h>
+#include <sstream>
+#include "documentdbconfigscout.h"
+
+
+using vespa::config::search::AttributesConfig;
+using vespa::config::search::core::ProtonConfig;
+using search::index::SchemaBuilder;
+using vespalib::JSONStringer;
+using vespalib::FileHeader;
+using vespalib::Executor;
+using vespalib::IllegalStateException;
+using vespalib::StateExplorer;
+using vespalib::make_string;
+using vespalib::makeTask;
+using vespalib::makeClosure;
+using namespace proton::matching;
+using namespace search;
+using namespace search::docsummary;
+using namespace search::engine;
+using namespace search::fef;
+using namespace search::index;
+using namespace search::transactionlog;
+using searchcorespi::index::IThreadService;
+using search::TuneFileDocumentDB;
+using storage::spi::Timestamp;
+using search::common::FileHeaderContext;
+using proton::initializer::InitializerTask;
+using proton::initializer::TaskRunner;
+using search::makeLambdaTask;
+
+namespace proton {
+
+
+DocumentDB::DocumentDB(const vespalib::string &baseDir,
+                       const DocumentDBConfig::SP & configSnapshot,
+                       const vespalib::string &tlsSpec,
+                       matching::QueryLimiter & queryLimiter,
+                       const vespalib::Clock &clock,
+                       const DocTypeName &docTypeName,
+                       const ProtonConfig &protonCfg,
+                       IDocumentDBOwner & owner,
+                       vespalib::ThreadExecutor & warmupExecutor,
+                       vespalib::ThreadStackExecutorBase & summaryExecutor,
+                       search::transactionlog::Writer * tlsDirectWriter,
+                       MetricsWireService &metricsWireService,
+                       const FileHeaderContext &fileHeaderContext,
+                       ConfigStore::UP config_store,
+                       InitializeThreads initializeThreads)
+    : IDocumentDBConfigOwner(),
+      IReplayConfig(),
+      FeedHandler::IOwner(),
+      IDocumentSubDB::IOwner(),
+      IClusterStateChangedHandler(),
+      IWipeOldRemovedFieldsHandler(),
+      search::transactionlog::SyncProxy(),
+      _docTypeName(docTypeName),
+      _baseDir(baseDir + "/" + _docTypeName.toString()),
+      // Only one thread per executor, or performDropFeedView() will fail.
+      _writeService(std::max(1, protonCfg.indexing.threads)),
+      _initializeThreads(initializeThreads),
+      _initConfigSnapshot(),
+      _initConfigSerialNum(0u),
+      _pendingConfigSnapshot(configSnapshot),
+      _configLock(),
+      _activeConfigSnapshot(),
+      _activeConfigSnapshotGeneration(0),
+      _activeConfigSnapshotSerialNum(0u),
+      _initGate(),
+      _clusterStateHandler(_writeService.master()),
+      _bucketHandler(_writeService.master()),
+      _protonSummaryCfg(protonCfg.summary),
+      _protonIndexCfg(protonCfg.index),
+      _config_store(std::move(config_store)),
+      _sessionManager(new matching::SessionManager(protonCfg.grouping.sessionmanager.maxentries)),
+      _metricsWireService(metricsWireService),
+      _metricsHook(*this, _docTypeName.getName(), protonCfg.numthreadspersearch),
+      _feedView(),
+      _refCountMonitor(),
+      _refCount(0u),
+      _syncFeedViewEnabled(false),
+      _owner(owner),
+      _state(),
+      _writeFilter(),
+      _feedHandler(_writeService,
+                   tlsSpec,
+                   docTypeName,
+                   getMetricsCollection().getMetrics().feed,
+                   _state,
+                   *this,
+                   _writeFilter,
+                   *this,
+                   tlsDirectWriter),
+      _historySchema(),
+      _unionSchema(),
+      _subDBs(*this,
+              *this,
+              _feedHandler,
+              _docTypeName,
+              _writeService,
+              warmupExecutor,
+              summaryExecutor,
+              fileHeaderContext,
+              metricsWireService,
+              getMetricsCollection().getMetrics(),
+              queryLimiter,
+              clock,
+              _configLock,
+              _baseDir,
+              protonCfg),
+      _maintenanceController(_writeService.master(), _docTypeName),
+      _visibility(_feedHandler, _writeService, _feedView),
+      _lidSpaceCompactionHandlers(),
+      _jobTrackers(),
+      _lastDocStoreCacheStats(),
+      _calc()
+{
+    assert(configSnapshot.get() != NULL);
+
+    LOG(debug, "DocumentDB(%s): Creating database in directory '%s'",
+        _docTypeName.toString().c_str(), _baseDir.c_str());
+
+    _feedHandler.init(_config_store->getOldestSerialNum());
+    _feedHandler.setBucketDBHandler(&_subDBs.getBucketDBHandler());
+    saveInitialConfig(*configSnapshot);
+    resumeSaveConfig();
+    SerialNum configSerial = _config_store->getPrevValidSerial(
+            _feedHandler.getPrunedSerialNum() + 1);
+    assert(configSerial > 0);
+    DocumentDBConfig::SP loaded_config;
+    _config_store->loadConfig(*configSnapshot, configSerial,
+                              loaded_config, _historySchema);
+    // Grab relevant parts from pending config
+    loaded_config = DocumentDBConfigScout::scout(loaded_config,
+                                                 *_pendingConfigSnapshot.get());
+    // Ignore configs that are not relevant during replay of transaction log
+    loaded_config = DocumentDBConfig::makeReplayConfig(loaded_config);
+
+    reconfigureSchema(*loaded_config, *loaded_config);
+    _initConfigSnapshot = loaded_config;
+    _initConfigSerialNum = configSerial;
+    // Forward changes of cluster state to feed view via us
+    _clusterStateHandler.addClusterStateChangedHandler(this);
+    // Forward changes of cluster state to bucket handler
+    _clusterStateHandler.addClusterStateChangedHandler(&_bucketHandler);
+    for (auto subDb : _subDBs) {
+        _lidSpaceCompactionHandlers.push_back(ILidSpaceCompactionHandler::UP
+                (new LidSpaceCompactionHandler(*subDb,
+                        _docTypeName.getName())));
+    }
+    _writeFilter.setConfig(loaded_config->getMaintenanceConfigSP()->
+                           getAttributeUsageFilterConfig());
+}
+
+void DocumentDB::setActiveConfig(const DocumentDBConfig::SP &config,
+                                 SerialNum serialNum) {
+    vespalib::LockGuard guard(_configLock);
+    _activeConfigSnapshot = config;
+    if (_activeConfigSnapshotGeneration < config->getGeneration()) {
+        _activeConfigSnapshotGeneration = config->getGeneration();
+    }
+    _activeConfigSnapshotSerialNum = serialNum;
+}
+
+DocumentDBConfig::SP DocumentDB::getActiveConfig() const {
+    vespalib::LockGuard guard(_configLock);
+    return _activeConfigSnapshot;
+}
+
+void
+DocumentDB::internalInit()
+{
+    (void) _state.enterLoadState();
+    _writeService.master().execute(makeTask(makeClosure(this,
+                                           &DocumentDB::initManagers)));
+}
+
+
+void
+DocumentDB::initManagers()
+{
+    // Called by executor thread
+    DocumentDBConfig::SP configSnapshot(_initConfigSnapshot);
+    _initConfigSnapshot.reset();
+    InitializerTask::SP rootTask =
+        _subDBs.createInitializer(*configSnapshot, _initConfigSerialNum,
+                                  _unionSchema, _protonSummaryCfg,
+                                  _protonIndexCfg);
+    InitializeThreads initializeThreads = _initializeThreads;
+    _initializeThreads.reset();
+    std::shared_ptr<TaskRunner> taskRunner(std::make_shared<TaskRunner>
+                                           (*initializeThreads));
+    // Note: explicit listing in lambda to keep variables live
+    auto doneTask = makeLambdaTask([initializeThreads, taskRunner,
+                                    configSnapshot, this]()
+                                   { initFinish(configSnapshot); });
+    taskRunner->runTask(rootTask, _writeService.master(), std::move(doneTask));
+}
+
+
+void
+DocumentDB::initFinish(DocumentDBConfig::SP configSnapshot)
+{
+    // Called by executor thread
+    _bucketHandler.setReadyBucketHandler(
+            _subDBs.getReadySubDB()->getDocumentMetaStoreContext().get());
+    _subDBs.initViews(*configSnapshot, _sessionManager);
+    _syncFeedViewEnabled = true;
+    syncFeedView();
+    // Check that feed view has been activated.
+    assert(_feedView.get().get() != NULL);
+    setActiveConfig(configSnapshot, _initConfigSerialNum);
+    startTransactionLogReplay();
+}
+
+
+void
+DocumentDB::newConfigSnapshot(DocumentDBConfig::SP snapshot)
+{
+    // Called by executor thread
+    _pendingConfigSnapshot.set(snapshot);
+    {
+        vespalib::LockGuard guard(_configLock);
+        if (_activeConfigSnapshot.get() == NULL) {
+            LOG(debug,
+                "DocumentDB(%s): Ignoring new available config snapshot. "
+                "The document database does not have"
+                " an active config snapshot yet", _docTypeName.toString().c_str());
+            return;
+        }
+        if (!_state.getAllowReconfig()) {
+            LOG(warning,
+                "DocumentDB(%s): Ignoring new available config snapshot. "
+                "The document database is not allowed to"
+                " reconfigure yet. Wait until replay is done before"
+                " you try to reconfigure again", _docTypeName.toString().c_str());
+            return;
+        }
+    }
+    _writeService.master().execute(makeTask(makeClosure(this,
+                                           &DocumentDB::performReconfig,
+                                           _pendingConfigSnapshot.get())));
+}
+
+
+void
+DocumentDB::enterReprocessState()
+{
+    // Called by executor thread
+    assert(_writeService.master().isCurrentThread());
+    if (!_state.enterReprocessState()) {
+        return;
+    }
+    ReprocessingRunner &runner = _subDBs.getReprocessingRunner();
+    if (!runner.empty()) {
+        runner.run();
+        NoopOperation op;
+        _feedHandler.storeOperation(op);
+        sync(op.getSerialNum());
+    }
+    _subDBs.onReprocessDone(_feedHandler.getSerialNum());
+    enterOnlineState();
+}
+
+
+void
+DocumentDB::enterOnlineState()
+{
+    // Called by executor thread
+    // Ensure that all replayed operations are committed to memory structures
+    _feedView.get()->forceCommit(_feedHandler.getSerialNum());
+    _writeService.sync();
+
+    (void) _state.enterOnlineState();
+    // Consider delayed pruning of transaction log and config history
+    _feedHandler.considerDelayedPrune();
+    performStartMaintenance();
+}
+
+void
+DocumentDB::performReconfig(DocumentDBConfig::SP configSnapshot)
+{
+    // Called by executor thread
+    applyConfig(configSnapshot, getCurrentSerialNumber());
+    if (_state.getState() == DDBState::State::APPLY_LIVE_CONFIG) {
+        enterReprocessState();
+    }
+}
+
+
+bool
+DocumentDB::handleRejectedConfig(DocumentDBConfig::SP &configSnapshot,
+                                 const ConfigValidator::Result &cvr,
+                                 const DDBState::ConfigState &cs)
+{
+    _state.setConfigState(cs);
+    if (cs == DDBState::ConfigState::NEED_RESTART) {
+        LOG(warning, "DocumentDB(%s): Cannot apply new config snapshot directly: '%s'."
+            " Search node must be restarted for new config to take effect",
+            _docTypeName.toString().c_str(), cvr.what().c_str());
+    } else {
+        LOG(error, "DocumentDB(%s): Cannot apply new config snapshot, new schema is in conflict"
+            " with old schema or history schema: '%s'."
+            " Feed interface is disabled until old config is redeployed",
+            _docTypeName.toString().c_str(), cvr.what().c_str());
+    }
+    LOG(info, "DocumentDB(%s): Config from config server rejected: %s",
+        _docTypeName.toString().c_str(),
+        (cs == DDBState::ConfigState::NEED_RESTART ? "need restart" : "feed disabled"));
+    // Use generation from rejected config (white lie ?)
+    _activeConfigSnapshotGeneration = configSnapshot->getGeneration();
+    DocumentDBConfig::SP oaconfig = _activeConfigSnapshot->
+                                    getOriginalConfig();
+    if (!oaconfig ||
+        _state.getState() != DDBState::State::APPLY_LIVE_CONFIG) {
+        return false;
+    }
+    configSnapshot = oaconfig;
+    return true;
+}
+
+
+void
+DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot,
+                        SerialNum serialNum)
+{
+    // Always called by executor thread:
+    // Called by performReconfig() by executor thread during normal
+    // feed mode and when switching to normal feed mode after replay.
+    // Called by replayConfig() in visitor callback by executor thread
+    // when using config from transaction log.
+    ConfigComparisonResult cmpres;
+    Schema::SP oldSchema;
+    bool fallbackConfig = false;
+    {
+        vespalib::LockGuard guard(_configLock);
+        assert(_activeConfigSnapshot.get());
+        if (_activeConfigSnapshot.get() == configSnapshot.get() ||
+            *_activeConfigSnapshot == *configSnapshot) {
+            // generation might have changed but config is unchanged.
+            _activeConfigSnapshot = configSnapshot;
+            _activeConfigSnapshotGeneration = configSnapshot->getGeneration();
+            if (_state.getRejectedConfig()) {
+                // Illegal reconfig has been reverted.
+                _state.clearRejectedConfig();
+                LOG(info,
+                    "DocumentDB(%s): Config from config server accepted (reverted config)",
+                    _docTypeName.toString().c_str());
+            }
+            return;
+        }
+
+        oldSchema = _activeConfigSnapshot->getSchemaSP();
+        ConfigValidator::Result cvr =
+            ConfigValidator::validate(ConfigValidator::Config
+                    (*configSnapshot->getSchemaSP(),
+                            configSnapshot->getAttributesConfig()),
+                            ConfigValidator::Config
+                            (*oldSchema, _activeConfigSnapshot->getAttributesConfig()),
+                            *_historySchema);
+        DDBState::ConfigState cs = _state.calcConfigState(cvr.type());
+        if (DDBState::getRejectedConfig(cs))
+        {
+            fallbackConfig = handleRejectedConfig(configSnapshot, cvr, cs);
+            if (!fallbackConfig) {
+                return;
+            }
+        }
+        cmpres = _activeConfigSnapshot->compare(*configSnapshot);
+    }
+    const ReconfigParams params(cmpres);
+    if (params.shouldSchemaChange()) {
+        reconfigureSchema(*configSnapshot, *_activeConfigSnapshot);
+    }
+    // Save config via config manager if replay is done.
+    bool equalConfig =
+        *DocumentDBConfig::preferOriginalConfig(configSnapshot) ==
+        *DocumentDBConfig::preferOriginalConfig(_activeConfigSnapshot);
+    assert(!fallbackConfig || equalConfig);
+    bool tlsReplayDone = _feedHandler.getTransactionLogReplayDone();
+    if (!equalConfig && tlsReplayDone) {
+        sync(_feedHandler.getSerialNum());
+        serialNum = _feedHandler.incSerialNum();
+        _config_store->saveConfig(*configSnapshot, *_historySchema, serialNum);
+        // save entry in transaction log
+        NewConfigOperation op(serialNum, *_config_store);
+        _feedHandler.storeOperation(op);
+        sync(op.getSerialNum());
+    }
+    {
+        bool elidedConfigSave = equalConfig && tlsReplayDone;
+        // Flush changes to attributes and memory index, cf. visibilityDelay
+        _feedView.get()->forceCommit(elidedConfigSave ? serialNum :
+                                     serialNum - 1);
+        _writeService.sync();
+        fastos::TimeStamp visibilityDelay =
+            configSnapshot->getMaintenanceConfigSP()->getVisibilityDelay();
+        _visibility.setVisibilityDelay(visibilityDelay);
+    }
+    if (params.shouldSubDbsChange()) {
+        _subDBs.applyConfig(*configSnapshot, *_activeConfigSnapshot, serialNum, params);
+        if (serialNum < _feedHandler.getSerialNum()) {
+            // Not last entry in tls.  Reprocessing should already be done.
+            _subDBs.getReprocessingRunner().reset();
+        }
+        if (_state.getState() == DDBState::State::ONLINE) {
+            // Changes applied while online should not trigger reprocessing
+            assert(_subDBs.getReprocessingRunner().empty());
+        }
+    }
+    if (params.shouldIndexManagerChange()) {
+        setIndexSchema(*configSnapshot);
+    }
+    if (!fallbackConfig) { 
+        if (_state.getRejectedConfig()) {
+            LOG(info, "DocumentDB(%s): Rejected config replaced with new config",
+                _docTypeName.toString().c_str());
+        }
+        _state.clearRejectedConfig();
+    }
+    setActiveConfig(configSnapshot, serialNum);
+    forwardMaintenanceConfig();
+    _writeFilter.setConfig(configSnapshot->getMaintenanceConfigSP()->
+                           getAttributeUsageFilterConfig());
+}
+
+
+void
+DocumentDB::reconfigureSchema(const DocumentDBConfig &configSnapshot,
+                              const DocumentDBConfig &oldConfigSnapshot)
+{
+    // Called by CTOR and executor thread
+    const Schema &newSchema = *configSnapshot.getSchemaSP();
+    const Schema &oldSchema = *oldConfigSnapshot.getSchemaSP();
+    Schema::SP oldHistory = _historySchema;
+    _historySchema =
+        SchemaUtil::makeHistorySchema(newSchema, oldSchema, *oldHistory);
+    _unionSchema = SchemaUtil::makeUnionSchema(newSchema, *_historySchema);
+}
+
+
+namespace {
+void
+doNothing(IFeedView::SP)
+{
+    // Called by index executor, delays when feed view is dropped.
+}
+}  // namespace
+
+void
+DocumentDB::performDropFeedView(IFeedView::SP feedView)
+{
+    // Called by executor task, delays when feed view is dropped.
+    // Also called by DocumentDB::receive() method to keep feed view alive
+
+    _writeService.attributeFieldWriter().sync();
+
+    // Feed view is kept alive in the closure's shared ptr.
+    _writeService.index().execute(makeTask(makeClosure(this,
+                                                &proton::DocumentDB::
+                                                performDropFeedView2,
+                                                feedView)));
+}
+
+
+void
+DocumentDB::performDropFeedView2(IFeedView::SP feedView)
+{
+    // Called by executor task, delays when feed view is dropped.
+    // Also called by DocumentDB::receive() method to keep feed view alive
+    _writeService.indexFieldInverter().sync();
+    _writeService.indexFieldWriter().sync();
+
+    // Feed view is kept alive in the closure's shared ptr.
+    _writeService.master().execute(makeTask(makeClosure(&doNothing, feedView)));
+}
+
+
+namespace {
+void
+waitForRefCountZero(uint32_t &ref_count, vespalib::Monitor &monitor)
+{
+    vespalib::MonitorGuard guard(monitor);
+    while (ref_count != 0) {
+        guard.wait();
+    }
+}
+}  // namespace
+
+
+void
+DocumentDB::close()
+{
+    {
+        vespalib::LockGuard guard(_configLock);
+        _state.enterShutdownState();
+    }
+    _writeService.master().sync(); // Complete all tasks that didn't observe shutdown
+    // Wait until inflight feed operations to this document db has left.
+    // Caller should have removed document DB from feed router.
+    waitForRefCountZero(_refCount, _refCountMonitor);
+    // Abort any ongoing maintenance
+    stopMaintenance();
+
+    // The attributes in the ready sub db is also the total set of attributes.
+    LegacyDocumentDBMetrics &metrics = getMetricsCollection().getMetrics();
+    _metricsWireService.cleanAttributes(metrics.ready.attributes, &metrics.attributes);
+    _metricsWireService.cleanAttributes(metrics.notReady.attributes, NULL);
+    _writeService.sync();
+    _writeService.master().execute(makeTask(makeClosure(this, &DocumentDB::closeSubDBs)));
+    _writeService.sync();
+    // What about queued tasks ?
+    _writeService.shutdown();
+    _maintenanceController.kill();
+    _feedHandler.close();
+    // Assumes that feed engine has been closed.  If only this document DB
+    // is going away while system is still up and running then caller must
+    // ensure that routing has been torn down and pending messages have been
+    // drained.  This goes for all facets: feeding, tls replay,
+    // matching, summary fetch, flushing and reconfig.
+    _feedView.clear();
+    _subDBs.clearViews();
+    _sessionManager->close();
+    _state.enterDeadState();
+}
+
+DocumentDB::~DocumentDB()
+{
+    close();
+    // Remove forwarding of cluster state change
+    _clusterStateHandler.removeClusterStateChangedHandler(&_bucketHandler);
+    _clusterStateHandler.removeClusterStateChangedHandler(this);
+    
+}
+
+void
+DocumentDB::closeSubDBs()
+{
+    _subDBs.close();
+}
+
+size_t
+DocumentDB::getNumDocs() const
+{
+    return _subDBs.getReadySubDB()->getNumDocs();
+}
+
+size_t
+DocumentDB::getNumActiveDocs() const
+{
+    return _subDBs.getReadySubDB()->getNumActiveDocs();
+}
+
+void
+DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot)
+{
+    // Only called from ctor
+
+    vespalib::LockGuard guard(_configLock);
+    if (_config_store->getBestSerialNum() != 0)
+        return;				// Initial config already present
+
+    SerialNum confSerial = _feedHandler.incSerialNum();
+    // Elide save of new config entry in transaction log, it would be
+    // pruned at once anyway.
+    // save noop entry in transaction log
+    NoopOperation op;
+    _feedHandler.storeOperation(op);
+    sync(op.getSerialNum());
+    // Wipe everything in transaction log before initial config.
+    try {
+        _feedHandler.tlsPrune(confSerial);  // throws on error
+    } catch (const vespalib::IllegalStateException & e) {
+        LOG(warning, "DocumentDB(%s): saveInitialConfig() failed pruning due to '%s'",
+            _docTypeName.toString().c_str(), e.what());
+    }
+    _config_store->saveConfig(configSnapshot, Schema(), confSerial);
+}
+
+
+void
+DocumentDB::resumeSaveConfig(void)
+{
+    SerialNum bestSerial = _config_store->getBestSerialNum();
+    if (bestSerial == 0)
+        return;
+    if (bestSerial != _feedHandler.getSerialNum() + 1)
+        return;
+    // proton was interrupted when saving later config.
+    SerialNum confSerial = _feedHandler.incSerialNum();
+    // resume operation, i.e. save config entry in transaction log
+    NewConfigOperation op(confSerial, *_config_store);
+    _feedHandler.storeOperation(op);
+    sync(op.getSerialNum());
+}
+
+
+void
+DocumentDB::onTransactionLogReplayDone()
+{
+    // Called by executor thread
+    _subDBs.onReplayDone();
+    if (!_owner.isInitializing()) {
+        // This document db is added when system is up,
+        // must signal that all existing buckets must be checked.
+        notifyAllBucketsChanged();
+    }
+}
+
+
+void
+DocumentDB::onPerformPrune(SerialNum oldestSerial)
+{
+    if (!getAllowPrune()) {
+        assert(_state.getClosed());
+        return;
+    }
+    _config_store->prune(oldestSerial);
+}
+
+
+bool
+DocumentDB::getAllowPrune(void) const
+{
+    return _state.getAllowPrune();
+}
+
+
+bool
+DocumentDB::isFeedBlockedByRejectedConfig()
+{
+    return _state.isFeedBlockedByRejectedConfig();
+}
+
+
+void
+DocumentDB::start()
+{
+    LOG(debug,
+        "DocumentDB(%s): Database starting.",
+        _docTypeName.toString().c_str());
+
+    internalInit();
+}
+
+
+void
+DocumentDB::waitForInitDone()
+{
+    _initGate.await();
+}
+
+
+void
+DocumentDB::startTransactionLogReplay()
+{
+    // This configSnapshot is only used to reuse DocumentTypeRepo
+    // and TuneFile when loading configs during replay.
+    DocumentDBConfig::SP configSnapshot = getActiveConfig();
+    IDocumentSubDB *readySubDB = _subDBs.getReadySubDB();
+    SerialNum oldestFlushedSerial = getOldestFlushedSerial();
+    SerialNum newestFlushedSerial = getNewestFlushedSerial();
+    (void) _state.enterReplayTransactionLogState();
+    _feedHandler.replayTransactionLog(readySubDB->getIndexManager()->
+                                      getFlushedSerialNum(),
+                                      readySubDB->getSummaryManager()->
+                                      getBackingStore().lastSyncToken(),
+                                      oldestFlushedSerial,
+                                      newestFlushedSerial,
+                                      *_config_store);
+    _initGate.countDown();
+
+    LOG(debug,
+        "DocumentDB(%s): Database started.",
+        _docTypeName.toString().c_str());
+}
+
+BucketGuard::UP DocumentDB::lockBucket(const document::BucketId &bucket)
+{
+    BucketGuard::UP guard(std::make_unique<BucketGuard>
+                          (bucket, _maintenanceController));
+    _visibility.commitAndWait();
+    return std::move(guard);
+}
+
+
+SerialNum
+DocumentDB::getOldestFlushedSerial()
+{
+    return _subDBs.getOldestFlushedSerial();
+}
+
+SerialNum
+DocumentDB::getNewestFlushedSerial()
+{
+    return _subDBs.getNewestFlushedSerial();
+}
+
+search::engine::SearchReply::UP
+DocumentDB::match(const ISearchHandler::SP &,
+                  const search::engine::SearchRequest &req,
+                  vespalib::ThreadBundle &threadBundle) const
+{
+    // Ignore input searchhandler. Use readysubdb's searchhandler instead.
+    ISearchHandler::SP view(_subDBs.getReadySubDB()->getSearchView());
+    return view->match(view, req, threadBundle);
+}
+
+DocsumReply::UP
+DocumentDB::getDocsums(const DocsumRequest & request)
+{
+    ISearchHandler::SP view(_subDBs.getReadySubDB()->getSearchView());
+    return view->getDocsums(request);
+}
+
+IFlushTarget::List
+DocumentDB::getFlushTargets()
+{
+    IFlushTarget::List flushTargets = _subDBs.getFlushTargets();
+    return _jobTrackers.trackFlushTargets(flushTargets);
+}
+
+void
+DocumentDB::flushDone(SerialNum oldestSerial)
+{
+    _feedHandler.flushDone(oldestSerial);
+}
+
+void
+DocumentDB::setIndexSchema(const DocumentDBConfig &configSnapshot)
+{
+    // Called by executor thread
+    _subDBs.getReadySubDB()->setIndexSchema(configSnapshot.getSchemaSP(),
+                                            _unionSchema);
+
+    // TODO: Adjust tune.
+}
+
+
+void
+DocumentDB::reconfigure(const DocumentDBConfig::SP & snapshot)
+{
+    _writeService.master().execute(makeTask(makeClosure(this,
+                                           &DocumentDB::newConfigSnapshot,
+                                           snapshot)));
+}
+
+void
+DocumentDB::enterRedoReprocessState()
+{
+    assert(_writeService.master().isCurrentThread());
+    ReprocessingRunner &runner = _subDBs.getReprocessingRunner();
+    if (!runner.empty()) {
+        if (!_state.enterRedoReprocessState()) {
+            return;
+        }
+        runner.run();
+        _subDBs.onReprocessDone(_feedHandler.getSerialNum());
+        NoopOperation op;
+        _feedHandler.storeOperation(op);
+        sync(op.getSerialNum());
+    }
+    enterApplyLiveConfigState();
+}
+
+
+void
+DocumentDB::enterApplyLiveConfigState()
+{
+    assert(_writeService.master().isCurrentThread());
+    // Enable reconfig and queue currently pending config as executor task.
+    {
+        vespalib::LockGuard guard(_configLock);
+        (void) _state.enterApplyLiveConfigState();
+    }
+    _writeService.master().execute(makeTask(makeClosure(this,
+                                           &DocumentDB::performReconfig,
+                                           _pendingConfigSnapshot.get())));
+}
+
+
+void
+DocumentDB::retain()
+{
+    vespalib::MonitorGuard guard(_refCountMonitor);
+    ++_refCount;
+}
+
+
+void
+DocumentDB::release()
+{
+    vespalib::MonitorGuard guard(_refCountMonitor);
+    --_refCount;
+    if (_refCount == 0)
+        guard.broadcast();
+}
+
+
+StatusReport::UP
+DocumentDB::reportStatus() const
+{
+    StatusReport::Params params("documentdb:" + _docTypeName.toString());
+    const DDBState::State rawState = _state.getState();
+    {
+        const vespalib::string state(DDBState::getStateString(rawState));
+        const vespalib::string configState(DDBState::getConfigStateString(_state.getConfigState()));
+        params.internalState(state).internalConfigState(configState);
+    }
+
+    if (_initGate.getCount() != 0) {
+        return StatusReport::create(params.state(StatusReport::PARTIAL).
+                message("DocumentDB initializing components"));
+    } else if (_feedHandler.isDoingReplay()) {
+        float progress = _feedHandler.getReplayProgress() * 100.0f;
+        vespalib::string msg = vespalib::make_string("DocumentDB replay transaction log on startup (%u%% done)",
+                static_cast<uint32_t>(progress));
+        return StatusReport::create(params.state(StatusReport::PARTIAL).
+                progress(progress).
+                message(msg));
+    } else if (rawState == DDBState::State::APPLY_LIVE_CONFIG) {
+        return StatusReport::create(params.state(StatusReport::PARTIAL).
+                message("DocumentDB apply live config on startup"));
+    } else if (rawState == DDBState::State::REPROCESS ||
+               rawState == DDBState::State::REDO_REPROCESS)
+    {
+        float progress = _subDBs.getReprocessingProgress() * 100.0f;
+        vespalib::string msg = make_string("DocumentDB reprocess on startup (%u%% done)",
+                        static_cast<uint32_t>(progress));
+        return StatusReport::create(params.state(StatusReport::PARTIAL).
+                progress(progress).
+                message(msg));
+    } else if (_state.getRejectedConfig()) {
+        return StatusReport::create(params.state(StatusReport::PARTIAL).
+                message("DocumentDB rejecting config"));
+    } else {
+        return StatusReport::create(params.state(StatusReport::UPOK));
+    }
+}
+
+
+void
+DocumentDB::wipeHistory(void)
+{
+    // Called from RPC handler
+    _writeService.master().execute(makeTask(makeClosure(this,
+                                           &DocumentDB::performWipeHistory)));
+    _writeService.master().sync();
+}
+
+
+void
+DocumentDB::performWipeHistory()
+{
+    // Called by executor thread
+    if (_historySchema->empty())
+        return;
+    if (_feedHandler.getTransactionLogReplayDone()) {
+        sync(_feedHandler.getSerialNum()); // Sync before wiping history
+        DocumentDBConfig::SP configSnapshot = getActiveConfig();
+        SerialNum wipeSerial = _feedHandler.incSerialNum();
+        Schema::UP newHistory(new Schema);
+        writeWipeHistoryTransactionLogEntry(wipeSerial, 0,
+                                            *configSnapshot, *newHistory);
+        internalWipeHistory(wipeSerial, std::move(newHistory), *_historySchema);
+    }
+}
+
+
+void DocumentDB::writeWipeHistoryTransactionLogEntry(
+        SerialNum wipeSerial, fastos::TimeStamp wipeTimeLimit,
+        const DocumentDBConfig &configSnapshot,
+        const Schema &newHistorySchema) {
+    // Caller must have synced transaction log
+    _config_store->saveConfig(configSnapshot, newHistorySchema, wipeSerial);
+    // save entry in transaction log
+    WipeHistoryOperation op(wipeSerial, wipeTimeLimit);
+    _feedHandler.storeOperation(op);
+    sync(op.getSerialNum());
+}
+
+
+void
+DocumentDB::internalWipeHistory(SerialNum wipeSerial,
+                                Schema::UP newHistorySchema,
+                                const Schema &wipeSchema)
+{
+    // Called by executor thread
+    _subDBs.wipeHistory(wipeSerial, *newHistorySchema, wipeSchema);
+    _historySchema.reset(newHistorySchema.release());
+    _unionSchema = getActiveConfig()->getSchemaSP();
+}
+
+
+void
+DocumentDB::replayConfig(search::SerialNum serialNum)
+{
+    // Called by executor thread during transaction log replay.
+    DocumentDBConfig::SP configSnapshot = getActiveConfig();
+    if (configSnapshot.get() == NULL) {
+        LOG(warning,
+            "DocumentDB(%s): Missing old config when replaying config, serialNum=%" PRIu64,
+            _docTypeName.toString().c_str(), serialNum);
+        return;
+    }
+    // Load historyschema before applyConfig to preserve the history
+    // field timestamps.
+    _config_store->loadConfig(*configSnapshot, serialNum,
+                            configSnapshot, _historySchema);
+    // Grab relevant parts from pending config
+    configSnapshot = DocumentDBConfigScout::scout(configSnapshot,
+                                                 *_pendingConfigSnapshot.get());
+    // Ignore configs that are not relevant during replay of transaction log
+    configSnapshot = DocumentDBConfig::makeReplayConfig(configSnapshot);
+    applyConfig(configSnapshot, serialNum);
+    LOG(info,
+        "DocumentDB(%s): Replayed config with serialNum=%" PRIu64,
+        _docTypeName.toString().c_str(), serialNum);
+}
+
+void
+DocumentDB::replayWipeHistory(search::SerialNum serialNum,
+                              fastos::TimeStamp wipeTimeLimit)
+{
+    // Called by executor thread
+    DocumentDBConfig::SP configSnapshot = getActiveConfig();
+    if (configSnapshot.get() == NULL) {
+        LOG(warning,
+            "DocumentDB(%s): Missing old config when replaying wipe history, serialNum=%" PRIu64,
+            _docTypeName.toString().c_str(),
+            serialNum);
+        return;
+    }
+    Schema::UP wipeSchemaOwner;
+    Schema *wipeSchema;
+    Schema::UP newHistory;
+    if (wipeTimeLimit) {
+        wipeSchemaOwner = _historySchema->getOldFields(wipeTimeLimit);
+        wipeSchema = wipeSchemaOwner.get();
+        newHistory = Schema::set_difference(*_historySchema, *wipeSchema);
+    } else {  // wipeTimeLimit == 0 means old style wipeHistory.
+        wipeSchema = _historySchema.get();
+        newHistory.reset(new Schema);
+    }
+    LOG(info, "DocumentDB(%s): Replayed history wipe with serialNum=%" PRIu64,
+        _docTypeName.toString().c_str(), serialNum);
+    internalWipeHistory(serialNum, std::move(newHistory), *wipeSchema);
+}
+
+
+void
+DocumentDB::listSchema(std::vector<vespalib::string> &fieldNames,
+                       std::vector<vespalib::string> &fieldDataTypes,
+                       std::vector<vespalib::string> &fieldCollectionTypes,
+                       std::vector<vespalib::string> &fieldLocations)
+{
+    DocumentDBConfig::SP activeSnapshot = getActiveConfig();
+    if (activeSnapshot.get() == NULL ||
+        activeSnapshot->getSchemaSP().get() == NULL)
+    {
+        return;
+    }
+    SchemaUtil::listSchema(*activeSnapshot->getSchemaSP(),
+                           fieldNames,
+                           fieldDataTypes,
+                           fieldCollectionTypes,
+                           fieldLocations);
+}
+
+
+int64_t DocumentDB::getActiveGeneration() const {
+    vespalib::LockGuard guard(_configLock);
+    return _activeConfigSnapshotGeneration;
+}
+
+
+void
+DocumentDB::syncFeedView(void)
+{
+    // Called by executor or while in rendezvous with executor
+
+    if (!_syncFeedViewEnabled)
+        return;
+    IFeedView::SP oldFeedView(_feedView.get());
+    IFeedView::SP newFeedView(_subDBs.getFeedView());
+    _feedView.set(newFeedView);
+    _feedHandler.setActiveFeedView(newFeedView.get());
+    _subDBs.createRetrievers();
+    _subDBs.maintenanceSync(_maintenanceController, _visibility);
+
+    // Ensure that old feed view is referenced until all index executor tasks
+    // depending on it has completed.
+    performDropFeedView(oldFeedView);
+}
+
+
+bool
+DocumentDB::hasDocument(const document::DocumentId &id)
+{
+    return _subDBs.getReadySubDB()->hasDocument(id);
+}
+
+
+void
+DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config)
+{
+    // Called by executor thread
+    _maintenanceController.killJobs();
+    MaintenanceJobsInjector::injectJobs(_maintenanceController,
+            config,
+            _feedHandler, // IHeartBeatHandler
+            *_sessionManager, // ISessionCachePruner
+            *this, // IWipeOldRemovedFieldsHandler
+            _lidSpaceCompactionHandlers,
+            _feedHandler, // IOperationStorer
+            _maintenanceController, // IFrozenBucketHandler
+            _docTypeName.getName(),
+            _feedHandler, // IPruneRemovedDocumentsHandler
+            _feedHandler, // IDocumentMoveHandler
+            _clusterStateHandler, // IBucketModifiedHandler
+            _clusterStateHandler, // IClusterStateChangedNotifier
+            _bucketHandler, // IBucketStateChangedNotifier
+            _calc, // IBucketStateCalculator::SP
+            _jobTrackers,
+            _visibility,  // ICommitable
+            _subDBs.getReadySubDB()->getAttributeManager(),
+            _subDBs.getNotReadySubDB()->getAttributeManager(),
+            _writeFilter);
+}
+
+void
+DocumentDB::performStartMaintenance(void)
+{
+    // Called by executor thread
+    // Only start once, after replay done
+
+    DocumentDBMaintenanceConfig::SP maintenanceConfig;
+    {
+        vespalib::LockGuard guard(_configLock);
+        if (_state.getClosed())
+            return;
+        assert(_activeConfigSnapshot.get() != NULL);
+        maintenanceConfig = _activeConfigSnapshot->getMaintenanceConfigSP();
+    }
+    if (_maintenanceController.getStopping()) {
+        return;
+    }
+    injectMaintenanceJobs(*maintenanceConfig);
+    _maintenanceController.start(maintenanceConfig);
+}
+
+void
+DocumentDB::stopMaintenance(void)
+{
+    _maintenanceController.stop();
+}
+
+void
+DocumentDB::forwardMaintenanceConfig(void)
+{
+    // Called by executor thread
+    DocumentDBConfig::SP activeConfig = getActiveConfig();
+    assert(activeConfig.get() != NULL);
+    DocumentDBMaintenanceConfig::SP
+        maintenanceConfig(activeConfig->getMaintenanceConfigSP());
+    if (!_state.getClosed()) {
+        if (_maintenanceController.getStarted() &&
+            !_maintenanceController.getStopping()) {
+            injectMaintenanceJobs(*maintenanceConfig);
+        }
+        _maintenanceController.newConfig(maintenanceConfig);
+    }
+}
+
+void
+DocumentDB::notifyClusterStateChanged(
+        const IBucketStateCalculator::SP &newCalc)
+{
+    // Called by executor thread
+    _calc = newCalc; // Save for maintenance job injection
+    // Forward changes of cluster state to feed view
+    IFeedView::SP feedView(_feedView.get());
+    if (feedView.get() != NULL) {
+        // Try downcast to avoid polluting API
+        CombiningFeedView *cfv = dynamic_cast<CombiningFeedView *>
+                                 (feedView.get());
+        if (cfv != NULL)
+            cfv->setCalculator(newCalc);
+    }
+    _subDBs.setBucketStateCalculator(newCalc);
+}
+
+
+namespace {
+
+void notifyBucketsChanged(const documentmetastore::IBucketHandler &metaStore,
+                          IBucketModifiedHandler &handler,
+                          const vespalib::string &name)
+{
+    BucketDBOwner::Guard buckets = metaStore.getBucketDB().takeGuard();
+    for (const auto &kv : *buckets) {
+        handler.notifyBucketModified(kv.first);
+    }
+    LOG(debug, "notifyBucketsChanged(%s, %zu)", name.c_str(), buckets->size());
+}
+
+}
+
+
+void
+DocumentDB::notifyAllBucketsChanged()
+{
+    // Called by executor thread
+    notifyBucketsChanged(_subDBs.getReadySubDB()->getDocumentMetaStoreContext().get(),
+                         _clusterStateHandler, "ready");
+    notifyBucketsChanged(_subDBs.getRemSubDB()->getDocumentMetaStoreContext().get(),
+                         _clusterStateHandler, "removed");
+    notifyBucketsChanged(_subDBs.getNotReadySubDB()->getDocumentMetaStoreContext().get(),
+                         _clusterStateHandler, "notready");
+}
+
+
+namespace {
+
+vespalib::string
+getSchemaFieldsList(const Schema &schema)
+{
+    vespalib::asciistream oss;
+    for (uint32_t i = 0; i < schema.getNumIndexFields(); ++i) {
+        if (i > 0) oss << ",";
+        oss << schema.getIndexField(i).getName();
+    }
+    for (uint32_t i = 0; i < schema.getNumAttributeFields(); ++i) {
+        if (!oss.str().empty()) oss << ",";
+        oss << schema.getAttributeField(i).getName();
+    }
+    return oss.str();
+}
+
+}
+
+void
+DocumentDB::wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit)
+{
+    // Called by executor thread
+
+    if (_historySchema->empty())
+        return;
+    DocumentDBConfig::SP configSnapshot = getActiveConfig();
+
+    Schema::UP wipeSchema = _historySchema->getOldFields(wipeTimeLimit);
+    Schema::UP newHistorySchema =
+        Schema::set_difference(*_historySchema, *wipeSchema);
+
+    sync(_feedHandler.getSerialNum()); // Sync before wiping history
+    SerialNum wipeSerial = _feedHandler.incSerialNum();
+    writeWipeHistoryTransactionLogEntry(wipeSerial, wipeTimeLimit,
+                                        *configSnapshot, *newHistorySchema);
+    internalWipeHistory(wipeSerial, std::move(newHistorySchema), *wipeSchema);
+
+    LOG(debug, "DocumentDB(%s): Done wipeOldRemovedFields: wipe(%s), history(%s) timeLimit(%" PRIu64 ")",
+        _docTypeName.toString().c_str(),
+        getSchemaFieldsList(*wipeSchema).c_str(),
+        getSchemaFieldsList(*_historySchema).c_str(),
+        static_cast<uint64_t>(wipeTimeLimit.sec()));
+}
+
+searchcorespi::IIndexManagerFactory::SP
+DocumentDB::getIndexManagerFactory(const vespalib::stringref &name) const
+{
+    return _owner.getIndexManagerFactory(name);
+}
+
+namespace {
+
+void
+updateIndexMetrics(LegacyDocumentDBMetrics::IndexMetrics &metrics,
+                   const search::SearchableStats &stats)
+{
+    metrics.memoryUsage.set(stats.memoryUsage());
+    metrics.docsInMemory.set(stats.docsInMemory());
+    metrics.diskUsage.set(stats.sizeOnDisk());
+}
+
+struct TempAttributeMetric
+{
+    uint64_t _memoryUsage;
+    uint64_t _bitVectors;
+
+    TempAttributeMetric()
+        : _memoryUsage(0),
+          _bitVectors(0)
+    {
+    }
+};
+
+struct TempAttributeMetrics
+{
+    typedef std::map<vespalib::string, TempAttributeMetric> AttrMap;
+    TempAttributeMetric _total;
+    AttrMap _attrs;
+};
+
+bool
+isReadySubDB(const IDocumentSubDB *subDb, const DocumentSubDBCollection &sub_dbs)
+{
+    return subDb == sub_dbs.getReadySubDB();
+}
+
+bool
+isNotReadySubDB(const IDocumentSubDB *subDb, const DocumentSubDBCollection &sub_dbs)
+{
+    return subDb == sub_dbs.getNotReadySubDB();
+}
+
+void
+fillTempAttributeMetrics(TempAttributeMetrics &metrics,
+                         const vespalib::string &attrName,
+                         size_t mem, uint32_t bitVectors)
+{
+    metrics._total._memoryUsage += mem;
+    metrics._total._bitVectors += bitVectors;
+    TempAttributeMetric &m = metrics._attrs[attrName];
+    m._memoryUsage += mem;
+    m._bitVectors += bitVectors;
+}
+
+void
+fillTempAttributeMetrics(TempAttributeMetrics &totalMetrics,
+                         TempAttributeMetrics &readyMetrics,
+                         TempAttributeMetrics &notReadyMetrics,
+                         const DocumentSubDBCollection &sub_dbs)
+{
+    for (const auto subDb : sub_dbs) {
+        proton::IAttributeManager::SP attrMgr(subDb->getAttributeManager());
+        if (attrMgr.get()) {
+            TempAttributeMetrics *subMetrics =
+                    (isReadySubDB(subDb, sub_dbs) ? &readyMetrics :
+                            (isNotReadySubDB(subDb, sub_dbs) ? &notReadyMetrics : NULL));
+            std::vector<search::AttributeGuard> list;
+            attrMgr->getAttributeListAll(list);
+            for (const auto &attr : list) {
+                const search::attribute::Status &status = attr->getStatus();
+                size_t mem = status.getAllocated();
+                uint32_t bitVectors = status.getBitVectors();
+                fillTempAttributeMetrics(totalMetrics, attr->getName(), mem, bitVectors);
+                if (subMetrics != NULL) {
+                    fillTempAttributeMetrics(*subMetrics, attr->getName(), mem, bitVectors);
+                }
+            }
+        }
+    }
+}
+
+void
+updateAttributeMetrics(AttributeMetrics &metrics,
+                       const TempAttributeMetrics &tmpMetrics)
+{
+    for (const auto &kv : tmpMetrics._attrs) {
+        AttributeMetrics::List::Entry::LP entry = metrics.list.get(kv.first);
+        if (entry.get()) {
+            entry->memoryUsage.set(kv.second._memoryUsage);
+            entry->bitVectors.set(kv.second._bitVectors);
+        } else {
+            LOG(debug, "could not update metrics for attribute: '%s'",
+                kv.first.c_str());
+        }
+    }
+    metrics.memoryUsage.set(tmpMetrics._total._memoryUsage);
+    metrics.bitVectors.set(tmpMetrics._total._bitVectors);
+}
+
+void
+updateAttributeMetrics(LegacyDocumentDBMetrics &metrics,
+                       const DocumentSubDBCollection &sub_dbs)
+{
+    TempAttributeMetrics totalMetrics;
+    TempAttributeMetrics readyMetrics;
+    TempAttributeMetrics notReadyMetrics;
+    fillTempAttributeMetrics(totalMetrics, readyMetrics, notReadyMetrics, sub_dbs);
+    updateAttributeMetrics(metrics.attributes, totalMetrics);
+    updateAttributeMetrics(metrics.ready.attributes, readyMetrics);
+    updateAttributeMetrics(metrics.notReady.attributes, notReadyMetrics);
+}
+
+void
+updateMatchingMetrics(LegacyDocumentDBMetrics::MatchingMetrics &metrics,
+                      const IDocumentSubDB &ready)
+{
+    MatchingStats stats;
+    for (const auto &kv : metrics.rank_profiles) {
+        MatchingStats rp_stats = ready.getMatcherStats(kv.first);
+        kv.second->update(rp_stats);
+        stats.add(rp_stats);
+    }
+    metrics.update(stats);
+}
+
+void
+updateDocstoreMetrics(LegacyDocumentDBMetrics::DocstoreMetrics &metrics,
+                      const DocumentSubDBCollection &sub_dbs,
+                      CacheStats &lastCacheStats)
+{
+    size_t memoryUsage = 0;
+    CacheStats cache_stats;
+    for (const auto subDb : sub_dbs) {
+        const ISummaryManager::SP &summaryMgr = subDb->getSummaryManager();
+        cache_stats += summaryMgr->getBackingStore().getCacheStats();
+        memoryUsage += summaryMgr->getBackingStore().memoryUsed();
+    }
+    metrics.memoryUsage.set(memoryUsage);
+    size_t lookups = cache_stats.hits + cache_stats.misses;
+    metrics.cacheLookups.set(lookups);
+    size_t last_count = lastCacheStats.hits + lastCacheStats.misses;
+        // For the above code to add sane values to the metric, the following
+        // must be true
+    if (lookups < last_count || cache_stats.hits < lastCacheStats.hits) {
+        LOG(warning, "Not adding document db metrics as values calculated "
+                     "are corrupt. %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 ".",
+            lookups, last_count, cache_stats.hits, lastCacheStats.hits);
+    } else {
+        if (lookups - last_count > 0xffffffffull
+            || cache_stats.hits - lastCacheStats.hits > 0xffffffffull)
+        {
+            LOG(warning, "Document db metrics to add are suspiciously high."
+                         " %" PRIu64 ", %" PRIu64 ".",
+                lookups - last_count, cache_stats.hits - lastCacheStats.hits);
+        }
+        metrics.cacheHitRate.addTotalValueWithCount(
+                cache_stats.hits - lastCacheStats.hits, lookups - last_count);
+    }
+    metrics.hits = cache_stats.hits;
+    metrics.cacheElements.set(cache_stats.elements);
+    metrics.cacheMemoryUsed.set(cache_stats.memory_used);
+    lastCacheStats = cache_stats;
+}
+
+void
+updateDocumentStoreMetrics(DocumentDBTaggedMetrics::SubDBMetrics::
+                           DocumentStoreMetrics &metrics,
+                           IDocumentSubDB *subDb)
+{
+    const ISummaryManager::SP &summaryMgr = subDb->getSummaryManager();
+    search::IDocumentStore &backingStore = summaryMgr->getBackingStore();
+    search::DataStoreStorageStats storageStats(backingStore.getStorageStats());
+    metrics.diskUsage.set(storageStats.diskUsage());
+    metrics.diskBloat.set(storageStats.diskBloat());
+    metrics.maxBucketSpread.set(storageStats.maxBucketSpread());
+}
+
+template <typename MetricSetType>
+void
+updateLidSpaceMetrics(MetricSetType &metrics,
+                      const search::IDocumentMetaStore &metaStore)
+{
+    LidUsageStats stats = metaStore.getLidUsageStats();
+    metrics.lidLimit.set(stats.getLidLimit());
+    metrics.usedLids.set(stats.getUsedLids());
+    metrics.lowestFreeLid.set(stats.getLowestFreeLid());
+    metrics.highestUsedLid.set(stats.getHighestUsedLid());
+    metrics.lidBloatFactor.set(stats.getLidBloatFactor());
+    metrics.lidFragmentationFactor.set(stats.getLidFragmentationFactor());
+}
+
+}  // namespace
+
+void
+DocumentDB::updateMetrics(DocumentDBMetricsCollection &metrics)
+{
+    updateMetrics(metrics.getMetrics());
+    updateMetrics(metrics.getTaggedMetrics());
+}
+
+void
+DocumentDB::updateMetrics(LegacyDocumentDBMetrics &metrics)
+{
+    updateIndexMetrics(metrics.index,
+                       _subDBs.getReadySubDB()->getSearchableStats());
+    updateAttributeMetrics(metrics, _subDBs);
+    updateMatchingMetrics(metrics.matching, *_subDBs.getReadySubDB());
+    metrics.executor.update(_writeService.getMasterExecutor().getStats());
+    metrics.indexExecutor.update(_writeService.getIndexExecutor().getStats());
+    metrics.sessionManager.update(_sessionManager->getGroupingStats());
+    updateDocstoreMetrics(metrics.docstore, _subDBs, _lastDocStoreCacheStats);
+    metrics.numDocs.set(getNumDocs());
+
+    DocumentMetaStoreReadGuards dmss(_subDBs);
+    
+    metrics.numActiveDocs.set(dmss.numActiveDocs());
+    metrics.numIndexedDocs.set(dmss.numIndexedDocs());
+    metrics.numStoredDocs.set(dmss.numStoredDocs());
+    metrics.numRemovedDocs.set(dmss.numRemovedDocs());
+
+    updateLidSpaceMetrics(metrics.ready.docMetaStore, dmss.readydms->get());
+    updateLidSpaceMetrics(metrics.notReady.docMetaStore, dmss.notreadydms->get());
+    updateLidSpaceMetrics(metrics.removed.docMetaStore, dmss.remdms->get());
+
+    metrics.numBadConfigs.set(_state.getRejectedConfig() ? 1u : 0u);
+}
+
+void
+DocumentDB::
+updateMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics)
+{
+    AttributeUsageFilter &writeFilter(_writeFilter);
+    AttributeUsageStats attributeUsageStats =
+        writeFilter.getAttributeUsageStats();
+    bool feedBlocked = !writeFilter.acceptWriteOperation();
+    double enumStoreUsed =
+        attributeUsageStats.enumStoreUsage().getUsage().usage();
+    double multiValueUsed =
+        attributeUsageStats.multiValueUsage().getUsage().usage();
+    metrics.resourceUsage.enumStore.set(enumStoreUsed);
+    metrics.resourceUsage.multiValue.set(multiValueUsed);
+    metrics.resourceUsage.feedingBlocked.set(feedBlocked ? 1 : 0);
+}
+
+void
+DocumentDB::updateMetrics(DocumentDBTaggedMetrics &metrics)
+{
+    _jobTrackers.updateMetrics(metrics.job);
+
+    updateMetrics(metrics.attribute);
+    updateDocumentStoreMetrics(metrics.ready.documentStore,
+                               _subDBs.getReadySubDB());
+    updateDocumentStoreMetrics(metrics.removed.documentStore,
+                               _subDBs.getRemSubDB());
+    updateDocumentStoreMetrics(metrics.notReady.documentStore,
+                               _subDBs.getNotReadySubDB());
+    DocumentMetaStoreReadGuards dmss(_subDBs);
+    updateLidSpaceMetrics(metrics.ready.lidSpace, dmss.readydms->get());
+    updateLidSpaceMetrics(metrics.notReady.lidSpace, dmss.notreadydms->get());
+    updateLidSpaceMetrics(metrics.removed.lidSpace, dmss.remdms->get());
+}
+
+void
+DocumentDB::sync(SerialNum syncTo)
+{
+    LOG(spam,
+        "DocumentDB(%s): sync(): serialNum=%" PRIu64,
+        _docTypeName.toString().c_str(), syncTo);
+    _feedHandler.syncTls(syncTo);
+}
+
+
+void
+DocumentDB::waitForOnlineState()
+{
+    _state.waitForOnlineState();
+}
+
+uint32_t
+DocumentDB::getDistributionKey() const
+{
+    return _owner.getDistributionKey();
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
new file mode 100644
index 00000000000..0c49c2c0170
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -0,0 +1,476 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+
+#include "buckethandler.h"
+#include "clusterstatehandler.h"
+#include "configstore.h"
+#include "ddbstate.h"
+#include "documentdbconfig.h"
+#include "documentsubdbcollection.h"
+#include "feedhandler.h"
+#include "i_lid_space_compaction_handler.h"
+#include "ifeedview.h"
+#include "ireplayconfig.h"
+#include "iwipeoldremovedfieldshandler.h"
+#include "maintenancecontroller.h"
+#include "protonconfigurer.h"
+#include "searchable_doc_subdb_configurer.h"
+#include "searchable_document_retriever.h"
+#include "searchabledocsubdb.h"
+#include "summaryadapter.h"
+#include "visibilityhandler.h"
+
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_writer.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/common/statusreport.h>
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+#include <vespa/searchcore/proton/index/i_index_writer.h>
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
+#include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h>
+#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h>
+#include <vespa/searchcore/proton/persistenceengine/bucket_guard.h>
+#include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
+#include <vespa/searchlib/docstore/cachestats.h>
+#include <vespa/searchlib/engine/searchrequest.h>
+#include <vespa/searchlib/transactionlog/syncproxy.h>
+#include <vespa/vespalib/util/varholder.h>
+#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
+
+using vespa::config::search::core::ProtonConfig;
+
+namespace search
+{
+
+namespace common
+{
+
+class FileHeaderContext;
+
+}
+
+namespace transactionlog { class TransLogClient; }
+}  // namespace search
+
+namespace proton {
+class MetricsWireService;
+class IDocumentDBOwner;
+
+/**
+ * The document database contains all the necessary structures required per
+ * document type. It has an internal single-threaded Executor to process input
+ * to ensure that there are never multiple writers. Unless explicitly stated,
+ * none of the methods of this class are thread-safe.
+ */
+class DocumentDB : public IDocumentDBConfigOwner,
+                   public IReplayConfig,
+                   public FeedHandler::IOwner,
+                   public IDocumentSubDB::IOwner,
+                   public IClusterStateChangedHandler,
+                   public IWipeOldRemovedFieldsHandler,
+                   public search::transactionlog::SyncProxy
+{
+private:
+    class MetricsUpdateHook : public metrics::MetricManager::UpdateHook {
+        DocumentDBMetricsCollection _metrics;
+        DocumentDB &_db;
+    public:
+        MetricsUpdateHook(DocumentDB &s, const std::string &doc_type, size_t maxNumThreads)
+            : metrics::MetricManager::UpdateHook("documentdb-hook"),
+              _metrics(doc_type, maxNumThreads),
+              _db(s) {}
+        void updateMetrics(const MetricLockGuard & ) override { _db.updateMetrics(_metrics); }
+        DocumentDBMetricsCollection &getMetrics() { return _metrics; }
+    };
+
+
+    using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
+
+    DocTypeName                   _docTypeName;
+    vespalib::string              _baseDir;
+    // Only one thread per executor, or dropFeedView() will fail.
+    ExecutorThreadingService      _writeService;
+    // threads for initializer tasks during proton startup
+    InitializeThreads             _initializeThreads;
+
+    typedef search::SerialNum      SerialNum;
+    typedef fastos::TimeStamp      TimeStamp;
+    typedef vespalib::Closure      Closure;
+    typedef search::index::Schema  Schema;
+    // variables related to reconfig
+    DocumentDBConfig::SP          _initConfigSnapshot;
+    SerialNum                     _initConfigSerialNum;
+    vespalib::VarHolder<DocumentDBConfig::SP> _pendingConfigSnapshot;
+    vespalib::Lock                _configLock;  // protects _active* below.
+    DocumentDBConfig::SP          _activeConfigSnapshot;
+    int64_t                       _activeConfigSnapshotGeneration;
+    SerialNum                     _activeConfigSnapshotSerialNum;
+
+    vespalib::Gate                _initGate;
+
+    typedef DocumentDBConfig::ComparisonResult ConfigComparisonResult;
+
+    ClusterStateHandler           _clusterStateHandler;
+    BucketHandler                 _bucketHandler;
+    ProtonConfig::Summary         _protonSummaryCfg;
+    ProtonConfig::Index           _protonIndexCfg;
+    ConfigStore::UP               _config_store;
+    matching::SessionManager::SP  _sessionManager; // TODO: This should not have to be a shared pointer.
+    MetricsWireService             &_metricsWireService;
+    MetricsUpdateHook             _metricsHook;
+    vespalib::VarHolder<IFeedView::SP>      _feedView;
+    vespalib::Monitor             _refCountMonitor;
+    uint32_t                      _refCount;
+    bool                          _syncFeedViewEnabled;
+    IDocumentDBOwner             &_owner;
+    DDBState                      _state;
+    AttributeUsageFilter          _writeFilter;
+    FeedHandler                   _feedHandler;
+
+    // Members only accessed by executor thread
+    // (+ ctor and after executor stops)
+    Schema::SP                    _historySchema; // Removed fields
+    // current schema + _historySchema
+    Schema::SP                    _unionSchema;
+    // End members only accessed by executor thread.
+
+    DocumentSubDBCollection       _subDBs;
+    MaintenanceController         _maintenanceController;
+    VisibilityHandler             _visibility;
+    ILidSpaceCompactionHandler::Vector _lidSpaceCompactionHandlers;
+    DocumentDBJobTrackers         _jobTrackers;
+
+    // Last updated cache statistics. Necessary due to metrics implementation is upside down.
+    search::CacheStats            _lastDocStoreCacheStats;
+    IBucketStateCalculator::SP    _calc;
+
+    void setActiveConfig(const DocumentDBConfig::SP &config, SerialNum serialNum);
+    DocumentDBConfig::SP getActiveConfig() const;
+    void internalInit();
+    void initManagers();
+    void initFinish(DocumentDBConfig::SP configSnapshot);
+    void performReconfig(DocumentDBConfig::SP configSnapshot);
+    void closeSubDBs();
+
+    bool
+    handleRejectedConfig(DocumentDBConfig::SP &configSnapshot,
+                         const ConfigValidator::Result &cvr,
+                         const DDBState::ConfigState &cs);
+    void applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum);
+
+    /**
+     * Save initial config if we don't have any saved config snapshots.
+     *
+     * @param configSnapshot initial config snapshot.
+     */
+    void saveInitialConfig(const DocumentDBConfig &configSnapshot);
+
+    /**
+     * Resume interrupted config save if needed.
+     */
+    void resumeSaveConfig(void);
+
+    void
+    reconfigureSchema(const DocumentDBConfig &configSnapshot,
+                      const DocumentDBConfig &oldConfigSnapshot);
+
+    void setIndexSchema(const DocumentDBConfig &configSnapshot);
+
+    /**
+     * Redo interrupted reprocessing if last entry in transaction log
+     * is a config change.
+     */
+    virtual void enterRedoReprocessState();
+    void enterApplyLiveConfigState();
+
+    /**
+     * Drop old field view in a controlled manner.  The feed view will
+     * be kept alive until the index executor is done with all current
+     * tasks.
+     *
+     * Called by executor thread.
+     *
+     * @param feedView	shared pointer to feed view to be dropped.
+     */
+    void performDropFeedView(IFeedView::SP feedView);
+    void performDropFeedView2(IFeedView::SP feedView);
+
+    /**
+     * Implements FeedHandler::IOwner
+     */
+    virtual void onTransactionLogReplayDone() __attribute__((noinline));
+    virtual void onPerformPrune(SerialNum oldestSerial);
+    virtual bool isFeedBlockedByRejectedConfig();
+
+    /**
+     * Implements FeedHandler::IOwner
+     **/
+    virtual void performWipeHistory();
+    virtual bool getAllowPrune(void) const;
+
+    void
+    writeWipeHistoryTransactionLogEntry(
+            SerialNum wipeSerial,
+            TimeStamp wipeTimeLimit,
+            const DocumentDBConfig &configSnapshot,
+            const Schema &newHistorySchema);
+
+    void internalWipeHistory(SerialNum wipeSerial, Schema::UP newHistorySchema, const Schema &wipeSchema);
+
+    void startTransactionLogReplay();
+
+
+    /**
+     * Implements IClusterStateChangedHandler
+     */
+    virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc);
+    void notifyAllBucketsChanged();
+
+    /**
+     * Implements IWipeOldRemovedFieldsHandler
+     */
+    virtual void wipeOldRemovedFields(TimeStamp wipeTimeLimit);
+    void updateMetrics(LegacyDocumentDBMetrics &metrics);
+    void updateMetrics(DocumentDBTaggedMetrics &metrics);
+    void updateMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics);
+
+public:
+    typedef std::unique_ptr<DocumentDB> UP;
+    typedef std::shared_ptr<DocumentDB> SP;
+
+    /**
+     * Constructs a new document database for the given document type.
+     *
+     * @param baseDir The base directory to use for persistent data.
+     * @param configId The config id used to subscribe to config for this
+     *                 database.
+     * @param tlsSpec The frt connection spec for the TLS.
+     * @param docType The document type that this database will handle.
+     * @param docMgrCfg Current document manager config
+     * @param docMgrSP  The document manager holding the document type.
+     * @param protonCfg The global proton config this database is a part of.
+     * @param tuneFileDocumentDB file tune config for this database.
+     * @param config_store Access to read and write configs.
+     */
+    DocumentDB(const vespalib::string &baseDir,
+               const DocumentDBConfig::SP & currentSnapshot,
+               const vespalib::string &tlsSpec,
+               matching::QueryLimiter & queryLimiter,
+               const vespalib::Clock &clock,
+               const DocTypeName &docTypeName,
+               const ProtonConfig &protonCfg,
+               IDocumentDBOwner & owner,
+               vespalib::ThreadExecutor & warmupExecutor,
+               vespalib::ThreadStackExecutorBase & summaryExecutor,
+               search::transactionlog::Writer * tlsDirectWriter,
+               MetricsWireService &metricsWireService,
+               const search::common::FileHeaderContext &fileHeaderContext,
+               ConfigStore::UP config_store,
+               InitializeThreads initializeThreads);
+
+    /**
+     * Expose a cost view of the session manager. This is used by the
+     * document db explorer.
+     **/
+    const matching::SessionManager &session_manager() const {
+        return *_sessionManager;
+    }
+
+    /**
+     * Frees any allocated resources. This will also stop the internal thread
+     * and wait for it to finish. All pending tasks are deleted.
+     */
+    ~DocumentDB();
+
+    /**
+     * Starts initialization of the document db in the init & executor threads,
+     * and after that replay of the transaction log.
+     * Should be used during normal startup.
+     */
+    void start();
+
+    /**
+     * Used to wait for init completion without also waiting for a
+     * full replay to complete.
+     **/
+    void waitForInitDone();
+
+    /**
+     Close down all threads and make sure everything is ready to be shutdown.
+     */
+    void close();
+
+    /**
+     * Obtain the metrics collection for this document db.
+     *
+     * @return document db metrics
+     **/
+    DocumentDBMetricsCollection &getMetricsCollection() { return _metricsHook.getMetrics(); }
+
+    /**
+     * Obtain the metrics update hook for this document db.
+     *
+     * @return metrics update hook
+     **/
+    metrics::MetricManager::UpdateHook & getMetricsUpdateHook(void) {
+        return _metricsHook;
+    }
+
+    /**
+     * Returns the number of documents that are contained in this database.
+     *
+     * @return The document count.
+     */
+    size_t getNumDocs() const;
+
+    /**
+     * Returns the number of documents that are active for search in this database.
+     *
+     * @return The active-document count.
+     */
+    size_t getNumActiveDocs() const;
+
+    /**
+     * Returns the base directory that this document database uses when
+     * persisting data to disk.
+     *
+     * @return The directory name.
+     */
+    const vespalib::string &getBaseDirectory() const { return _baseDir; }
+
+
+    const DocumentSubDBCollection &getDocumentSubDBs() const { return _subDBs; }
+    IDocumentSubDB *getReadySubDB() { return _subDBs.getReadySubDB(); }
+    const IDocumentSubDB *getReadySubDB() const { return _subDBs.getReadySubDB(); }
+
+    bool hasDocument(const document::DocumentId &id);
+
+    /**
+     * Returns the feed handler for this database.
+     */
+    FeedHandler & getFeedHandler(void) { return _feedHandler; }
+
+    /**
+     * Returns the bucket handler for this database.
+     */
+    BucketHandler & getBucketHandler(void) { return _bucketHandler; }
+
+    /**
+     * Returns the cluster state handler for this database.
+     */
+    ClusterStateHandler & getClusterStateHandler() { return _clusterStateHandler; }
+
+    /**
+     * Create a set of document retrievers for this database. Note
+     * that the returned objects will not retain/release the database,
+     * and may only be used as long as the database is retained by
+     * some other means. The returned objects will protect from
+     * reconfiguration, however.
+     */
+    std::shared_ptr<std::vector<IDocumentRetriever::SP> >
+    getDocumentRetrievers() { return _subDBs.getRetrievers(); }
+
+    MaintenanceController &getMaintenanceController() {
+        return _maintenanceController;
+    }
+
+    BucketGuard::UP lockBucket(const document::BucketId &bucket);
+
+    virtual SerialNum getOldestFlushedSerial();
+
+    virtual SerialNum getNewestFlushedSerial();
+
+    search::engine::SearchReply::UP
+    match(const ISearchHandler::SP &searchHandler,
+          const search::engine::SearchRequest &req,
+          vespalib::ThreadBundle &threadBundle) const;
+
+    search::engine::DocsumReply::UP
+    getDocsums(const search::engine::DocsumRequest & request);
+
+    IFlushTarget::List getFlushTargets();
+    void flushDone(SerialNum oldestSerial);
+
+    virtual SerialNum
+    getCurrentSerialNumber() const
+    {
+	// Called by flush scheduler thread, by executor task or
+	// visitor callback.
+        // XXX: Contains future value during replay.
+        return _feedHandler.getSerialNum();
+    }
+
+    StatusReport::UP reportStatus() const;
+
+    /**
+     * Reference counting
+     */
+    void retain();
+    void release();
+
+    bool getRejectedConfig() const { return _state.getRejectedConfig(); }
+    void wipeHistory(void);
+
+    /**
+     * Implements IReplayConfig API.
+     */
+    virtual void replayConfig(SerialNum serialNum);
+
+    virtual void replayWipeHistory(SerialNum serialNum, TimeStamp wipeTimeLimit);
+
+    const DocTypeName & getDocTypeName(void) const { return _docTypeName; }
+
+    void
+    listSchema(std::vector<vespalib::string> &fieldNames,
+               std::vector<vespalib::string> &fieldDataTypes,
+               std::vector<vespalib::string> &fieldCollectionTypes,
+               std::vector<vespalib::string> &fieldLocations);
+
+    void newConfigSnapshot(DocumentDBConfig::SP snapshot);
+
+    // Implements DocumentDBConfigOwner
+    void reconfigure(const DocumentDBConfig::SP & snapshot);
+
+    int64_t getActiveGeneration() const;
+
+    // Implements IDocSubDB::IOwner
+    void syncFeedView() override;
+
+    searchcorespi::IIndexManagerFactory::SP
+    getIndexManagerFactory(const vespalib::stringref & name) const override;
+
+    vespalib::string getName() const override { return _docTypeName.getName(); }
+    uint32_t getDistributionKey() const override;
+
+    /**
+     * Implements FeedHandler::IOwner
+     **/
+    void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config);
+    void performStartMaintenance(void);
+    void stopMaintenance(void);
+    void forwardMaintenanceConfig(void);
+
+    /**
+     * Updates metrics collection object, and resets executor stats.
+     * Called by the metrics update hook (typically in the context of
+     * the metric manager). Do not call this function in multiple
+     * threads at once.
+     **/
+    void updateMetrics(DocumentDBMetricsCollection &metrics);
+
+    /**
+     * Implement search::transactionlog::SyncProxy API.
+     *
+     * Sync transaction log to syncTo.
+     */
+    virtual void sync(SerialNum syncTo);
+    void enterReprocessState();
+    void enterOnlineState();
+    void waitForOnlineState();
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_commit_job.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_commit_job.cpp
new file mode 100644
index 00000000000..b3293eb2852
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_commit_job.cpp
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "documentdb_commit_job.h"
+
+namespace proton {
+
+DocumentDBCommitJob::DocumentDBCommitJob(ICommitable & committer, fastos::TimeStamp visibilityDelay) :
+    IMaintenanceJob("documentdb_commit", visibilityDelay.sec(), visibilityDelay.sec()),
+    _committer(committer)
+{
+}
+
+bool
+DocumentDBCommitJob::run()
+{
+    _committer.commit();
+    return true;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_commit_job.h b/searchcore/src/vespa/searchcore/proton/server/documentdb_commit_job.h
new file mode 100644
index 00000000000..84ebc463dc4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_commit_job.h
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_maintenance_job.h"
+#include "icommitable.h"
+
+namespace proton {
+
+/**
+ * Job that regularly commits the documentdb.
+ */
+class DocumentDBCommitJob : public IMaintenanceJob
+{
+private:
+    ICommitable & _committer;
+
+public:
+    DocumentDBCommitJob(ICommitable & committer, fastos::TimeStamp visibilityDelay);
+
+    bool run() override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
new file mode 100644
index 00000000000..a2487671134
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
@@ -0,0 +1,245 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "documentdbconfig.h"
+
+using namespace config;
+using namespace vespa::config::search::summary;
+using namespace vespa::config::search;
+
+using document::DocumentTypeRepo;
+using document::DocumenttypesConfig;
+using search::TuneFileDocumentDB;
+using search::index::Schema;
+using vespa::config::search::SummarymapConfig;
+
+namespace proton {
+
+DocumentDBConfig::ComparisonResult::ComparisonResult()
+    : rankProfilesChanged(false),
+      indexschemaChanged(false),
+      attributesChanged(false),
+      summaryChanged(false),
+      summarymapChanged(false),
+      juniperrcChanged(false),
+      _documenttypesChanged(false),
+      _documentTypeRepoChanged(false),
+      _tuneFileDocumentDBChanged(false),
+      _schemaChanged(false),
+      _maintenanceChanged(false)
+{
+}
+
+DocumentDBConfig::DocumentDBConfig(
+               int64_t generation,
+               const RankProfilesConfigSP &rankProfiles,
+               const IndexschemaConfigSP &indexschema,
+               const AttributesConfigSP &attributes,
+               const SummaryConfigSP &summary,
+               const SummarymapConfigSP &summarymap,
+               const JuniperrcConfigSP &juniperrc,
+               const DocumenttypesConfigSP &documenttypes,
+               const DocumentTypeRepo::SP &repo,
+               const search::TuneFileDocumentDB::SP &tuneFileDocumentDB,
+               const Schema::SP &schema,
+               const DocumentDBMaintenanceConfig::SP &maintenance,
+               const vespalib::string &configId,
+               const vespalib::string &docTypeName,
+               const config::ConfigSnapshot & extraConfigs)
+    : _configId(configId),
+      _docTypeName(docTypeName),
+      _generation(generation),
+      _rankProfiles(rankProfiles),
+      _indexschema(indexschema),
+      _attributes(attributes),
+      _summary(summary),
+      _summarymap(summarymap),
+      _juniperrc(juniperrc),
+      _documenttypes(documenttypes),
+      _repo(repo),
+      _tuneFileDocumentDB(tuneFileDocumentDB),
+      _schema(schema),
+      _maintenance(maintenance),
+      _extraConfigs(extraConfigs),
+      _orig()
+{
+}
+
+
+DocumentDBConfig::
+DocumentDBConfig(const DocumentDBConfig &cfg)
+    : _configId(cfg._configId),
+      _docTypeName(cfg._docTypeName),
+      _generation(cfg._generation),
+      _rankProfiles(cfg._rankProfiles),
+      _indexschema(cfg._indexschema),
+      _attributes(cfg._attributes),
+      _summary(cfg._summary),
+      _summarymap(cfg._summarymap),
+      _juniperrc(cfg._juniperrc),
+      _documenttypes(cfg._documenttypes),
+      _repo(cfg._repo),
+      _tuneFileDocumentDB(cfg._tuneFileDocumentDB),
+      _schema(cfg._schema),
+      _maintenance(cfg._maintenance),
+      _extraConfigs(cfg._extraConfigs),
+      _orig(cfg._orig)
+{
+}
+
+bool
+DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const
+{
+    return equals<RankProfilesConfig>(_rankProfiles.get(),
+                                      rhs._rankProfiles.get()) &&
+           equals<IndexschemaConfig>(_indexschema.get(),
+                                     rhs._indexschema.get()) &&
+           equals<AttributesConfig>(_attributes.get(),
+                                    rhs._attributes.get()) &&
+           equals<SummaryConfig>(_summary.get(),
+                                 rhs._summary.get()) &&
+           equals<SummarymapConfig>(_summarymap.get(),
+                                    rhs._summarymap.get()) &&
+           equals<JuniperrcConfig>(_juniperrc.get(),
+                                   rhs._juniperrc.get()) &&
+           equals<DocumenttypesConfig>(_documenttypes.get(),
+                                       rhs._documenttypes.get()) &&
+           _repo.get() == rhs._repo.get() &&
+           equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(),
+                                      rhs._tuneFileDocumentDB.get()) &&
+           equals<Schema>(_schema.get(),
+                          rhs._schema.get()) &&
+        equals<DocumentDBMaintenanceConfig>(_maintenance.get(),
+                rhs._maintenance.get());
+}
+
+
+DocumentDBConfig::ComparisonResult
+DocumentDBConfig::compare(const DocumentDBConfig &rhs) const
+{
+    ComparisonResult retval;
+    retval.rankProfilesChanged =
+        !equals<RankProfilesConfig>(_rankProfiles.get(),
+                                    rhs._rankProfiles.get());
+    retval.indexschemaChanged =
+        !equals<IndexschemaConfig>(_indexschema.get(), rhs._indexschema.get());
+    retval.attributesChanged =
+        !equals<AttributesConfig>(_attributes.get(), rhs._attributes.get());
+    retval.summaryChanged =
+        !equals<SummaryConfig>(_summary.get(), rhs._summary.get());
+    retval.summarymapChanged =
+        !equals<SummarymapConfig>(_summarymap.get(), rhs._summarymap.get());
+    retval.juniperrcChanged =
+        !equals<JuniperrcConfig>(_juniperrc.get(), rhs._juniperrc.get());
+    retval._documenttypesChanged =
+        !equals<DocumenttypesConfig>(_documenttypes.get(),
+                rhs._documenttypes.get());
+    retval._documentTypeRepoChanged = _repo.get() != rhs._repo.get();
+    retval._tuneFileDocumentDBChanged =
+        !equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(),
+                rhs._tuneFileDocumentDB.get());
+    retval._schemaChanged =
+        !equals<Schema>(_schema.get(), rhs._schema.get());
+    retval._maintenanceChanged =
+        !equals<DocumentDBMaintenanceConfig>(_maintenance.get(),
+                rhs._maintenance.get());
+    return retval;
+}
+
+
+bool
+DocumentDBConfig::valid(void) const
+{
+    return _rankProfiles.get() != NULL &&
+            _indexschema.get() != NULL &&
+             _attributes.get() != NULL &&
+                _summary.get() != NULL &&
+             _summarymap.get() != NULL &&
+              _juniperrc.get() != NULL &&
+          _documenttypes.get() != NULL &&
+                   _repo.get() != NULL &&
+     _tuneFileDocumentDB.get() != NULL &&
+                 _schema.get() != NULL &&
+            _maintenance.get() != NULL;
+}
+
+namespace
+{
+
+template <class Config>
+std::shared_ptr<Config>
+emptyConfig(std::shared_ptr<Config> config)
+{
+    std::shared_ptr<Config> empty(std::make_shared<Config>());
+    
+    if (!config || *config != *empty) {
+        return empty;
+    }
+    return config;
+}
+
+}
+
+
+DocumentDBConfig::SP
+DocumentDBConfig::makeReplayConfig(const SP & orig)
+{
+    const DocumentDBConfig &o = *orig;
+    
+    SP ret = std::make_shared<DocumentDBConfig>(
+                o._generation,
+                emptyConfig(o._rankProfiles),
+                o._indexschema,
+                o._attributes,
+                o._summary,
+                o._summarymap,
+                o._juniperrc,
+                o._documenttypes,
+                o._repo,
+                o._tuneFileDocumentDB,
+                o._schema,
+                o._maintenance,
+                o._configId,
+                o._docTypeName,
+                o._extraConfigs);
+    ret->_orig = orig;
+    return ret;
+}
+
+
+DocumentDBConfig::SP
+DocumentDBConfig::getOriginalConfig() const
+{
+    return _orig;
+}
+
+
+DocumentDBConfig::SP
+DocumentDBConfig::preferOriginalConfig(const SP & self)
+{
+    return (self && self->_orig) ? self->_orig : self;
+}
+
+
+DocumentDBConfig::SP
+DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes) const
+{
+    return std::make_shared<DocumentDBConfig>(
+            _generation,
+            _rankProfiles,
+            _indexschema,
+            attributes,
+            _summary,
+            _summarymap,
+            _juniperrc,
+            _documenttypes,
+            _repo,
+            _tuneFileDocumentDB,
+            _schema,
+            _maintenance,
+            _configId,
+            _docTypeName,
+            _extraConfigs);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
new file mode 100644
index 00000000000..659b11ef8c1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
@@ -0,0 +1,199 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "document_db_maintenance_config.h"
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchsummary/config/config-juniperrc.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/config-attributes.h>
+#include <vespa/config-indexschema.h>
+#include <vespa/config-rank-profiles.h>
+#include <vespa/config-summary.h>
+#include <vespa/config-summarymap.h>
+#include <vespa/config/retriever/configkeyset.h>
+#include <vespa/config/retriever/configsnapshot.h>
+
+namespace proton {
+
+class DocumentDBConfig
+{
+public:
+    class ComparisonResult
+    {
+    public:
+        bool rankProfilesChanged;
+        bool indexschemaChanged;
+        bool attributesChanged;
+        bool summaryChanged;
+        bool summarymapChanged;
+        bool juniperrcChanged;
+        bool _documenttypesChanged;
+        bool _documentTypeRepoChanged;
+        bool _tuneFileDocumentDBChanged;
+        bool _schemaChanged;
+        bool _maintenanceChanged;
+
+        ComparisonResult();
+    };
+
+    typedef std::shared_ptr<DocumentDBConfig> SP;
+    typedef std::shared_ptr<vespa::config::search::IndexschemaConfig> IndexschemaConfigSP;
+    typedef std::shared_ptr<vespa::config::search::AttributesConfig> AttributesConfigSP;
+    typedef std::shared_ptr<vespa::config::search::RankProfilesConfig> RankProfilesConfigSP;
+    typedef std::shared_ptr<vespa::config::search::SummaryConfig> SummaryConfigSP;
+    typedef std::shared_ptr<vespa::config::search::SummarymapConfig> SummarymapConfigSP;
+    typedef std::shared_ptr<vespa::config::search::summary::JuniperrcConfig> JuniperrcConfigSP;
+    typedef std::shared_ptr<document::DocumenttypesConfig> DocumenttypesConfigSP;
+    typedef DocumentDBMaintenanceConfig::SP MaintenanceConfigSP;
+
+private:
+    vespalib::string               _configId;
+    vespalib::string               _docTypeName;
+    int64_t                        _generation;
+    RankProfilesConfigSP           _rankProfiles;
+    IndexschemaConfigSP            _indexschema;
+    AttributesConfigSP             _attributes;
+    SummaryConfigSP                _summary;
+    SummarymapConfigSP             _summarymap;
+    JuniperrcConfigSP              _juniperrc;
+    DocumenttypesConfigSP          _documenttypes;
+    document::DocumentTypeRepo::SP _repo;
+    search::TuneFileDocumentDB::SP _tuneFileDocumentDB;
+    search::index::Schema::SP      _schema;
+    MaintenanceConfigSP            _maintenance;
+    config::ConfigSnapshot         _extraConfigs;
+    SP _orig;
+
+
+    template <typename T>
+    bool equals(const T * lhs, const T * rhs) const
+    {
+        if (lhs == NULL)
+            return rhs == NULL;
+        return rhs != NULL && *lhs == *rhs;
+    }
+public:
+    DocumentDBConfig(int64_t generation,
+                     const RankProfilesConfigSP &rankProfiles,
+                     const IndexschemaConfigSP &indexschema,
+                     const AttributesConfigSP &attributes,
+                     const SummaryConfigSP &summary,
+                     const SummarymapConfigSP &summarymap,
+                     const JuniperrcConfigSP &juniperrc,
+                     const DocumenttypesConfigSP &documenttypesConfig,
+                     const document::DocumentTypeRepo::SP &repo,
+                     const search::TuneFileDocumentDB::SP & tuneFileDocumentDB,
+                     const search::index::Schema::SP &schema,
+                     const DocumentDBMaintenanceConfig::SP &maintenance,
+                     const vespalib::string &configId,
+                     const vespalib::string &docTypeName,
+                     const config::ConfigSnapshot & extraConfig = config::ConfigSnapshot());
+
+    DocumentDBConfig(const DocumentDBConfig &cfg);
+
+    const vespalib::string & getConfigId() const { return _configId; }
+    void setConfigId(const vespalib::string &configId) { _configId = configId; }
+
+    const vespalib::string &getDocTypeName() const { return _docTypeName; }
+
+    int64_t getGeneration(void) const { return _generation; }
+
+    const vespa::config::search::RankProfilesConfig &
+    getRankProfilesConfig() const { return *_rankProfiles; }
+
+    const vespa::config::search::IndexschemaConfig &
+    getIndexschemaConfig() const { return *_indexschema; }
+
+    const vespa::config::search::AttributesConfig &
+    getAttributesConfig() const { return *_attributes; }
+
+    const vespa::config::search::SummaryConfig &
+    getSummaryConfig() const { return *_summary; }
+
+    const vespa::config::search::SummarymapConfig &
+    getSummarymapConfig() const { return *_summarymap; }
+
+    const vespa::config::search::summary::JuniperrcConfig &
+    getJuniperrcConfig() const { return *_juniperrc; }
+
+    const document::DocumenttypesConfig &
+    getDocumenttypesConfig(void) const { return *_documenttypes; }
+
+    const RankProfilesConfigSP &
+    getRankProfilesConfigSP(void) const { return _rankProfiles; }
+
+    const IndexschemaConfigSP &
+    getIndexschemaConfigSP(void) const { return _indexschema; }
+
+    const AttributesConfigSP &
+    getAttributesConfigSP(void) const { return _attributes; }
+
+    const SummaryConfigSP &
+    getSummaryConfigSP(void) const { return _summary; }
+
+    const SummarymapConfigSP &
+    getSummarymapConfigSP(void) const { return _summarymap; }
+
+    const JuniperrcConfigSP &
+    getJuniperrcConfigSP(void) const { return _juniperrc; }
+
+    const DocumenttypesConfigSP &
+    getDocumenttypesConfigSP(void) const { return _documenttypes; }
+
+    const document::DocumentTypeRepo::SP &
+    getDocumentTypeRepoSP() const { return _repo; }
+
+    const document::DocumentType *
+    getDocumentType() const { return _repo->getDocumentType(getDocTypeName()); }
+
+    const search::index::Schema::SP &
+    getSchemaSP(void) const { return _schema; }
+
+    const MaintenanceConfigSP &
+    getMaintenanceConfigSP(void) const { return _maintenance; }
+
+    const search::TuneFileDocumentDB::SP &
+    getTuneFileDocumentDBSP(void) const { return _tuneFileDocumentDB; }
+
+    bool
+    operator==(const DocumentDBConfig &rhs) const;
+
+     /**
+      * Compare this snapshot with the given one.
+      */
+    ComparisonResult
+    compare(const DocumentDBConfig &rhs) const;
+
+    bool valid(void) const;
+
+    const config::ConfigSnapshot & getExtraConfigs() const { return _extraConfigs; }
+    void setExtraConfigs(const config::ConfigSnapshot &extraConfigs) { _extraConfigs = extraConfigs; }
+
+    /**
+     * Only keep configs needed for replay of transaction log.
+     */
+    static SP makeReplayConfig(const SP & orig);
+
+    /**
+     * Return original config if this is a replay config, otherwise return
+     * empty shared pointer.
+     */
+    SP getOriginalConfig() const;
+
+    /**
+     * Return original config if cfg is a replay config, otherwise return
+     * cfg.
+     */
+    static SP preferOriginalConfig(const SP & cfg);
+
+    /**
+     * Create modified attributes config.
+     */
+    SP newFromAttributesConfig(const AttributesConfigSP &attributes) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
new file mode 100644
index 00000000000..d566949eb3b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -0,0 +1,268 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "documentdbconfigmanager.h"
+#include <vespa/log/log.h>
+#include <vespa/searchcommon/common/schemaconfigurer.h>
+#include <vespa/searchlib/index/schemautil.h>
+LOG_SETUP(".proton.server.documentdbconfigmanager");
+#include <vespa/config/helper/legacy.h>
+
+using namespace config;
+using namespace vespa::config::search::core;
+using namespace vespa::config::search::summary;
+using namespace vespa::config::search;
+
+using document::DocumentTypeRepo;
+using search::TuneFileDocumentDB;
+using search::index::Schema;
+using search::index::SchemaBuilder;
+using fastos::TimeStamp;
+
+namespace proton {
+
+const ConfigKeySet
+DocumentDBConfigManager::createConfigKeySet() const
+{
+    ConfigKeySet set;
+    set.add<RankProfilesConfig,
+            IndexschemaConfig,
+            AttributesConfig,
+            SummaryConfig,
+            SummarymapConfig,
+            JuniperrcConfig>(_configId);
+    set.add(_extraConfigKeys);
+    return set;
+}
+
+Schema::SP
+DocumentDBConfigManager::
+buildNewSchema(const AttributesConfig & newAttributesConfig,
+               const SummaryConfig & newSummaryConfig,
+               const IndexschemaConfig & newIndexschemaConfig)
+{
+    // Called with lock held
+    Schema::SP schema(new Schema);
+    SchemaBuilder::build(newAttributesConfig, *schema);
+    SchemaBuilder::build(newSummaryConfig, *schema);
+    SchemaBuilder::build(newIndexschemaConfig, *schema);
+    return schema;
+}
+
+
+Schema::SP
+DocumentDBConfigManager::
+buildSchema(const AttributesConfig & newAttributesConfig,
+            const SummaryConfig & newSummaryConfig,
+            const IndexschemaConfig & newIndexschemaConfig)
+{
+    // Called with lock held
+    Schema::SP oldSchema;
+    if (_pendingConfigSnapshot.get() != NULL)
+        oldSchema = _pendingConfigSnapshot->getSchemaSP();
+    if (oldSchema.get() == NULL)
+        return buildNewSchema(newAttributesConfig,
+                              newSummaryConfig, newIndexschemaConfig);
+    const DocumentDBConfig &old = *_pendingConfigSnapshot;
+    if (old.getAttributesConfig() != newAttributesConfig ||
+        old.getSummaryConfig() != newSummaryConfig ||
+        old.getIndexschemaConfig() != newIndexschemaConfig) {
+        Schema::SP schema(buildNewSchema(newAttributesConfig,
+                                  newSummaryConfig, newIndexschemaConfig));
+        return (*oldSchema == *schema) ? oldSchema : schema;
+    }
+    return oldSchema;
+}
+
+
+static DocumentDBMaintenanceConfig::SP
+buildMaintenanceConfig(const BootstrapConfig::SP &bootstrapConfig,
+                       const vespalib::string &docTypeName)
+{
+    typedef ProtonConfig::Documentdb DdbConfig;
+    ProtonConfig &proton(bootstrapConfig->getProtonConfig());
+
+    TimeStamp visibilityDelay;
+    // Use document type to find document db config in proton config
+    uint32_t index;
+    for (index = 0; index < proton.documentdb.size(); ++index) {
+        const DdbConfig &ddbConfig = proton.documentdb[index];
+        if (docTypeName == ddbConfig.inputdoctypename)
+            break;
+    }
+    double pruneRemovedDocumentsInterval = proton.pruneremoveddocumentsinterval;
+    double pruneRemovedDocumentsAge = proton.pruneremoveddocumentsage;
+
+    if (index < proton.documentdb.size()) {
+        const DdbConfig &ddbConfig = proton.documentdb[index];
+        visibilityDelay = TimeStamp::Seconds(std::min(proton.maxvisibilitydelay, ddbConfig.visibilitydelay));
+    }
+    return DocumentDBMaintenanceConfig::SP(
+            new DocumentDBMaintenanceConfig(
+                    DocumentDBPruneRemovedDocumentsConfig(
+                            pruneRemovedDocumentsInterval,
+                            pruneRemovedDocumentsAge),
+                    DocumentDBHeartBeatConfig(),
+                    DocumentDBWipeOldRemovedFieldsConfig(
+                            proton.wipeoldremovedfieldsinterval,
+                            proton.wipeoldremovedfieldsage),
+                    proton.grouping.sessionmanager.pruning.interval,
+                    visibilityDelay,
+                    DocumentDBLidSpaceCompactionConfig(
+                            proton.lidspacecompaction.interval,
+                            proton.lidspacecompaction.allowedlidbloat,
+                            proton.lidspacecompaction.allowedlidbloatfactor),
+                    AttributeUsageFilterConfig(
+                            proton.writefilter.attribute.enumstorelimit,
+                            proton.writefilter.attribute.multivaluelimit),
+                    proton.writefilter.sampleinterval));
+}
+
+
+void
+DocumentDBConfigManager::update(const ConfigSnapshot & snapshot)
+{
+    typedef DocumentDBConfig::RankProfilesConfigSP RankProfilesConfigSP;
+    typedef DocumentDBConfig::IndexschemaConfigSP IndexschemaConfigSP;
+    typedef DocumentDBConfig::AttributesConfigSP AttributesConfigSP;
+    typedef DocumentDBConfig::SummaryConfigSP SummaryConfigSP;
+    typedef DocumentDBConfig::SummarymapConfigSP SummarymapConfigSP;
+    typedef DocumentDBConfig::JuniperrcConfigSP JuniperrcConfigSP;
+    typedef DocumentDBConfig::MaintenanceConfigSP MaintenanceConfigSP;
+
+    DocumentDBConfig::SP current = _pendingConfigSnapshot;
+    RankProfilesConfigSP newRankProfilesConfig;
+    IndexschemaConfigSP newIndexschemaConfig;
+    AttributesConfigSP newAttributesConfig;
+    SummaryConfigSP newSummaryConfig;
+    SummarymapConfigSP newSummarymapConfig;
+    JuniperrcConfigSP newJuniperrcConfig;
+    MaintenanceConfigSP oldMaintenanceConfig;
+    MaintenanceConfigSP newMaintenanceConfig;
+
+    if (!_ignoreForwardedConfig) {
+        if (_bootstrapConfig->getDocumenttypesConfigSP().get() == NULL ||
+            _bootstrapConfig->getDocumentTypeRepoSP().get() == NULL ||
+            _bootstrapConfig->getProtonConfigSP().get() == NULL ||
+            _bootstrapConfig->getTuneFileDocumentDBSP().get() == NULL)
+            return;
+    }
+
+
+    int64_t generation = snapshot.getGeneration();
+    LOG(debug,
+        "Forwarded generation %" PRId64 ", generation %" PRId64,
+        _bootstrapConfig->getGeneration(), generation);
+    if (!_ignoreForwardedConfig &&
+        _bootstrapConfig->getGeneration() != generation)
+            return;
+
+    int64_t currentGeneration = -1;
+    if (current.get() != NULL) {
+        newRankProfilesConfig = current->getRankProfilesConfigSP();
+        newIndexschemaConfig = current->getIndexschemaConfigSP();
+        newAttributesConfig = current->getAttributesConfigSP();
+        newSummaryConfig = current->getSummaryConfigSP();
+        newSummarymapConfig = current->getSummarymapConfigSP();
+        newJuniperrcConfig = current->getJuniperrcConfigSP();
+        oldMaintenanceConfig = current->getMaintenanceConfigSP();
+        currentGeneration = current->getGeneration();
+    }
+
+    if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration))
+        newRankProfilesConfig =
+            RankProfilesConfigSP(
+                    snapshot.getConfig<RankProfilesConfig>(_configId).
+                    release());
+    if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) {
+        std::unique_ptr<IndexschemaConfig> indexschemaConfig =
+            snapshot.getConfig<IndexschemaConfig>(_configId);
+        search::index::Schema schema;
+        search::index::SchemaBuilder::build(*indexschemaConfig, schema);
+        if (!search::index::SchemaUtil::validateSchema(schema)) {
+            LOG(error,
+                "Cannot use bad index schema, validation failed");
+            abort();
+        }
+        newIndexschemaConfig =
+            IndexschemaConfigSP(indexschemaConfig.release());
+    }
+    if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration))
+        newAttributesConfig =
+            AttributesConfigSP(snapshot.getConfig<AttributesConfig>(_configId).
+                               release());
+    if (snapshot.isChanged<SummaryConfig>(_configId, currentGeneration))
+        newSummaryConfig =
+            SummaryConfigSP(snapshot.getConfig<SummaryConfig>(_configId).
+                            release());
+    if (snapshot.isChanged<SummarymapConfig>(_configId, currentGeneration))
+        newSummarymapConfig =
+            SummarymapConfigSP(snapshot.getConfig<SummarymapConfig>(_configId).
+                               release());
+    if (snapshot.isChanged<JuniperrcConfig>(_configId, currentGeneration))
+        newJuniperrcConfig =
+            JuniperrcConfigSP(
+                    snapshot.getConfig<JuniperrcConfig>(_configId).release());
+
+    Schema::SP schema(buildSchema(*newAttributesConfig,
+                                  *newSummaryConfig,
+                                  *newIndexschemaConfig));
+    newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig,
+                                                  _docTypeName);
+    if (newMaintenanceConfig.get() != NULL &&
+        oldMaintenanceConfig.get() != NULL &&
+        *newMaintenanceConfig == *oldMaintenanceConfig) {
+        newMaintenanceConfig = oldMaintenanceConfig;
+    }
+    ConfigSnapshot extraConfigs(snapshot.subset(_extraConfigKeys));
+    DocumentDBConfig::SP newSnapshot(
+            new DocumentDBConfig(generation,
+                                 newRankProfilesConfig,
+                                 newIndexschemaConfig,
+                                 newAttributesConfig,
+                                 newSummaryConfig,
+                                 newSummarymapConfig,
+                                 newJuniperrcConfig,
+                                 _bootstrapConfig->getDocumenttypesConfigSP(),
+                                 _bootstrapConfig->getDocumentTypeRepoSP(),
+                                 _bootstrapConfig->getTuneFileDocumentDBSP(),
+                                 schema,
+                                 newMaintenanceConfig,
+                                 _configId,
+                                 _docTypeName,
+                                 extraConfigs));
+    assert(newSnapshot->valid());
+    {
+        vespalib::LockGuard lock(_pendingConfigLock);
+        _pendingConfigSnapshot = newSnapshot;
+    }
+}
+
+
+DocumentDBConfigManager::
+DocumentDBConfigManager(const vespalib::string &configId,
+                        const vespalib::string &docTypeName)
+    : _configId(configId),
+      _docTypeName(docTypeName),
+      _bootstrapConfig(),
+      _pendingConfigSnapshot(),
+      _ignoreForwardedConfig(true),
+      _pendingConfigLock(),
+      _extraConfigKeys()
+{
+}
+
+void
+DocumentDBConfigManager::
+forwardConfig(const BootstrapConfig::SP & config)
+{
+    {
+        if (!_ignoreForwardedConfig &&
+            config->getGeneration() < _bootstrapConfig->getGeneration())
+            return;	// Enforce time direction
+        _bootstrapConfig = config;
+        _ignoreForwardedConfig = false;
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
new file mode 100644
index 00000000000..525505fcfb0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/config/config.h>
+#include "bootstrapconfig.h"
+#include "documentdbconfig.h"
+
+namespace proton {
+
+
+/**
+ * This class manages the subscription for documentdb configs.
+ */
+class DocumentDBConfigManager
+{
+public:
+    typedef std::shared_ptr<DocumentDBConfigManager> SP;
+
+private:
+    vespalib::string     _configId;
+    vespalib::string     _docTypeName;
+    BootstrapConfig::SP  _bootstrapConfig;
+    DocumentDBConfig::SP _pendingConfigSnapshot;
+    bool                 _ignoreForwardedConfig;
+    vespalib::Lock       _pendingConfigLock;
+    config::ConfigKeySet _extraConfigKeys;
+
+    search::index::Schema::SP
+    buildNewSchema(const vespa::config::search::AttributesConfig & newAttributesConfig,
+                   const vespa::config::search::SummaryConfig & newSummaryConfig,
+                   const vespa::config::search::IndexschemaConfig & newIndexschemaConfig);
+
+    search::index::Schema::SP
+    buildSchema(const vespa::config::search::AttributesConfig & newAttributesConfig,
+                const vespa::config::search::SummaryConfig & newSummaryConfig,
+                const vespa::config::search::IndexschemaConfig & newIndexschemaConfig);
+public:
+    DocumentDBConfigManager(const vespalib::string &configId,
+                            const vespalib::string &docTypeName);
+    void update(const config::ConfigSnapshot & snapshot);
+
+    DocumentDBConfig::SP
+    getConfig() const {
+        vespalib::LockGuard lock(_pendingConfigLock);
+        return _pendingConfigSnapshot;
+    }
+
+    void forwardConfig(const BootstrapConfig::SP & config);
+    const config::ConfigKeySet createConfigKeySet(void) const;
+    void setExtraConfigKeys(const config::ConfigKeySet & extraConfigKeys) { _extraConfigKeys = extraConfigKeys; }
+    const config::ConfigKeySet & getExtraConfigKeys() const { return _extraConfigKeys; }
+    const vespalib::string & getConfigId() const { return _configId; }
+};
+
+/**
+ * Simple helper class to use a config holder in tests and fileconfig manager.
+ */
+class DocumentDBConfigHelper
+{
+public:
+    DocumentDBConfigHelper(const config::DirSpec &spec,
+                           const vespalib::string &docTypeName,
+                           const config::ConfigKeySet &extraConfigKeys = config::ConfigKeySet())
+        : _mgr("", docTypeName),
+          _retriever()
+    {
+        _mgr.setExtraConfigKeys(extraConfigKeys);
+        _retriever.reset(new config::ConfigRetriever(_mgr.createConfigKeySet(),
+                                                     config::IConfigContext::SP(new config::ConfigContext(spec))));
+    }
+
+    bool
+    nextGeneration(int timeoutInMillis)
+    {
+        config::ConfigSnapshot
+            snapshot(_retriever->getBootstrapConfigs(timeoutInMillis));
+        if (snapshot.empty())
+            return false;
+        _mgr.update(snapshot);
+        return true;
+    }
+
+    DocumentDBConfig::SP
+    getConfig(void) const
+    {
+        return _mgr.getConfig();
+    }
+
+    void
+    forwardConfig(const BootstrapConfig::SP & config)
+    {
+        _mgr.forwardConfig(config);
+    }
+private:
+    DocumentDBConfigManager _mgr;
+    std::unique_ptr<config::ConfigRetriever> _retriever;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp
new file mode 100644
index 00000000000..2ebadce7e38
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "documentdbconfigscout.h"
+#include <vespa/searchcore/proton/attribute/attributesconfigscout.h>
+
+using vespa::config::search::AttributesConfig;
+
+namespace proton
+{
+
+
+DocumentDBConfig::SP
+DocumentDBConfigScout::scout(const DocumentDBConfig::SP &config,
+                             const DocumentDBConfig &liveConfig)
+{
+    AttributesConfigScout acScout(liveConfig.getAttributesConfig());
+    std::shared_ptr<AttributesConfig>
+        ac(acScout.adjust(config->getAttributesConfig()));
+    if (*ac == config->getAttributesConfig())
+        return config; // no change
+    return config->newFromAttributesConfig(ac);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.h
new file mode 100644
index 00000000000..3f8cd329249
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.h
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentdbconfig.h"
+
+namespace proton
+{
+
+/**
+ * Class to create adjusted document db config that minimizes the number of
+ * proton restarts needed due to config changes.  Grab the portions from
+ * live (supposedly future) config that is safe to apply early during
+ * initialization and replay.
+ */
+class DocumentDBConfigScout
+{
+public:
+    static DocumentDBConfig::SP
+    scout(const DocumentDBConfig::SP &config,
+          const DocumentDBConfig &liveConfig);
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
new file mode 100644
index 00000000000..e2315f4d633
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
@@ -0,0 +1,152 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.documentretriever");
+
+#include "documentretriever.h"
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchcore/proton/attribute/document_field_retriever.h>
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+#include <vespa/searchlib/docstore/idocumentstore.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/vespalib/geo/zcurve.h>
+
+using document::Document;
+using document::DocumentType;
+using document::DocumentTypeRepo;
+using document::PositionDataType;
+using document::StructFieldValue;
+using search::AttributeGuard;
+using search::DocumentIdT;
+using search::IAttributeManager;
+using search::IDocumentStore;
+using search::index::Schema;
+using storage::spi::Timestamp;
+using vespalib::geo::ZCurve;
+
+namespace proton {
+
+DocumentRetriever
+::DocumentRetriever(const DocTypeName &docTypeName,
+                    const DocumentTypeRepo &repo,
+                    const Schema &schema,
+                    const IDocumentMetaStoreContext &meta_store,
+                    const IAttributeManager &attr_manager,
+                    const IDocumentStore &doc_store)
+    : DocumentRetrieverBase(docTypeName, repo, meta_store, true),
+      _schema(schema),
+      _attr_manager(attr_manager),
+      _doc_store(doc_store),
+      _possiblePositionFields()
+{
+    const DocumentType * documentType = repo.getDocumentType(docTypeName.getName());
+    document::Field::Set fields = documentType->getFieldSet();
+    int32_t positionDataTypeId = PositionDataType::getInstance().getId();
+    LOG(debug, "checking document type '%s' for position fields", docTypeName.getName().c_str());
+    for (const document::Field * field : fields) {
+        if (field->getDataType().getId() == positionDataTypeId) {
+            LOG(debug, "Field '%s' is a position field", field->getName().c_str());
+            const vespalib::string & zcurve_name = PositionDataType::getZCurveFieldName(field->getName());
+            AttributeGuard::UP attr = attr_manager.getAttribute(zcurve_name);
+            if (attr) {
+                LOG(debug, "Field '%s' is a registered attribute field", zcurve_name.c_str());
+                _possiblePositionFields.emplace_back(field, zcurve_name);
+            }
+        }
+    }
+}
+
+namespace {
+
+FieldValue::UP positionFromZcurve(int64_t zcurve) {
+    int32_t x, y;
+    ZCurve::decode(zcurve, &x, &y);
+
+    FieldValue::UP value = PositionDataType::getInstance().createFieldValue();
+    StructFieldValue *position = static_cast<StructFieldValue *>(value.get());
+    position->set(PositionDataType::FIELD_X, x);
+    position->set(PositionDataType::FIELD_Y, y);
+    return value;
+}
+
+void fillInPositionFields(Document &doc, DocumentIdT lid, const DocumentRetriever::PositionFields & possiblePositionFields, const IAttributeManager & attr_manager)
+{
+    for (const auto & it : possiblePositionFields) {
+        if (doc.hasValue(*it.first)) {
+            AttributeGuard::UP attr = attr_manager.getAttribute(it.second);
+            if (attr.get() && attr->valid()) {
+                int64_t zcurve = (*attr)->getInt(lid);
+                doc.setValue(*it.first, *positionFromZcurve(zcurve));
+            }
+        }
+    }
+}
+
+class PopulateVisitor : public search::IDocumentVisitor
+{
+public:
+    PopulateVisitor(const DocumentRetriever & retriever, search::IDocumentVisitor & visitor) :
+        _retriever(retriever),
+        _visitor(visitor)
+    { }
+    void visit(uint32_t lid, document::Document::UP doc) override {
+        if (doc) {
+            _retriever.populate(lid, *doc);
+            _visitor.visit(lid, std::move(doc));
+        }
+    }
+private:
+    const DocumentRetriever  & _retriever;
+    search::IDocumentVisitor & _visitor;
+};
+
+}  // namespace
+
+Document::UP DocumentRetriever::getDocument(DocumentIdT lid) const
+{
+    Document::UP doc = _doc_store.read(lid, getDocumentTypeRepo());
+    if (doc) {
+        populate(lid, *doc);
+    }
+    return doc;
+}
+
+void DocumentRetriever::visitDocuments(const LidVector & lids, search::IDocumentVisitor & visitor, ReadConsistency) const
+{
+    PopulateVisitor populater(*this, visitor);
+    _doc_store.visit(lids, getDocumentTypeRepo(), populater);
+}
+
+void DocumentRetriever::populate(DocumentIdT lid, Document & doc) const
+{
+    for (uint32_t i = 0; i < _schema.getNumAttributeFields(); ++i) {
+        const Schema::AttributeField &field = _schema.getAttributeField(i);
+        AttributeGuard::UP attr = _attr_manager.getAttribute(field.getName());
+        if (attr.get() && attr->valid()) {
+            DocumentFieldRetriever::populate(lid, doc, field, **attr, _schema.isIndexField(field.getName()));
+        }
+    }
+    fillInPositionFields(doc, lid, _possiblePositionFields, _attr_manager);
+}
+
+const Schema &
+DocumentRetriever::getSchema(void) const
+{
+    return _schema;
+}
+
+
+const IAttributeManager *
+DocumentRetriever::getAttrMgr(void) const
+{
+    return &_attr_manager;
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.h b/searchcore/src/vespa/searchcore/proton/server/documentretriever.h
new file mode 100644
index 00000000000..94446a0c738
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentretrieverbase.h"
+
+namespace search {
+class IDocumentMetaStore;
+class IAttributeManager;
+class IDocumentStore;
+namespace index { class Schema; }
+}  // namespace search
+
+namespace proton {
+
+class DocumentRetriever : public DocumentRetrieverBase {
+public:
+    typedef std::vector<std::pair<const document::Field *, vespalib::string>> PositionFields;
+    DocumentRetriever(const DocTypeName &docTypeName,
+                      const document::DocumentTypeRepo &repo,
+                      const search::index::Schema &schema,
+                      const IDocumentMetaStoreContext &meta_store,
+                      const search::IAttributeManager &attr_manager,
+                      const search::IDocumentStore &doc_store);
+
+    document::Document::UP getDocument(search::DocumentIdT lid) const override;
+    void visitDocuments(const LidVector & lids, search::IDocumentVisitor & visitor, ReadConsistency) const override;
+    void populate(search::DocumentIdT lid, document::Document & doc) const;
+private:
+    const search::index::Schema     &_schema;
+    const search::IAttributeManager &_attr_manager;
+    const search::IDocumentStore    &_doc_store;
+    PositionFields                   _possiblePositionFields;
+
+    const search::index::Schema & getSchema(void) const override;
+    const search::IAttributeManager * getAttrMgr(void) const override;
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp
new file mode 100644
index 00000000000..2aadee9b320
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".documentretrieverbase");
+
+#include "documentretrieverbase.h"
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchlib/common/idocumentmetastore.h>
+#include <vespa/searchcore/proton/common/cachedselect.h>
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h>
+
+using document::DocumentId;
+using document::GlobalId;
+using search::index::Schema;
+
+namespace
+{
+
+const DocumentId docId("doc:test:1");
+const Schema emptySchema;
+
+}
+
+namespace proton
+{
+
+DocumentRetrieverBase::DocumentRetrieverBase(
+        const DocTypeName &docTypeName,
+        const document::DocumentTypeRepo &repo,
+        const IDocumentMetaStoreContext &meta_store,
+        bool hasFields)
+    : IDocumentRetriever(),
+      _docTypeName(docTypeName),
+      _repo(repo),
+      _meta_store(meta_store),
+      _selectCache(256u),
+      _lock(),
+      _emptyDoc(),
+      _hasFields(hasFields)
+{
+    const document::DocumentType *
+        docType(_repo.getDocumentType(_docTypeName.getName()));
+    _emptyDoc.reset(new document::Document(*docType, docId));
+    _emptyDoc->setRepo(_repo);
+}
+
+const document::DocumentTypeRepo &
+DocumentRetrieverBase::getDocumentTypeRepo() const {
+    return _repo;
+}
+
+void
+DocumentRetrieverBase::getBucketMetaData(
+        const storage::spi::Bucket &bucket,
+        search::DocumentMetaData::Vector &result) const {
+    IDocumentMetaStoreContext::IReadGuard::UP readGuard = _meta_store.getReadGuard();
+    const search::IDocumentMetaStore &meta_store = readGuard->get();
+    meta_store.getMetaData(bucket, result);
+}
+
+search::DocumentMetaData
+DocumentRetrieverBase::getDocumentMetaData(const DocumentId &id) const {
+    const GlobalId &gid = id.getGlobalId();
+    IDocumentMetaStoreContext::IReadGuard::UP readGuard = _meta_store.getReadGuard();
+    const search::IDocumentMetaStore &meta_store = readGuard->get();
+    return meta_store.getMetaData(gid);
+}
+
+
+const search::index::Schema &
+DocumentRetrieverBase::getSchema(void) const
+{
+    return emptySchema;
+}
+
+
+const search::IAttributeManager *
+DocumentRetrieverBase::getAttrMgr(void) const
+{
+    return NULL;
+}
+
+    
+CachedSelect::SP
+DocumentRetrieverBase::parseSelect(const vespalib::string &selection) const
+{
+    {
+        vespalib::LockGuard guard(_lock);
+        if (_selectCache.hasKey(selection))
+            return _selectCache[selection];
+    }
+    
+    CachedSelect::SP nselect(new CachedSelect());
+    
+    nselect->set(selection,
+                 _docTypeName.getName(),
+                 *_emptyDoc,
+                 getDocumentTypeRepo(),
+                 getSchema(),
+                 getAttrMgr(),
+                 _hasFields);
+
+    vespalib::LockGuard guard(_lock);
+    if (_selectCache.hasKey(selection))
+        return _selectCache[selection];
+    _selectCache.insert(selection, nselect);
+    return nselect;
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h
new file mode 100644
index 00000000000..4e94461e5bc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/vespalib/stllike/lrucache_map.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+
+namespace proton
+{
+
+class IDocumentMetaStoreContext;
+
+class DocumentRetrieverBase : public IDocumentRetriever
+{
+    const DocTypeName                &_docTypeName;
+    const document::DocumentTypeRepo &_repo;
+    const IDocumentMetaStoreContext  &_meta_store;
+
+    typedef vespalib::lrucache_map<vespalib::LruParam<vespalib::string, CachedSelect::SP>> SelectCache;
+
+    mutable SelectCache    _selectCache;
+    vespalib::Lock         _lock;
+    document::Document::UP _emptyDoc;
+    const bool             _hasFields;
+
+protected:
+    virtual const search::index::Schema & getSchema(void) const;
+    virtual const search::IAttributeManager * getAttrMgr(void) const;
+public:
+    DocumentRetrieverBase(const DocTypeName &docTypeName,
+                          const document::DocumentTypeRepo &repo,
+                          const IDocumentMetaStoreContext &meta_store,
+                          bool hasFields);
+
+    const document::DocumentTypeRepo &getDocumentTypeRepo() const override;
+    void getBucketMetaData(const storage::spi::Bucket &bucket,
+                           search::DocumentMetaData::Vector &result) const override;
+    search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const override;
+    CachedSelect::SP parseSelect(const vespalib::string &selection) const override;
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
new file mode 100644
index 00000000000..541f7a107ca
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -0,0 +1,322 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.documentsubdbcollection");
+
+#include "combiningfeedview.h"
+#include "commit_and_wait_document_retriever.h"
+#include "document_subdb_collection_initializer.h"
+#include "documentsubdbcollection.h"
+#include "icommitable.h"
+#include "idocumentsubdb.h"
+#include "maintenancecontroller.h"
+#include "searchabledocsubdb.h"
+
+#include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h>
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h>
+
+using proton::matching::SessionManager;
+using search::index::Schema;
+using search::SerialNum;
+using vespa::config::search::core::ProtonConfig;
+
+namespace proton {
+
+DocumentSubDBCollection::DocumentSubDBCollection(
+        IDocumentSubDB::IOwner &owner,
+        search::transactionlog::SyncProxy &tlSyncer,
+        const IGetSerialNum &getSerialNum,
+        const DocTypeName &docTypeName,
+        searchcorespi::index::IThreadingService &writeService,
+        vespalib::ThreadExecutor &warmupExecutor,
+        vespalib::ThreadStackExecutorBase &summaryExecutor,
+        const search::common::FileHeaderContext &fileHeaderContext,
+        MetricsWireService &metricsWireService,
+        LegacyDocumentDBMetrics &metrics,
+        matching::QueryLimiter &queryLimiter,
+        const vespalib::Clock &clock,
+        vespalib::Lock &configLock,
+        const vespalib::string &baseDir,
+        const ProtonConfig &protonCfg)
+    : _subDBs(),
+      _calc(),
+      _readySubDbId(0),
+      _remSubDbId(1),
+      _notReadySubDbId(2),
+      _retrievers(),
+      _reprocessingRunner(),
+      _bucketDB(),
+      _bucketDBHandler()
+{
+
+    _bucketDB = std::make_shared<BucketDBOwner>();
+    _bucketDBHandler.reset(new bucketdb::BucketDBHandler(*_bucketDB));
+    search::GrowStrategy attributeGrow(protonCfg.grow.initial,
+                                       protonCfg.grow.factor,
+                                       protonCfg.grow.add);
+    size_t attributeGrowNumDocs(protonCfg.grow.numdocs);
+    size_t numSearcherThreads = protonCfg.numsearcherthreads;
+
+    StoreOnlyDocSubDB::Context context(owner,
+                                       tlSyncer,
+                                       getSerialNum,
+                                       fileHeaderContext,
+                                       writeService,
+                                       summaryExecutor,
+                                       _bucketDB,
+                                       *_bucketDBHandler,
+                                       metrics,
+                                       configLock);
+    _subDBs.push_back
+        (new SearchableDocSubDB
+            (SearchableDocSubDB::Config(FastAccessDocSubDB::Config
+                (StoreOnlyDocSubDB::Config(docTypeName,
+                        "0.ready",
+                        baseDir,
+                        attributeGrow,
+                        attributeGrowNumDocs,
+                        _readySubDbId,
+                        SubDbType::READY),
+                        true,
+                        true,
+                        false),
+                        numSearcherThreads),
+                SearchableDocSubDB::Context(FastAccessDocSubDB::Context
+                        (context,
+                        metrics.ready.attributes,
+                        &metrics.attributes,
+                        metricsWireService),
+                        queryLimiter,
+                        clock,
+                        warmupExecutor)));
+    _subDBs.push_back
+        (new StoreOnlyDocSubDB(StoreOnlyDocSubDB::Config(docTypeName,
+                                                     "1.removed",
+                                                     baseDir,
+                                                     attributeGrow,
+                                                     attributeGrowNumDocs,
+                                                     _remSubDbId,
+                                                     SubDbType::REMOVED),
+                             context));
+    _subDBs.push_back
+        (new FastAccessDocSubDB(FastAccessDocSubDB::Config
+                (StoreOnlyDocSubDB::Config(docTypeName,
+                        "2.notready",
+                        baseDir,
+                        attributeGrow,
+                        attributeGrowNumDocs,
+                        _notReadySubDbId,
+                        SubDbType::NOTREADY),
+                        true,
+                        true,
+                        true),
+                FastAccessDocSubDB::Context(context,
+                        metrics.notReady.attributes,
+                        NULL,
+                        metricsWireService)));
+}
+
+
+DocumentSubDBCollection::~DocumentSubDBCollection()
+{
+    for (auto subDb : _subDBs) {
+        delete subDb;
+    }
+    _bucketDB.reset();
+}
+
+void
+DocumentSubDBCollection::createRetrievers()
+{
+    RetrieversSP retrievers(new std::vector<IDocumentRetriever::SP>);
+    retrievers->resize(_subDBs.size());
+    uint32_t i = 0;
+    for (auto subDb : _subDBs) {
+        (*retrievers)[i++].reset(subDb->getDocumentRetriever().release());
+    }
+    _retrievers.set(retrievers);
+}
+
+namespace {
+
+IDocumentRetriever::SP
+wrapRetriever(const IDocumentRetriever::SP &retriever,
+              ICommitable &commit)
+{
+    return std::make_shared<CommitAndWaitDocumentRetriever>(retriever, commit);
+}
+
+}
+
+
+void DocumentSubDBCollection::maintenanceSync(MaintenanceController &mc,
+                                              ICommitable &commit) {
+    RetrieversSP retrievers = getRetrievers();
+    MaintenanceDocumentSubDB readySubDB(
+            getReadySubDB()->getDocumentMetaStoreContext().getSP(),
+            wrapRetriever((*retrievers)[_readySubDbId], commit),
+            _readySubDbId);
+    MaintenanceDocumentSubDB remSubDB(
+            getRemSubDB()->getDocumentMetaStoreContext().getSP(),
+            (*retrievers)[_remSubDbId],
+            _remSubDbId);
+    MaintenanceDocumentSubDB notReadySubDB(
+            getNotReadySubDB()->getDocumentMetaStoreContext().getSP(),
+            wrapRetriever((*retrievers)[_notReadySubDbId], commit),
+            _notReadySubDbId);
+    mc.syncSubDBs(readySubDB, remSubDB, notReadySubDB);
+}
+
+initializer::InitializerTask::SP
+DocumentSubDBCollection::createInitializer(const DocumentDBConfig &configSnapshot,
+                                           SerialNum configSerialNum,
+                                           const Schema::SP &unionSchema,
+                                           const ProtonConfig::Summary & protonSummaryCfg,
+                                           const ProtonConfig::Index & indexCfg)
+{
+    DocumentSubDbCollectionInitializer::SP task =
+        std::make_shared<DocumentSubDbCollectionInitializer>();
+    for (auto subDb : _subDBs) {
+        DocumentSubDbInitializer::SP
+            subTask(subDb->createInitializer(configSnapshot,
+                                             configSerialNum,
+                                             unionSchema,
+                                             protonSummaryCfg,
+                                             indexCfg));
+        task->add(subTask);
+    }
+    return task;
+}
+
+
+void
+DocumentSubDBCollection::initViews(const DocumentDBConfig &configSnapshot,
+                                   const SessionManager::SP &sessionManager)
+{
+    for (auto subDb : _subDBs) {
+        subDb->initViews(configSnapshot, sessionManager);
+    }
+}
+
+
+void
+DocumentSubDBCollection::clearViews(void)
+{
+    for (auto subDb : _subDBs) {
+        subDb->clearViews();
+    }
+}
+
+
+void
+DocumentSubDBCollection::onReplayDone(void)
+{
+    for (auto subDb : _subDBs) {
+        subDb->onReplayDone();
+    }
+}
+
+
+void
+DocumentSubDBCollection::onReprocessDone(SerialNum serialNum)
+{
+    for (auto subDb : _subDBs) {
+        subDb->onReprocessDone(serialNum);
+    }
+}
+
+
+SerialNum
+DocumentSubDBCollection::getOldestFlushedSerial(void)
+{
+    SerialNum lowest = -1;
+    for (auto subDb : _subDBs) {
+        lowest = std::min(lowest, subDb->getOldestFlushedSerial());
+    }
+    return lowest;
+}
+
+
+SerialNum
+DocumentSubDBCollection::getNewestFlushedSerial(void)
+{
+    SerialNum highest = 0;
+    for (auto subDb : _subDBs) {
+        highest = std::max(highest, subDb->getNewestFlushedSerial());
+    }
+    return highest;
+}
+
+
+void
+DocumentSubDBCollection::wipeHistory(SerialNum wipeSerial,
+                                     const Schema &newHistorySchema,
+                                     const Schema &wipeSchema)
+{
+    for (auto subDb : _subDBs) {
+        subDb->wipeHistory(wipeSerial, newHistorySchema, wipeSchema);
+    }
+}
+
+
+void
+DocumentSubDBCollection::applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                                     const DocumentDBConfig &oldConfigSnapshot,
+                                     SerialNum serialNum,
+                                     const ReconfigParams params)
+{
+    _reprocessingRunner.reset();
+    for (auto subDb : _subDBs) {
+        IReprocessingTask::List tasks;
+        tasks = subDb->applyConfig(newConfigSnapshot, oldConfigSnapshot,
+                                   serialNum, params);
+        _reprocessingRunner.addTasks(tasks);
+    }
+}
+
+IFeedView::SP
+DocumentSubDBCollection::getFeedView()
+{
+    std::vector<IFeedView::SP> views;
+    views.reserve(_subDBs.size());
+
+    for (const auto subDb : _subDBs) {
+        views.push_back(subDb->getFeedView());
+    }
+    IFeedView::SP newFeedView;
+    assert(views.size() >= 1);
+    if (views.size() > 1) {
+        return IFeedView::SP(new CombiningFeedView(views, _calc));
+    } else {
+        assert(views.front() != NULL);
+        return views.front();
+    }
+}
+
+IFlushTarget::List
+DocumentSubDBCollection::getFlushTargets()
+{
+    IFlushTarget::List ret;
+    for (auto subDb : _subDBs) {
+        IFlushTarget::List iTargets(subDb->getFlushTargets());
+        ret.insert(ret.end(), iTargets.begin(), iTargets.end());
+    }
+    return ret;
+}
+
+double
+DocumentSubDBCollection::getReprocessingProgress() const
+{
+    return _reprocessingRunner.getProgress();
+}
+
+void
+DocumentSubDBCollection::close()
+{
+    for (auto subDb : _subDBs) {
+        subDb->close();
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
new file mode 100644
index 00000000000..c0aea4804c8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "ibucketstatecalculator.h"
+#include "idocumentsubdb.h"
+#include "ifeedview.h"
+#include "searchable_doc_subdb_configurer.h"
+#include <vespa/searchcore/config/config-proton.h>
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/searchlib/transactionlog/syncproxy.h>
+#include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h>
+#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
+#include <vespa/searchcore/proton/initializer/initializer_task.h>
+
+namespace proton {
+class DocumentDBConfig;
+class LegacyDocumentDBMetrics;
+class MaintenanceController;
+class MetricsWireService;
+class ICommitable;
+class IGetSerialNum;
+
+class DocumentSubDBCollection {
+public:
+    typedef std::vector<IDocumentSubDB *> SubDBVector;
+    typedef SubDBVector::const_iterator const_iterator;
+    typedef search::SerialNum      SerialNum;
+
+private:
+    SubDBVector _subDBs;
+    IBucketStateCalculator::SP _calc;
+    const uint32_t _readySubDbId;
+    const uint32_t _remSubDbId;
+    const uint32_t _notReadySubDbId;
+    typedef std::shared_ptr<std::vector<IDocumentRetriever::SP> > RetrieversSP;
+    vespalib::VarHolder<RetrieversSP> _retrievers;
+    typedef std::vector<std::shared_ptr<IReprocessingTask>> ReprocessingTasks;
+    ReprocessingRunner _reprocessingRunner;
+    std::shared_ptr<BucketDBOwner> _bucketDB;
+    std::unique_ptr<bucketdb::BucketDBHandler> _bucketDBHandler;
+
+public:
+    DocumentSubDBCollection(
+            IDocumentSubDB::IOwner &owner,
+            search::transactionlog::SyncProxy &tlSyncer,
+            const IGetSerialNum &getSerialNum,
+            const DocTypeName &docTypeName,
+            searchcorespi::index::IThreadingService &writeService,
+            vespalib::ThreadExecutor &warmupExecutor,
+            vespalib::ThreadStackExecutorBase &summaryExecutor,
+            const search::common::FileHeaderContext &fileHeaderContext,
+            MetricsWireService &metricsWireService,
+            LegacyDocumentDBMetrics &metrics,
+            matching::QueryLimiter & queryLimiter,
+            const vespalib::Clock &clock,
+            vespalib::Lock &configLock,
+            const vespalib::string &baseDir,
+            const vespa::config::search::core::ProtonConfig &protonCfg);
+    ~DocumentSubDBCollection();
+
+    void setBucketStateCalculator(const IBucketStateCalculator::SP &calc) {
+        _calc = calc;
+    }
+
+    void createRetrievers();
+    void maintenanceSync(MaintenanceController &mc, ICommitable &commit);
+
+    // Internally synchronized
+    std::shared_ptr<std::vector<IDocumentRetriever::SP> > getRetrievers() {
+        return _retrievers.get();
+    }
+
+    IDocumentSubDB *getReadySubDB() { return _subDBs[_readySubDbId]; }
+    const IDocumentSubDB *getReadySubDB() const { return _subDBs[_readySubDbId]; }
+    IDocumentSubDB *getRemSubDB() { return _subDBs[_remSubDbId]; }
+    const IDocumentSubDB *getRemSubDB() const { return _subDBs[_remSubDbId]; }
+    IDocumentSubDB *getNotReadySubDB() { return _subDBs[_notReadySubDbId]; }
+    const IDocumentSubDB *getNotReadySubDB() const { return _subDBs[_notReadySubDbId]; }
+
+    const_iterator begin() const { return _subDBs.begin(); }
+    const_iterator end() const { return _subDBs.end(); }
+
+    BucketDBOwner &getBucketDB() { return *_bucketDB; }
+
+    bucketdb::IBucketDBHandler &getBucketDBHandler() {
+        return *_bucketDBHandler;
+    }
+
+    initializer::InitializerTask::SP
+    createInitializer(const DocumentDBConfig &configSnapshot,
+                      SerialNum configSerialNum,
+                      const search::index::Schema::SP &unionSchema,
+                      const vespa::config::search::core::ProtonConfig::Summary &protonSummaryCfg,
+                      const vespa::config::search::core::ProtonConfig::Index & indexCfg);
+
+    void
+    initViews(const DocumentDBConfig &configSnapshot,
+              const matching::SessionManager::SP &sessionManager);
+
+    void clearViews(void);
+    void onReplayDone(void);
+    void onReprocessDone(SerialNum serialNum);
+    SerialNum getOldestFlushedSerial(void);
+    SerialNum getNewestFlushedSerial(void);
+
+    void
+    wipeHistory(SerialNum wipeSerial,
+                const search::index::Schema &newHistorySchema,
+                const search::index::Schema &wipeSchema);
+
+    void
+    applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                const DocumentDBConfig &oldConfigSnapshot,
+                SerialNum serialNum,
+                const ReconfigParams params);
+
+    IFeedView::SP getFeedView();
+    IFlushTarget::List getFlushTargets();
+    ReprocessingRunner &getReprocessingRunner() { return _reprocessingRunner; }
+    double getReprocessingProgress() const;
+    void close();
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp
new file mode 100644
index 00000000000..4fd9726f686
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.emptysearchview");
+
+#include "emptysearchview.h"
+
+using search::engine::DocsumReply;
+using search::engine::DocsumRequest;
+using search::engine::SearchReply;
+using search::engine::SearchRequest;
+
+namespace proton
+{
+
+EmptySearchView::EmptySearchView(void)
+    : boost::noncopyable(),
+      ISearchHandler()
+{
+}
+
+
+DocsumReply::UP
+EmptySearchView::getDocsums(const DocsumRequest &req)
+{
+    LOG(debug, "getDocsums(): resultClass(%s), numHits(%zu)",
+        req.resultClassName.c_str(), req.hits.size());
+    DocsumReply::UP reply(new DocsumReply());
+    for (size_t i = 0; i < req.hits.size(); ++i) {
+        reply->docsums.push_back(DocsumReply::Docsum());
+        reply->docsums.back().gid = req.hits[i].gid;
+    }
+    return reply;
+}
+
+SearchReply::UP
+EmptySearchView::match(const ISearchHandler::SP &,
+                       const SearchRequest &,
+                       vespalib::ThreadBundle &) const {
+    SearchReply::UP reply(new SearchReply);
+    reply->useCoverage = true;
+    return reply;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h
new file mode 100644
index 00000000000..3b22ecf43a5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
+
+namespace proton {
+
+class EmptySearchView : public boost::noncopyable,
+                         public ISearchHandler
+{
+public:
+    typedef std::shared_ptr<EmptySearchView> SP;
+
+    EmptySearchView(void);
+
+    /**
+     * Implements ISearchHandler
+     */
+    virtual search::engine::DocsumReply::UP
+    getDocsums(const search::engine::DocsumRequest & req);
+
+    virtual search::engine::SearchReply::UP
+    match(const ISearchHandler::SP &searchHandler,
+          const search::engine::SearchRequest &req,
+          vespalib::ThreadBundle &threadBundle) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp
new file mode 100644
index 00000000000..2c7627ea712
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.executor_thread_service");
+
+#include "executor_thread_service.h"
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/util/executor.h>
+#include <vespa/vespalib/util/sync.h>
+
+using vespalib::makeClosure;
+using vespalib::makeTask;
+using vespalib::Executor;
+using vespalib::Gate;
+using vespalib::Runnable;
+using vespalib::ThreadStackExecutorBase;
+
+namespace proton {
+
+namespace {
+
+void
+sampleThreadId(FastOS_ThreadId *threadId)
+{
+    *threadId = FastOS_Thread::GetCurrentThreadId();
+}
+
+FastOS_ThreadId
+getThreadId(ThreadStackExecutorBase &executor)
+{
+    FastOS_ThreadId id;
+    executor.execute(makeTask(makeClosure(&sampleThreadId, &id)));
+    executor.sync();
+    return id;
+}
+
+void
+runRunnable(Runnable *runnable, Gate *gate)
+{
+    runnable->run();
+    gate->countDown();
+}
+
+} // namespace
+
+ExecutorThreadService::ExecutorThreadService(ThreadStackExecutorBase &executor)
+    : _executor(executor),
+      _threadId(getThreadId(executor))
+{
+}
+
+void
+ExecutorThreadService::run(Runnable &runnable)
+{
+    if (isCurrentThread()) {
+        runnable.run();
+    } else {
+        Gate gate;
+        _executor.execute(makeTask(makeClosure(&runRunnable, &runnable, &gate)));
+        gate.await();
+    }
+}
+
+bool
+ExecutorThreadService::isCurrentThread() const
+{
+    FastOS_ThreadId currentThreadId = FastOS_Thread::GetCurrentThreadId();
+    return FastOS_Thread::CompareThreadIds(_threadId, currentThreadId);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h
new file mode 100644
index 00000000000..e2332d6ab94
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/index/i_thread_service.h>
+#include <vespa/vespalib/util/threadstackexecutorbase.h>
+
+namespace proton {
+
+/**
+ * Implementation of IThreadService using an underlying thread stack executor
+ * with a single thread.
+ */
+class ExecutorThreadService : public searchcorespi::index::IThreadService
+{
+private:
+    vespalib::ThreadStackExecutorBase &_executor;
+    FastOS_ThreadId                    _threadId;
+
+public:
+    ExecutorThreadService(vespalib::ThreadStackExecutorBase &executor);
+
+    /**
+     * Implements IThreadService
+     */
+    virtual vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) {
+        return _executor.execute(std::move(task));
+    }
+    virtual void run(vespalib::Runnable &runnable);
+    virtual vespalib::Syncable &sync() {
+        _executor.sync();
+        return *this;
+    }
+    virtual bool isCurrentThread() const;
+};
+
+} // namespace proton
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp
new file mode 100644
index 00000000000..86667232eef
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.executorthreadingservice");
+
+#include "executorthreadingservice.h"
+#include <vespa/vespalib/util/executor.h>
+#include <vespa/vespalib/util/sync.h>
+
+using vespalib::ThreadStackExecutorBase;
+
+namespace proton {
+
+ExecutorThreadingService::ExecutorThreadingService(uint32_t threads,
+                                                   uint32_t stackSize,
+                                                   uint32_t taskLimit)
+
+    : _masterExecutor(1, stackSize),
+      _indexExecutor(1, stackSize, taskLimit),
+      _masterService(_masterExecutor),
+      _indexService(_indexExecutor),
+      _indexFieldInverter(threads),
+      _indexFieldWriter(threads),
+      _attributeFieldWriter(threads)
+{
+}
+
+vespalib::Syncable &
+ExecutorThreadingService::sync()
+{
+    bool isMasterThread = _masterService.isCurrentThread();
+    if (!isMasterThread) {
+        _masterExecutor.sync();
+    }
+    _attributeFieldWriter.sync();
+    _indexExecutor.sync();
+    _indexFieldInverter.sync();
+    _indexFieldWriter.sync();
+    if (!isMasterThread) {
+        _masterExecutor.sync();
+    }
+    return *this;
+}
+
+void
+ExecutorThreadingService::shutdown()
+{
+    _masterExecutor.shutdown();
+    _masterExecutor.sync();
+    _attributeFieldWriter.sync();
+    _indexExecutor.shutdown();
+    _indexExecutor.sync();
+    _indexFieldInverter.sync();
+    _indexFieldWriter.sync();
+}
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h
new file mode 100644
index 00000000000..9c836d10f96
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "executor_thread_service.h"
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+
+namespace proton {
+
+/**
+ * Implementation of IThreadingService using 2 underlying thread stack executors
+ * with 1 thread each.
+ */
+class ExecutorThreadingService : public searchcorespi::index::IThreadingService
+{
+private:
+    vespalib::ThreadStackExecutor _masterExecutor;
+    vespalib::BlockingThreadStackExecutor _indexExecutor;
+    ExecutorThreadService _masterService;
+    ExecutorThreadService _indexService;
+    search::SequencedTaskExecutor _indexFieldInverter;
+    search::SequencedTaskExecutor _indexFieldWriter;
+    search::SequencedTaskExecutor _attributeFieldWriter;
+
+public:
+    /**
+     * Constructor.
+     *
+     * @stackSize The size of the stack of the underlying executors.
+     * @taskLimit The task limit for the index executor.
+     */
+    ExecutorThreadingService(uint32_t threads = 1,
+                             uint32_t stackSize = 128 * 1024,
+                             uint32_t taskLimit = 1000);
+
+    /**
+     * Implements vespalib::Syncable
+     */
+    virtual vespalib::Syncable &sync();
+
+    void shutdown();
+
+    // Expose the underlying executors for stats fetching and testing.
+    vespalib::ThreadStackExecutorBase &getMasterExecutor() {
+        return _masterExecutor;
+    }
+    vespalib::ThreadStackExecutorBase &getIndexExecutor() {
+        return _indexExecutor;
+    }
+
+    /**
+     * Implements IThreadingService
+     */
+    virtual searchcorespi::index::IThreadService &master() {
+        return _masterService;
+    }
+    virtual searchcorespi::index::IThreadService &index() {
+        return _indexService;
+    }
+
+    virtual search::ISequencedTaskExecutor &indexFieldInverter() override {
+        return _indexFieldInverter;
+    }
+
+    virtual search::ISequencedTaskExecutor &indexFieldWriter() override {
+        return _indexFieldWriter;
+    }
+
+    virtual search::ISequencedTaskExecutor &attributeFieldWriter() override {
+        return _attributeFieldWriter;
+    }
+};
+
+} // namespace proton
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
new file mode 100644
index 00000000000..cb8abd1cfd9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
@@ -0,0 +1,347 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.fast_access_doc_subdb");
+
+#include "attributeadapterfactory.h"
+#include "emptysearchview.h"
+#include "fast_access_doc_subdb.h"
+#include "fast_access_document_retriever.h"
+#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
+#include <vespa/searchcore/proton/attribute/attribute_factory.h>
+#include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h>
+#include <vespa/searchcore/proton/attribute/attribute_populator.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
+#include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h>
+#include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h>
+#include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h>
+#include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h>
+#include <vespa/searchcore/proton/reprocessing/reprocess_documents_task.h>
+#include <vespa/searchlib/docstore/document_store_visitor_progress.h>
+
+using proton::matching::SessionManager;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::SerialNum;
+using search::index::Schema;
+using proton::initializer::InitializerTask;
+
+namespace proton {
+
+namespace {
+
+constexpr search::SerialNum ATTRIBUTE_INIT_SERIAL = 1;
+
+struct AttributeGuardComp
+{
+    vespalib::string name;
+
+    AttributeGuardComp(const vespalib::string &n)
+        : name(n)
+    {
+    }
+
+    bool
+    operator()(const AttributeGuard &rhs) const
+    {
+        return name == rhs->getName();
+    };
+};
+
+proton::IAttributeManager::SP
+extractAttributeManager(const FastAccessFeedView::SP &feedView)
+{
+    const IAttributeWriter::SP &writer = feedView->getAttributeWriter();
+    return writer->getAttributeManager();
+}
+
+}
+
+InitializerTask::SP
+FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &configSnapshot,
+                                                      SerialNum configSerialNum,
+                                                      InitializerTask::SP documentMetaStoreInitTask,
+                                                      DocumentMetaStore::SP documentMetaStore,
+                                                      std::shared_ptr<AttributeManager::SP> attrMgrResult) const
+{
+    IAttributeFactory::SP attrFactory = std::make_shared<AttributeFactory>();
+    AttributeManager::SP baseAttrMgr =
+            std::make_shared<AttributeManager>(_baseDir + "/attribute",
+                                               getSubDbName(),
+                                               configSnapshot.getTuneFileDocumentDBSP()->_attr,
+                                               _fileHeaderContext,
+                                               _writeService.attributeFieldWriter(),
+                                               attrFactory);
+    return std::make_shared<AttributeManagerInitializer>(configSerialNum,
+                                                         documentMetaStoreInitTask,
+                                                         documentMetaStore,
+                                                         baseAttrMgr,
+                                                         (_hasAttributes ? configSnapshot.getAttributesConfig() : AttributesConfig()),
+                                                         _attributeGrow,
+                                                         _attributeGrowNumDocs,
+                                                         _fastAccessAttributesOnly,
+                                                         attrMgrResult);
+}
+
+void
+FastAccessDocSubDB::setupAttributeManager(AttributeManager::SP attrMgrResult)
+{
+    if (_addMetrics) {
+        // register attribute metrics
+        std::vector<AttributeGuard> list;
+        attrMgrResult->getAttributeListAll(list);
+        for (const auto &attr : list) {
+            const AttributeVector &v = *attr;
+            _metricsWireService.addAttribute(_subAttributeMetrics, _totalAttributeMetrics,
+                    v.getName());
+        }
+    }
+    _initAttrMgr = attrMgrResult;
+}
+
+
+AttributeCollectionSpec::UP
+FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg,
+                                        SerialNum serialNum) const
+{
+    uint32_t docIdLimit(_dms->getCommittedDocIdLimit());
+    AttributeCollectionSpecFactory factory(_attributeGrow,
+            _attributeGrowNumDocs, _fastAccessAttributesOnly);
+    return factory.create(attrCfg, docIdLimit, serialNum);
+}
+
+void
+FastAccessDocSubDB::initFeedView(const IAttributeWriter::SP &writer,
+                                 const DocumentDBConfig &configSnapshot)
+{
+    // Called by executor thread
+    FastAccessFeedView::UP feedView(new FastAccessFeedView(
+            getStoreOnlyFeedViewContext(configSnapshot),
+            getFeedViewPersistentParams(),
+            FastAccessFeedView::Context(writer, _docIdLimit)));
+
+    _fastUpdateFeedView.set(FastAccessFeedView::SP(feedView.release()));
+    _iFeedView.set(_fastUpdateFeedView.get());
+}
+
+AttributeManager::SP
+FastAccessDocSubDB::getAndResetInitAttributeManager()
+{
+    AttributeManager::SP retval = _initAttrMgr;
+    _initAttrMgr.reset();
+    return retval;
+}
+
+IFlushTarget::List
+FastAccessDocSubDB::getFlushTargetsInternal()
+{
+    IFlushTarget::List retval(Parent::getFlushTargetsInternal());
+    IFlushTarget::List tmp(getAttributeManager()->getFlushTargets());
+    retval.insert(retval.end(), tmp.begin(), tmp.end());
+    return retval;
+}
+
+void
+FastAccessDocSubDB::reconfigureAttributeMetrics(const proton::IAttributeManager &newMgr,
+                                                const proton::IAttributeManager &oldMgr)
+{
+    std::set<vespalib::string> toAdd;
+    std::set<vespalib::string> toRemove;
+    std::vector<AttributeGuard> newList;
+    std::vector<AttributeGuard> oldList;
+    newMgr.getAttributeList(newList);
+    oldMgr.getAttributeList(oldList);
+    for (const auto &newAttr : newList) {
+        if (std::find_if(oldList.begin(),
+                         oldList.end(),
+                         AttributeGuardComp(newAttr->getName())) ==
+            oldList.end()) {
+            toAdd.insert(newAttr->getName());
+        }
+    }
+    for (const auto &oldAttr : oldList) {
+        if (std::find_if(newList.begin(),
+                         newList.end(),
+                         AttributeGuardComp(oldAttr->getName())) ==
+            newList.end()) {
+            toRemove.insert(oldAttr->getName());
+        }
+    }
+    for (const auto &attrName : toAdd) {
+        LOG(debug, "reconfigureAttributeMetrics(): addAttribute='%s'", attrName.c_str());
+        _metricsWireService.addAttribute(_subAttributeMetrics, _totalAttributeMetrics, attrName);
+    }
+    for (const auto &attrName : toRemove) {
+        LOG(debug, "reconfigureAttributeMetrics(): removeAttribute='%s'", attrName.c_str());
+        _metricsWireService.removeAttribute(_subAttributeMetrics, _totalAttributeMetrics, attrName);
+    }
+}
+
+IReprocessingTask::UP
+FastAccessDocSubDB::createReprocessingTask(IReprocessingInitializer &initializer,
+                                           const document::DocumentTypeRepo::SP &docTypeRepo) const
+{
+    uint32_t docIdLimit = _metaStoreCtx->get().getCommittedDocIdLimit();
+    assert(docIdLimit > 0);
+    return IReprocessingTask::UP(new ReprocessDocumentsTask(initializer,
+            getSummaryManager(),
+            docTypeRepo,
+            getSubDbName(),
+            docIdLimit));
+}
+
+FastAccessDocSubDB::FastAccessDocSubDB(const Config &cfg,
+                                       const Context &ctx)
+    : Parent(cfg._storeOnlyCfg, ctx._storeOnlyCtx),
+      _hasAttributes(cfg._hasAttributes),
+      _fastAccessAttributesOnly(cfg._fastAccessAttributesOnly),
+      _initAttrMgr(),
+      _fastUpdateFeedView(),
+      _subAttributeMetrics(ctx._subAttributeMetrics),
+      _totalAttributeMetrics(ctx._totalAttributeMetrics),
+      _addMetrics(cfg._addMetrics),
+      _metricsWireService(ctx._metricsWireService),
+      _docIdLimit(0)
+{
+}
+
+DocumentSubDbInitializer::UP
+FastAccessDocSubDB::createInitializer(const DocumentDBConfig &configSnapshot,
+                                      SerialNum configSerialNum,
+                                      const search::index::Schema::SP &unionSchema,
+                                      const vespa::config::search::core::ProtonConfig::Summary &protonSummaryCfg,
+                                      const vespa::config::search::core::ProtonConfig::Index &indexCfg) const
+{
+    auto result = Parent::createInitializer(configSnapshot, configSerialNum, unionSchema, protonSummaryCfg, indexCfg);
+    auto attrMgrInitTask = createAttributeManagerInitializer(configSnapshot,
+                                                             configSerialNum,
+                                                             result->getDocumentMetaStoreInitTask(),
+                                                             result->result().documentMetaStore()->documentMetaStore(),
+                                                             result->writableResult().writableAttributeManager());
+    result->addDependency(attrMgrInitTask);
+    return result;
+}
+
+void
+FastAccessDocSubDB::setup(const DocumentSubDbInitializerResult &initResult)
+{
+    Parent::setup(initResult);
+    setupAttributeManager(initResult.attributeManager());
+    _docIdLimit.set(_dms->getCommittedDocIdLimit());
+}
+
+void
+FastAccessDocSubDB::initViews(const DocumentDBConfig &configSnapshot,
+                              const SessionManager::SP &sessionManager)
+{
+    // Called by executor thread
+    (void) sessionManager;
+    _iSearchView.set(ISearchHandler::SP(new EmptySearchView));
+    IAttributeWriter::SP writer(new AttributeWriter(getAndResetInitAttributeManager()));
+    {
+        vespalib::LockGuard guard(_configLock);
+        initFeedView(writer, configSnapshot);
+    }
+}
+
+IReprocessingTask::List
+FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                                const DocumentDBConfig &oldConfigSnapshot,
+                                SerialNum serialNum,
+                                const ReconfigParams params)
+{
+    IReprocessingTask::List tasks;
+    updateLidReuseDelayer(&newConfigSnapshot);
+    /*
+     * If attribute manager should change then document retriever
+     * might have to rewrite a different set of fields.  If document
+     * type repo has changed then the new repo is needed to handle
+     * documents using new fields, e.g. when moving documents from notready
+     * to ready document sub db.
+     */
+    if (params.shouldAttributeManagerChange() ||
+        newConfigSnapshot.getDocumentTypeRepoSP().get() != oldConfigSnapshot.getDocumentTypeRepoSP().get()) {
+        FastAccessDocSubDBConfigurer configurer(_fastUpdateFeedView,
+                IAttributeAdapterFactory::UP(new AttributeAdapterFactory), getSubDbName());
+        proton::IAttributeManager::SP oldMgr = extractAttributeManager(_fastUpdateFeedView.get());
+        AttributeCollectionSpec::UP attrSpec =
+            createAttributeSpec(newConfigSnapshot.getAttributesConfig(), serialNum);
+        IReprocessingInitializer::UP initializer =
+                configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec);
+        if (initializer->hasReprocessors()) {
+            tasks.push_back(IReprocessingTask::SP(createReprocessingTask(*initializer,
+                    newConfigSnapshot.getDocumentTypeRepoSP()).release()));
+        }
+        if (_addMetrics) {
+            proton::IAttributeManager::SP newMgr = extractAttributeManager(_fastUpdateFeedView.get());
+            reconfigureAttributeMetrics(*newMgr, *oldMgr);
+        }
+        _iFeedView.set(_fastUpdateFeedView.get());
+        _owner.syncFeedView();
+    }
+    return tasks;
+}
+
+proton::IAttributeManager::SP
+FastAccessDocSubDB::getAttributeManager() const
+{
+    return extractAttributeManager(_fastUpdateFeedView.get());
+}
+
+IDocumentRetriever::UP
+FastAccessDocSubDB::getDocumentRetriever()
+{
+    FastAccessFeedView::SP feedView = _fastUpdateFeedView.get();
+    proton::IAttributeManager::SP attrMgr = extractAttributeManager(feedView);
+    return IDocumentRetriever::UP(new FastAccessDocumentRetriever(feedView, attrMgr));
+}
+
+void
+FastAccessDocSubDB::onReplayDone()
+{
+    // Called by document db executor thread
+    Parent::onReplayDone();
+    // Normalize attribute vector sizes
+    uint32_t docIdLimit = _metaStoreCtx->get().getCommittedDocIdLimit();
+    assert(docIdLimit > 0);
+    IFeedView::SP feedView = _iFeedView.get();
+    IAttributeWriter::SP attrWriter =
+        static_cast<FastAccessFeedView &>(*feedView).getAttributeWriter();
+    attrWriter->onReplayDone(docIdLimit);
+}
+
+
+void
+FastAccessDocSubDB::onReprocessDone(SerialNum serialNum)
+{
+    IFeedView::SP feedView = _iFeedView.get();
+    IAttributeWriter::SP attrWriter =
+        static_cast<FastAccessFeedView &>(*feedView).getAttributeWriter();
+    attrWriter->commit(serialNum,
+                       std::shared_ptr<search::IDestructorCallback>());
+    _writeService.attributeFieldWriter().sync();
+    Parent::onReprocessDone(serialNum);
+}
+
+
+SerialNum
+FastAccessDocSubDB::getOldestFlushedSerial()
+{
+    SerialNum lowest(Parent::getOldestFlushedSerial());
+    proton::IAttributeManager::SP attrMgr(getAttributeManager());
+    lowest = std::min(lowest, attrMgr->getOldestFlushedSerialNumber());
+    return lowest;
+}
+
+SerialNum
+FastAccessDocSubDB::getNewestFlushedSerial()
+{
+    SerialNum highest(Parent::getNewestFlushedSerial());
+    proton::IAttributeManager::SP attrMgr(getAttributeManager());
+    highest = std::max(highest, attrMgr->getNewestFlushedSerialNumber());
+    return highest;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
new file mode 100644
index 00000000000..bb0b2a9ed5b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
@@ -0,0 +1,150 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "fast_access_doc_subdb_configurer.h"
+#include <vespa/searchcore/proton/metrics/metricswireservice.h>
+#include "storeonlydocsubdb.h"
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/common/docid_limit.h>
+
+namespace proton {
+
+/**
+ * The fast-access sub database keeps fast-access attribute fields in memory
+ * in addition to the underlying document store managed by the parent class.
+ *
+ * Partial updates and document selection on one of these attribute fields will be
+ * fast compared to only using the document store.
+ * This class is used as base class for the searchable sub database and directly by
+ * the "2.notready" sub database for handling not-ready documents.
+ * When used by the "2.notready" sub database attributes that are added without any files
+ * on disk will be populated based on the content of the document store upon initialization
+ * of the sub database.
+ */
+class FastAccessDocSubDB : public StoreOnlyDocSubDB
+{
+public:
+    struct Config
+    {
+        const StoreOnlyDocSubDB::Config _storeOnlyCfg;
+        const bool                      _hasAttributes;
+        const bool                      _addMetrics;
+        const bool                      _fastAccessAttributesOnly;
+        Config(const StoreOnlyDocSubDB::Config &storeOnlyCfg,
+               bool hasAttributes,
+               bool addMetrics,
+               bool fastAccessAttributesOnly)
+        : _storeOnlyCfg(storeOnlyCfg),
+          _hasAttributes(hasAttributes),
+          _addMetrics(addMetrics),
+          _fastAccessAttributesOnly(fastAccessAttributesOnly)
+        {
+        }
+    };
+
+    struct Context
+    {
+        const StoreOnlyDocSubDB::Context _storeOnlyCtx;
+        AttributeMetrics                &_subAttributeMetrics;
+        AttributeMetrics                *_totalAttributeMetrics;
+        MetricsWireService              &_metricsWireService;
+        Context(const StoreOnlyDocSubDB::Context &storeOnlyCtx,
+                AttributeMetrics &subAttributeMetrics,
+                AttributeMetrics *totalAttributeMetrics,
+                MetricsWireService &metricsWireService)
+        : _storeOnlyCtx(storeOnlyCtx),
+          _subAttributeMetrics(subAttributeMetrics),
+          _totalAttributeMetrics(totalAttributeMetrics),
+          _metricsWireService(metricsWireService)
+        {
+        }
+    };
+
+private:
+    typedef vespa::config::search::AttributesConfig AttributesConfig;
+    typedef FastAccessDocSubDBConfigurer Configurer;
+
+    const bool                    _hasAttributes;
+    const bool                    _fastAccessAttributesOnly;
+    AttributeManager::SP          _initAttrMgr;
+    Configurer::FeedViewVarHolder _fastUpdateFeedView;
+    AttributeMetrics             &_subAttributeMetrics;
+    AttributeMetrics             *_totalAttributeMetrics;
+
+    initializer::InitializerTask::SP
+    createAttributeManagerInitializer(const DocumentDBConfig &configSnapshot,
+                                      SerialNum configSerialNum,
+                                      initializer::InitializerTask::SP documentMetaStoreInitTask,
+                                      DocumentMetaStore::SP documentMetaStore,
+                                      std::shared_ptr<AttributeManager::SP> attrMgrResult) const;
+
+    void setupAttributeManager(AttributeManager::SP attrMgrResult);
+
+    void initFeedView(const IAttributeWriter::SP &writer,
+                      const DocumentDBConfig &configSnapshot);
+
+
+protected:
+    typedef StoreOnlyDocSubDB Parent;
+    typedef vespa::config::search::core::ProtonConfig ProtonConfig;
+
+    const bool           _addMetrics;
+    MetricsWireService  &_metricsWireService;
+    DocIdLimit           _docIdLimit;
+
+    AttributeCollectionSpec::UP createAttributeSpec(const AttributesConfig &attrCfg,
+                                                    SerialNum serialNum) const;
+
+    AttributeManager::SP getAndResetInitAttributeManager();
+
+    virtual IFlushTarget::List getFlushTargetsInternal();
+
+    void reconfigureAttributeMetrics(const proton::IAttributeManager &newMgr,
+                                     const proton::IAttributeManager &oldMgr);
+
+    IReprocessingTask::UP
+    createReprocessingTask(IReprocessingInitializer &initializer,
+                           const document::DocumentTypeRepo::SP &docTypeRepo) const;
+
+public:
+    FastAccessDocSubDB(const Config &cfg,
+                       const Context &ctx);
+
+    virtual ~FastAccessDocSubDB() {}
+
+    virtual DocumentSubDbInitializer::UP
+    createInitializer(const DocumentDBConfig &configSnapshot,
+                      SerialNum configSerialNum,
+                      const search::index::Schema::SP &unionSchema,
+                      const vespa::config::search::core::ProtonConfig::Summary &protonSummaryCfg,
+                      const vespa::config::search::core::ProtonConfig::Index &indexCfg) const override;
+
+    virtual void setup(const DocumentSubDbInitializerResult &initResult) override;
+
+    virtual void initViews(const DocumentDBConfig &configSnapshot,
+                           const proton::matching::SessionManager::SP &sessionManager);
+
+    virtual IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                                                const DocumentDBConfig &oldConfigSnapshot,
+                                                SerialNum serialNum,
+                                                const ReconfigParams params);
+
+    virtual proton::IAttributeManager::SP getAttributeManager() const;
+
+    virtual IDocumentRetriever::UP getDocumentRetriever();
+
+    virtual void
+    onReplayDone();
+
+    virtual void
+    onReprocessDone(SerialNum serialNum);
+
+    virtual SerialNum
+    getOldestFlushedSerial();
+
+    virtual SerialNum
+    getNewestFlushedSerial();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp
new file mode 100644
index 00000000000..f32b3144a62
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.fast_access_doc_subdb_configurer");
+
+#include "fast_access_doc_subdb_configurer.h"
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/common/document_type_inspector.h>
+#include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h>
+
+using document::DocumentTypeRepo;
+using search::index::Schema;
+
+namespace proton {
+
+typedef AttributeReprocessingInitializer::Config ARIConfig;
+
+void
+FastAccessDocSubDBConfigurer::reconfigureFeedView(const FastAccessFeedView::SP &curr,
+                                                  const Schema::SP &schema,
+                                                  const DocumentTypeRepo::SP &repo,
+                                                  const IAttributeWriter::SP &writer)
+{
+    _feedView.set(FastAccessFeedView::SP(new FastAccessFeedView(
+            StoreOnlyFeedView::Context(curr->getSummaryAdapter(),
+                    schema,
+                    curr->getDocumentMetaStore(),
+                    repo,
+                    curr->getWriteService(),
+                    curr->getLidReuseDelayer(),
+                    curr->getCommitTimeTracker()),
+            curr->getPersistentParams(),
+            FastAccessFeedView::Context(writer,
+                    curr->getDocIdLimit()))));
+}
+
+FastAccessDocSubDBConfigurer::FastAccessDocSubDBConfigurer(FeedViewVarHolder &feedView,
+                                                           IAttributeAdapterFactory::UP factory,
+                                                           const vespalib::string &subDbName)
+    : _feedView(feedView),
+      _factory(std::move(factory)),
+      _subDbName(subDbName)
+{
+}
+
+IReprocessingInitializer::UP
+FastAccessDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig,
+                                          const DocumentDBConfig &oldConfig,
+                                          const AttributeCollectionSpec &attrSpec)
+{
+    FastAccessFeedView::SP oldView = _feedView.get();
+    IAttributeWriter::SP writer =
+            _factory->create(oldView->getAttributeWriter(), attrSpec);
+    reconfigureFeedView(oldView, newConfig.getSchemaSP(), newConfig.getDocumentTypeRepoSP(), writer);
+
+    const document::DocumentType *newDocType = newConfig.getDocumentType();
+    const document::DocumentType *oldDocType = oldConfig.getDocumentType();
+    assert(newDocType != nullptr);
+    assert(oldDocType != nullptr);
+    return IReprocessingInitializer::UP(new AttributeReprocessingInitializer(
+            ARIConfig(writer->getAttributeManager(), *newConfig.getSchemaSP(),
+                    IDocumentTypeInspector::SP(new DocumentTypeInspector(*newDocType))),
+            ARIConfig(oldView->getAttributeWriter()->getAttributeManager(), *oldConfig.getSchemaSP(),
+                    IDocumentTypeInspector::SP(new DocumentTypeInspector(*oldDocType))),
+                    _subDbName));
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h
new file mode 100644
index 00000000000..5fd6ae8dd7c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "searchable_doc_subdb_configurer.h"
+#include "fast_access_feed_view.h"
+#include "iattributeadapterfactory.h"
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_initializer.h>
+
+namespace proton {
+
+/**
+ * Class used to reconfig the feed view used in a fast-access sub database
+ * when the set of fast-access attributes change.
+ */
+class FastAccessDocSubDBConfigurer
+{
+public:
+    typedef vespalib::VarHolder<FastAccessFeedView::SP> FeedViewVarHolder;
+
+private:
+    FeedViewVarHolder           &_feedView;
+    IAttributeAdapterFactory::UP _factory;
+    vespalib::string             _subDbName;
+
+    void reconfigureFeedView(const FastAccessFeedView::SP &curr,
+                             const search::index::Schema::SP &schema,
+                             const document::DocumentTypeRepo::SP &repo,
+                             const IAttributeWriter::SP &attrWriter);
+
+public:
+    FastAccessDocSubDBConfigurer(FeedViewVarHolder &feedView,
+                                 IAttributeAdapterFactory::UP factory,
+                                 const vespalib::string &subDbName);
+
+    IReprocessingInitializer::UP reconfigure(const DocumentDBConfig &newConfig,
+                                             const DocumentDBConfig &oldConfig,
+                                             const AttributeCollectionSpec &attrSpec);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h
new file mode 100644
index 00000000000..907939a5a50
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentretriever.h"
+#include "fast_access_feed_view.h"
+#include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include <vespa/searchcore/proton/docsummary/isummarymanager.h>
+
+namespace proton {
+
+/**
+ * The document retriever used by the fast-access sub database.
+ *
+ * Handles retrieving of documents by combining from the underlying attribute manager
+ * and document store.
+ */
+class FastAccessDocumentRetriever : public DocumentRetriever
+{
+private:
+    FastAccessFeedView::SP       _feedView;
+    IAttributeManager::SP        _attrMgr;
+
+public:
+    FastAccessDocumentRetriever(const FastAccessFeedView::SP &feedView,
+                                const IAttributeManager::SP &attrMgr)
+        : DocumentRetriever(feedView->getPersistentParams()._docTypeName,
+                            *feedView->getDocumentTypeRepo(),
+                            *feedView->getSchema(),
+                            *feedView->getDocumentMetaStore(),
+                            *attrMgr,
+                            feedView->getDocumentStore()),
+          _feedView(feedView),
+          _attrMgr(attrMgr)
+    {
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
new file mode 100644
index 00000000000..7363d71ef44
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.fast_access_feed_view");
+#include "fast_access_feed_view.h"
+#include "forcecommitcontext.h"
+#include "operationdonecontext.h"
+#include "removedonecontext.h"
+#include "putdonecontext.h"
+
+using document::Document;
+using document::DocumentUpdate;
+using document::FieldUpdate;
+using search::index::Schema;
+
+namespace proton {
+
+FastAccessFeedView::UpdateScope
+FastAccessFeedView::getUpdateScope(const DocumentUpdate &upd)
+{
+    UpdateScope updateScope;
+    for (const auto &update : upd.getUpdates()) {
+        const vespalib::string &fieldName = update.getField().getName();
+        if (!fastPartialUpdateAttribute(fieldName)) {
+            updateScope._nonAttributeFields = true;
+            break;
+        }
+    }
+    return updateScope;
+}
+
+/**
+ * NOTE: For both put, update and remove we only need to pass the 'onWriteDone'
+ * instance when we are going to commit as part of handling the operation.
+ * Otherwise we can drop it and ack the operation right away.
+ */
+void
+FastAccessFeedView::putAttributes(SerialNum serialNum,
+                                  search::DocumentIdT lid,
+                                  const Document &doc,
+                                  bool immediateCommit,
+                                  OnPutDoneType onWriteDone)
+{
+    _attributeWriter->put(serialNum, doc, lid, immediateCommit, onWriteDone);
+    if (immediateCommit && onWriteDone) {
+        onWriteDone->registerPutLid(lid, &_docIdLimit);
+    }
+}
+
+void
+FastAccessFeedView::updateAttributes(SerialNum serialNum,
+                                     search::DocumentIdT lid,
+                                     const DocumentUpdate &upd,
+                                     bool immediateCommit,
+                                     OnOperationDoneType onWriteDone)
+{
+    _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone);
+}
+
+void
+FastAccessFeedView::removeAttributes(SerialNum serialNum,
+                                     search::DocumentIdT lid,
+                                     bool immediateCommit,
+                                     OnRemoveDoneType onWriteDone)
+{
+    _attributeWriter->remove(serialNum, lid, immediateCommit, onWriteDone);
+}
+
+void
+FastAccessFeedView::removeAttributes(SerialNum serialNum,
+                                     const LidVector &lidsToRemove,
+                                     bool immediateCommit,
+                                     OnWriteDoneType onWriteDone)
+{
+    _attributeWriter->remove(lidsToRemove, serialNum, immediateCommit, onWriteDone);
+}
+
+void
+FastAccessFeedView::heartBeatAttributes(SerialNum serialNum)
+{
+    _attributeWriter->heartBeat(serialNum);
+}
+
+FastAccessFeedView::FastAccessFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx,
+                                       const PersistentParams &params,
+                                       const Context &ctx)
+    : Parent(storeOnlyCtx, params),
+      _attributeWriter(ctx._attrWriter),
+      _docIdLimit(ctx._docIdLimit)
+{
+}
+
+void
+FastAccessFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+{
+    getAttributeWriter()->compactLidSpace(op.getLidLimit(), op.getSerialNum());
+    Parent::handleCompactLidSpace(op);
+    // Drain pending PutDoneContext and ForceCommitContext objects
+    _writeService.sync();
+    _docIdLimit.set(op.getLidLimit());
+}
+
+void
+FastAccessFeedView::forceCommit(SerialNum serialNum,
+                                OnForceCommitDoneType onCommitDone)
+{
+    _attributeWriter->commit(serialNum, onCommitDone);
+    onCommitDone->
+        registerCommittedDocIdLimit(_metaStore.getCommittedDocIdLimit(),
+                                    &_docIdLimit);
+    Parent::forceCommit(serialNum, onCommitDone);
+}
+
+
+void
+FastAccessFeedView::sync()
+{
+    _writeService.attributeFieldWriter().sync();
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h
new file mode 100644
index 00000000000..338a846942f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "storeonlyfeedview.h"
+#include <vespa/searchcore/proton/attribute/i_attribute_writer.h>
+#include <vespa/searchcore/proton/common/docid_limit.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/document/fieldvalue/document.h>
+
+namespace proton {
+
+/**
+ * The feed view used by the fast-access sub database.
+ *
+ * Handles inserting/updating/removing of documents to the underlying
+ * fast-access attributes and document store.
+ */
+class FastAccessFeedView : public StoreOnlyFeedView
+{
+public:
+    typedef std::shared_ptr<FastAccessFeedView> SP;
+    typedef std::unique_ptr<FastAccessFeedView> UP;
+
+    struct Context
+    {
+        const IAttributeWriter::SP &_attrWriter;
+        DocIdLimit                  &_docIdLimit;
+        Context(const IAttributeWriter::SP &attrWriter,
+                DocIdLimit &docIdLimit)
+            : _attrWriter(attrWriter),
+              _docIdLimit(docIdLimit)
+        {
+        }
+    };
+
+private:
+    typedef StoreOnlyFeedView Parent;
+
+    const IAttributeWriter::SP _attributeWriter;
+    DocIdLimit                 &_docIdLimit;
+
+    virtual UpdateScope getUpdateScope(const document::DocumentUpdate &upd);
+
+    virtual void putAttributes(SerialNum serialNum,
+                               search::DocumentIdT lid,
+                               const document::Document &doc,
+                               bool immediateCommit,
+                               OnPutDoneType onWriteDone) override;
+
+    virtual void updateAttributes(SerialNum serialNum,
+                                  search::DocumentIdT lid,
+                                  const document::DocumentUpdate &upd,
+                                  bool immediateCommit,
+                                  OnOperationDoneType onWriteDone);
+
+    virtual void removeAttributes(SerialNum serialNum,
+                                  search::DocumentIdT lid,
+                                  bool immediateCommit,
+                                  OnRemoveDoneType onWriteDone);
+
+    virtual void removeAttributes(SerialNum serialNum,
+                                  const LidVector &lidsToRemove,
+                                  bool immediateCommit,
+                                  OnWriteDoneType onWriteDone);
+
+    virtual void heartBeatAttributes(SerialNum serialNum);
+
+protected:
+    virtual void
+    forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone)
+        override;
+
+public:
+    FastAccessFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx,
+                       const PersistentParams &params,
+                       const Context &ctx);
+
+    virtual const IAttributeWriter::SP &getAttributeWriter() const {
+        return _attributeWriter;
+    }
+
+    virtual DocIdLimit &getDocIdLimit() const {
+        return _docIdLimit;
+    }
+
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op);
+
+    virtual void
+    sync() override;
+
+    bool fastPartialUpdateAttribute(const vespalib::string &fieldName) {
+        search::AttributeVector *attribute =
+            _attributeWriter->getWritableAttribute(fieldName);
+        if (attribute == nullptr) {
+            // Partial update to non-attribute field must update document
+            return false;
+        }
+        search::attribute::BasicType::Type attrType = attribute->getBasicType();
+        // Partial update to tensor or predicate attribute must update document
+        return ((attrType != search::attribute::BasicType::Type::PREDICATE) &&
+                (attrType != search::attribute::BasicType::Type::TENSOR));
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h b/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h
new file mode 100644
index 00000000000..bfc653ac966
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/feedoperation/newconfigoperation.h>
+
+namespace proton {
+
+struct FeedConfigStore : NewConfigOperation::IStreamHandler {
+    virtual ~FeedConfigStore() {}
+
+    virtual void saveWipeHistoryConfig(search::SerialNum,
+                                       fastos::TimeStamp wipeTimeLimit) = 0;
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
new file mode 100644
index 00000000000..61e2a83bd37
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
@@ -0,0 +1,868 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.feedhandler");
+#include "feedhandler.h"
+#include "feedstates.h"
+#include "replaypacketdispatcher.h"
+#include "tlcproxy.h"
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/documentapi/messagebus/messages/documentreply.h>
+#include <vespa/documentapi/messagebus/messages/feedreply.h>
+#include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
+#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchcore/proton/feedoperation/operations.h>
+#include <vespa/searchcore/proton/persistenceengine/transport_latch.h>
+#include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
+#include "ddbstate.h"
+
+using document::BucketId;
+using document::Document;
+using document::DocumentTypeRepo;
+using documentapi::DocumentProtocol;
+using documentapi::DocumentReply;
+using documentapi::FeedReply;
+using documentapi::RemoveDocumentReply;
+using documentapi::UpdateDocumentReply;
+using storage::spi::PartitionId;
+using storage::spi::RemoveResult;
+using storage::spi::Result;
+using storage::spi::Timestamp;
+using storage::spi::Timestamp;
+using storage::spi::UpdateResult;
+using vespalib::Executor;
+using vespalib::IllegalStateException;
+using vespalib::ThreadStackExecutorBase;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+using vespalib::make_string;
+using vespalib::MonitorGuard;
+using vespalib::LockGuard;
+
+namespace proton {
+
+
+namespace {
+void
+setUpdateWasFound(mbus::Reply &reply, bool was_found)
+{
+    assert(static_cast<DocumentReply&>(reply).getType() ==
+           DocumentProtocol::REPLY_UPDATEDOCUMENT);
+    UpdateDocumentReply &update_rep = static_cast<UpdateDocumentReply&>(reply);
+    update_rep.setWasFound(was_found);
+}
+
+
+void
+setRemoveWasFound(mbus::Reply &reply, bool was_found)
+{
+    assert(static_cast<DocumentReply&>(reply).getType() ==
+           DocumentProtocol::REPLY_REMOVEDOCUMENT);
+    RemoveDocumentReply &remove_rep = static_cast<RemoveDocumentReply&>(reply);
+    remove_rep.setWasFound(was_found);
+}
+
+
+bool
+ignoreOperation(const DocumentOperation &op)
+{
+    return op.getPrevTimestamp() != 0 &&
+        op.getTimestamp() < op.getPrevTimestamp();
+}
+
+
+}  // namespace
+
+
+void FeedHandler::TlsMgrWriter::storeOperation(const FeedOperation &op) {
+    TlcProxy(*_tls_mgr.getSession(), _tlsDirectWriter).storeOperation(op);
+}
+bool FeedHandler::TlsMgrWriter::erase(SerialNum oldest_to_keep) {
+    return _tls_mgr.getSession()->erase(oldest_to_keep);
+}
+
+search::SerialNum
+FeedHandler::TlsMgrWriter::sync(SerialNum syncTo)
+{
+    for (int retryCount = 0; retryCount < 10; ++retryCount) {
+        
+        SerialNum syncedTo(0);
+        LOG(spam, "Trying tls sync(%" PRIu64 ")", syncTo);
+        bool res = _tls_mgr.getSession()->sync(syncTo, syncedTo);
+        if (!res) {
+            LOG(spam, "Tls sync failed, retrying");
+            sleep(1);
+            continue;
+        }
+        if (syncedTo >= syncTo) {
+            LOG(spam,
+                "Tls sync complete, reached %" PRIu64", returning",
+                syncedTo);
+            return syncedTo;
+        }
+        LOG(spam,
+            "Tls sync incomplete, reached %" PRIu64 ", retrying",
+            syncedTo);
+    }
+    throw vespalib::IllegalStateException(
+            vespalib::make_string(
+                    "Failed to sync TLS to token %" PRIu64 ".",
+                    syncTo));
+    return 0;
+}
+
+void
+FeedHandler::doHandleOperation(FeedToken token, FeedOperation::UP op)
+{
+    assert(_writeService.master().isCurrentThread());
+    vespalib::LockGuard guard(_feedLock);
+    _feedState->handleOperation(token, std::move(op));
+}
+
+namespace {
+template <typename ResultType>
+void configRejected(FeedToken *token, DocTypeName docTypeName) {
+    if (token) {
+        vespalib::string str =
+            make_string("Feed rejected for documenttype '%s'"
+                        " due to incompatible changes to search definition.",
+                        docTypeName.toString().c_str());
+        token->setResult(
+                ResultUP(new ResultType(Result::PERMANENT_ERROR, str)), false);
+        token->fail(documentapi::DocumentProtocol::ERROR_REJECTED, str);
+    }
+}
+
+void notifyConfigRejected(FeedToken *token, FeedOperation::Type type,
+                          DocTypeName docTypeName) {
+    if (type == FeedOperation::REMOVE) {
+        configRejected<RemoveResult>(token, docTypeName);
+    } else if (type == FeedOperation::UPDATE) {
+        configRejected<UpdateResult>(token, docTypeName);
+    } else {
+        configRejected<Result>(token, docTypeName);
+    }
+}
+}  // namespace
+
+
+void FeedHandler::performPut(FeedToken::UP token, PutOperation &op) {
+    op.assertValid();
+    _activeFeedView->preparePut(op);
+    if (ignoreOperation(op)) {
+        LOG(debug, "performPut(): ignoreOperation: docId(%s), "
+            "timestamp(%" PRIu64 "), prevTimestamp(%" PRIu64 ")",
+            op.getDocument()->getId().toString().c_str(),
+            (uint64_t)op.getTimestamp(),
+            (uint64_t)op.getPrevTimestamp());
+        if (token.get() != NULL) {
+            token->setResult(ResultUP(new Result), false);
+            token->ack(op.getType(), _metrics);
+        }
+        return;
+    }
+    storeOperation(op);
+    if (token.get() != NULL) {
+        token->setResult(ResultUP(new Result), false);
+        if (token->shouldTrace(1)) {
+            const document::DocumentId &docId = op.getDocument()->getId();
+            const document::GlobalId &gid = docId.getGlobalId();
+            token->trace(1,
+                         make_string(
+                                 "Indexing document '%s' (gid = '%s',"
+                                 " lid = '%u,%u' prevlid = '%u,%u').",
+                                 docId.toString().c_str(),
+                                 gid.toString().c_str(),
+                                 op.getSubDbId(),
+                                 op.getLid(),
+                                 op.getPrevSubDbId(),
+                                 op.getPrevLid()));
+        }
+    }
+    _activeFeedView->handlePut(token.get(), op);
+}
+
+
+void
+FeedHandler::performUpdate(FeedToken::UP token, UpdateOperation &op)
+{
+    _activeFeedView->prepareUpdate(op);
+    if (op.getPrevDbDocumentId().valid() && !op.getPrevMarkedAsRemoved()) {
+        performInternalUpdate(std::move(token), op);
+    } else if (op.getUpdate()->getCreateIfNonExistent()) {
+        createNonExistingDocument(std::move(token), op);
+    } else {
+        if (token.get() != NULL) {
+            token->setResult(ResultUP(new UpdateResult(Timestamp(0))), false);
+            if (token->shouldTrace(1)) {
+                const document::DocumentId &docId = op.getUpdate()->getId();
+                 token->trace(1,
+                             make_string(
+                                         "Document '%s' not found."
+                                         " Update operation ignored",
+                                         docId.toString().c_str()));
+            }
+            setUpdateWasFound(token->getReply(), false);
+            token->ack(op.getType(), _metrics);
+        }
+    }
+}
+
+
+void
+FeedHandler::performInternalUpdate(FeedToken::UP token, UpdateOperation &op)
+{
+    storeOperation(op);
+    if (token.get() != NULL) {
+        token->setResult(ResultUP(new UpdateResult(op.getPrevTimestamp())),
+                         true);
+        if (token->shouldTrace(1)) {
+            const document::DocumentId &docId = op.getUpdate()->getId();
+            const document::GlobalId &gid = docId.getGlobalId();
+            token->trace(1,
+                         make_string(
+                                     "Updating document '%s' (gid = '%s',"
+                                     " lid = '%u,%u' prevlid = '%u,%u').",
+                                     docId.toString().c_str(),
+                                     gid.toString().c_str(),
+                                     op.getSubDbId(),
+                                     op.getLid(),
+                                     op.getPrevSubDbId(),
+                                     op.getPrevLid()));
+        }
+        setUpdateWasFound(token->getReply(), true);
+    }
+    _activeFeedView->handleUpdate(token.get(), op);
+}
+
+
+void
+FeedHandler::createNonExistingDocument(FeedToken::UP token, const UpdateOperation &op)
+{
+    Document::SP doc(new Document(op.getUpdate()->getType(),
+                                  op.getUpdate()->getId()));
+    doc->setRepo(*_activeFeedView->getDocumentTypeRepo());
+    op.getUpdate()->applyTo(*doc);
+    PutOperation putOp(op.getBucketId(),
+                       op.getTimestamp(),
+                       doc);
+    _activeFeedView->preparePut(putOp);
+    storeOperation(putOp);
+    if (token.get() != NULL) {
+        token->setResult(ResultUP(new UpdateResult(putOp.getTimestamp())), true);
+        if (token->shouldTrace(1)) {
+            const document::DocumentId &docId = putOp.getDocument()->getId();
+            const document::GlobalId &gid = docId.getGlobalId();
+            token->trace(1, make_string("Creating non-existing document '%s' for update (gid='%s',"
+                                     " lid= %u,%u' prevlid='%u,%u').",
+                                     docId.toString().c_str(),
+                                     gid.toString().c_str(),
+                                     putOp.getSubDbId(),
+                                     putOp.getLid(),
+                                     putOp.getPrevSubDbId(),
+                                     putOp.getPrevLid()));
+        }
+        setUpdateWasFound(token->getReply(), true);
+    }
+    TransportLatch latch(1);
+    FeedToken putToken(latch, mbus::Reply::UP(new FeedReply(DocumentProtocol::REPLY_PUTDOCUMENT)));
+    _activeFeedView->handlePut(&putToken, putOp);
+    latch.await();
+    if (token.get() != NULL) {
+        token->ack();
+    }
+}
+
+
+void FeedHandler::performRemove(FeedToken::UP token, RemoveOperation &op) {
+    _activeFeedView->prepareRemove(op);
+    if (ignoreOperation(op)) {
+        LOG(debug, "performRemove(): ignoreOperation: docId(%s), "
+            "timestamp(%" PRIu64 "), prevTimestamp(%" PRIu64 ")",
+            op.getDocumentId().toString().c_str(),
+            (uint64_t)op.getTimestamp(),
+            (uint64_t)op.getPrevTimestamp());
+        if (token.get() != NULL) {
+            token->setResult(ResultUP(new RemoveResult(false)), false);
+            token->ack(op.getType(), _metrics);
+        }
+        return;
+    }
+    if (op.getPrevDbDocumentId().valid()) {
+        assert(op.getValidNewOrPrevDbdId());
+        assert(op.notMovingLidInSameSubDb());
+        storeOperation(op);
+        if (token.get() != NULL) {
+            bool documentWasFound = !op.getPrevMarkedAsRemoved();
+            token->setResult(ResultUP(new RemoveResult(documentWasFound)),
+                             documentWasFound);
+            if (token->shouldTrace(1)) {
+                const document::DocumentId &docId = op.getDocumentId();
+                const document::GlobalId &gid = docId.getGlobalId();
+                token->trace(1,
+                             make_string(
+                                     "Removing document '%s' (gid = '%s',"
+                                     " lid = '%u,%u' prevlid = '%u,%u').",
+                                     docId.toString().c_str(),
+                                     gid.toString().c_str(),
+                                     op.getSubDbId(),
+                                     op.getLid(),
+                                     op.getPrevSubDbId(),
+                                     op.getPrevLid()));
+            }
+            setRemoveWasFound(token->getReply(), documentWasFound);
+        }
+        _activeFeedView->handleRemove(token.get(), op);
+    } else if (op.hasDocType()) {
+        assert(op.getDocType() == _docTypeName.getName());
+        storeOperation(op);
+        if (token.get() != NULL) {
+            token->setResult(ResultUP(new RemoveResult(false)), false);
+            if (token->shouldTrace(1)) {
+                token->trace(1,
+                             make_string(
+                                     "Document '%s' not found."
+                                     " Remove operation stored.",
+                                     op.getDocumentId().toString().c_str()));
+            }
+            setRemoveWasFound(token->getReply(), false);
+        }
+        _activeFeedView->handleRemove(token.get(), op);
+    } else {
+        if (token.get() != NULL) {
+            token->setResult(ResultUP(new RemoveResult(false)), false);
+            if (token->shouldTrace(1)) {
+                token->trace(1,
+                             make_string(
+                                     "Document '%s' not found."
+                                     " Remove operation ignored",
+                                     op.getDocumentId().toString().c_str()));
+            }
+            setRemoveWasFound(token->getReply(), false);
+            token->ack(op.getType(), _metrics);
+        }
+    }
+}
+
+void
+FeedHandler::performGarbageCollect(FeedToken::UP token)
+{
+    _owner.performWipeHistory();
+    if (token.get() != NULL) {
+        token->ack();
+    }
+}
+
+
+void
+FeedHandler::performCreateBucket(FeedToken::UP token,
+                                 CreateBucketOperation &op)
+{
+    storeOperation(op);
+    _bucketDBHandler->handleCreateBucket(op.getBucketId());
+    if (token) {
+        token->ack();
+    }
+}
+
+
+void FeedHandler::performDeleteBucket(FeedToken::UP token,
+                                      DeleteBucketOperation &op) {
+    _activeFeedView->prepareDeleteBucket(op);
+    storeOperation(op);
+    // Delete documents in bucket
+    _activeFeedView->handleDeleteBucket(op);
+    // Delete bucket itself, should no longer have documents.
+    _bucketDBHandler->handleDeleteBucket(op.getBucketId());
+    if (token) {
+        token->ack();
+    }
+}
+
+
+void FeedHandler::performSplit(FeedToken::UP token, SplitBucketOperation &op) {
+    storeOperation(op);
+    _bucketDBHandler->handleSplit(op.getSerialNum(),
+                                  op.getSource(),
+                                  op.getTarget1(),
+                                  op.getTarget2());
+    if (token) {
+        token->ack();
+    }
+}
+
+
+void FeedHandler::performJoin(FeedToken::UP token, JoinBucketsOperation &op) {
+    storeOperation(op);
+    _bucketDBHandler->handleJoin(op.getSerialNum(),
+                                 op.getSource1(),
+                                 op.getSource2(),
+                                 op.getTarget());
+    if (token) {
+        token->ack();
+    }
+}
+
+
+void
+FeedHandler::performSync()
+{
+    assert(_writeService.master().isCurrentThread());
+    _activeFeedView->sync();
+}
+
+void
+FeedHandler::performEof()
+{
+    assert(_writeService.master().isCurrentThread());
+    _writeService.sync();
+    LOG(debug,
+        "Visiting done for transaction log domain '%s', eof received",
+        _tlsMgr.getDomainName().c_str());
+    _owner.onTransactionLogReplayDone();
+    _tlsMgr.replayDone();
+    changeToNormalFeedState();
+    _owner.enterRedoReprocessState();
+}
+
+
+void
+FeedHandler::performFlushDone(SerialNum oldestSerial)
+{
+    assert(_writeService.master().isCurrentThread());
+    // XXX: oldestSerial can go backwards when attribute vectors are
+    // resurrected.  This can be avoided if resurrected attribute vectors
+    // pretends to have been flushed at resurrect time.
+    if (oldestSerial <= _prunedSerialNum) {
+        return;                                // Cannot unprune.
+    } 
+    if (!_owner.getAllowPrune()) {
+        _prunedSerialNum = oldestSerial;
+        _delayedPrune = true;
+        return;
+    }
+    _delayedPrune = false;
+    performPrune(oldestSerial);
+}
+
+
+void
+FeedHandler::performPrune(SerialNum oldestSerial)
+{
+    try {
+        tlsPrune(oldestSerial);  // throws on error
+        LOG(debug, "Pruned TLS to token %" PRIu64 ".", oldestSerial);
+        _owner.onPerformPrune(oldestSerial);
+    } catch (const vespalib::IllegalStateException & e) {
+        LOG(warning, "FeedHandler::performPrune failed due to '%s'.", e.what());
+    }
+}
+
+
+void
+FeedHandler::considerDelayedPrune()
+{
+    if (_delayedPrune) {
+        _delayedPrune = false;
+        performPrune(_prunedSerialNum);
+    }
+}
+
+
+FeedState::SP
+FeedHandler::getFeedState() const
+{
+    FeedState::SP state;
+    {
+        vespalib::LockGuard guard(_feedLock);
+        state = _feedState;
+    }
+    return state;
+}
+
+
+void
+FeedHandler::changeFeedState(FeedState::SP newState)
+{
+    vespalib::LockGuard guard(_feedLock);
+    changeFeedState(newState, guard);
+}
+
+
+void
+FeedHandler::changeFeedState(FeedState::SP newState,
+                             const vespalib::LockGuard &)
+{
+    LOG(debug,
+        "Change feed state from '%s' -> '%s'",
+        _feedState->getName().c_str(), newState->getName().c_str());
+    _feedState = newState;
+}
+
+
+FeedHandler::FeedHandler(IThreadingService &writeService,
+                         const vespalib::string &tlsSpec,
+                         const DocTypeName &docTypeName,
+                         PerDocTypeFeedMetrics &metrics,
+                         DDBState &state,
+                         IOwner &owner,
+                         const IResourceWriteFilter &writeFilter,
+                         IReplayConfig &replayConfig,
+                         search::transactionlog::Writer *tlsDirectWriter,
+                         TlsWriter *tls_writer)
+    : boost::noncopyable(),
+      search::transactionlog::TransLogClient::Session::Callback(),
+      IDocumentMoveHandler(),
+      IPruneRemovedDocumentsHandler(),
+      IHeartBeatHandler(),
+      IOperationStorer(),
+      IGetSerialNum(),
+      _writeService(writeService),
+      _docTypeName(docTypeName),
+      _state(state),
+      _owner(owner),
+      _writeFilter(writeFilter),
+      _replayConfig(replayConfig),
+      _tlsMgr(tlsSpec, docTypeName.getName()),
+      _tlsMgrWriter(_tlsMgr, tlsDirectWriter),
+      _tlsWriter(tls_writer ? *tls_writer : _tlsMgrWriter),
+      _tlsReplayProgress(),
+      _serialNum(0),
+      _prunedSerialNum(0),
+      _delayedPrune(false),
+      _feedLock(),
+      _feedState(std::make_shared<InitState>(getDocTypeName())),
+      _activeFeedView(NULL),
+      _bucketDBHandler(nullptr),
+      _metrics(metrics),
+      _syncLock(),
+      _syncedSerialNum(0),
+      _allowSync(false)
+{
+}
+
+
+FeedHandler::~FeedHandler()
+{
+}
+
+
+// Called on DocumentDB creatio
+void
+FeedHandler::init(SerialNum oldestConfigSerial)
+{
+    _tlsMgr.init(oldestConfigSerial, _prunedSerialNum, _serialNum);
+    _allowSync = true;
+    syncTls(_serialNum);
+}
+
+
+void
+FeedHandler::close()
+{
+    if (_allowSync) {
+        syncTls(_serialNum);
+    }
+    _allowSync = false;
+    _tlsMgr.close();
+}
+
+
+void
+FeedHandler::replayTransactionLog(SerialNum flushedIndexMgrSerial,
+                                  SerialNum flushedSummaryMgrSerial,
+                                  SerialNum oldestFlushedSerial,
+                                  SerialNum newestFlushedSerial,
+                                  ConfigStore &config_store)
+{
+    assert(_activeFeedView);
+    assert(_bucketDBHandler);
+    FeedState::SP state = std::make_shared<ReplayTransactionLogState>
+                          (getDocTypeName(),
+                           _activeFeedView,
+                           *_bucketDBHandler,
+                           _replayConfig,
+                           config_store);
+    changeFeedState(state);
+    // Resurrected attribute vector might cause oldestFlushedSerial to
+    // be lower than _prunedSerialNum, so don't warn for now.
+    (void) oldestFlushedSerial;
+    assert(_serialNum >= newestFlushedSerial);
+
+    TransactionLogManager::prepareReplay(
+            _tlsMgr.getClient(),
+            _docTypeName.getName(),
+            flushedIndexMgrSerial,
+            flushedSummaryMgrSerial,
+            config_store);
+
+    _tlsReplayProgress = _tlsMgr.startReplay(_prunedSerialNum, _serialNum, *this);
+}
+
+
+void
+FeedHandler::flushDone(SerialNum oldestSerial)
+{
+    // Called by flush worker thread after performing a flush task
+    _writeService.master().execute(
+            makeTask(
+                    makeClosure(
+                            this,
+                            &FeedHandler::performFlushDone,
+                            oldestSerial)));
+}
+
+void FeedHandler::changeToNormalFeedState(void) {
+    changeFeedState(FeedState::SP(new NormalState(*this)));
+}
+
+void
+FeedHandler::waitForReplayDone()
+{
+    _tlsMgr.waitForReplayDone();
+}
+
+void FeedHandler::setReplayDone() {
+    _tlsMgr.changeReplayDone();
+}
+
+bool FeedHandler::getReplayDone() const {
+    return _tlsMgr.getReplayDone();
+}
+
+bool
+FeedHandler::isDoingReplay() const {
+    return _tlsMgr.isDoingReplay();
+}
+
+bool FeedHandler::getTransactionLogReplayDone() const {
+    return _tlsMgr.getReplayDone();
+}
+
+void FeedHandler::storeOperation(FeedOperation &op) {
+    if (!op.getSerialNum()) {
+        op.setSerialNum(incSerialNum());
+    }
+    _tlsWriter.storeOperation(op);
+}
+
+void FeedHandler::tlsPrune(SerialNum oldest_to_keep) {
+    if (!_tlsWriter.erase(oldest_to_keep)) {
+        throw vespalib::IllegalStateException(vespalib::make_string(
+                        "Failed to prune TLS to token %" PRIu64 ".",
+                        oldest_to_keep));
+    }
+    _prunedSerialNum = oldest_to_keep;
+}
+
+namespace {
+
+bool
+isRejectableFeedOperation(FeedOperation::Type type)
+{
+    return type == FeedOperation::PUT || type == FeedOperation::UPDATE;
+}
+
+template <typename ResultType>
+void feedOperationRejected(FeedToken *token, const vespalib::string &opType, const vespalib::string &docId,
+                           DocTypeName docTypeName, const vespalib::string &rejectMessage)
+{
+    if (token) {
+        vespalib::string message = make_string("%s operation rejected for document '%s' of type '%s': '%s'",
+                                               opType.c_str(), docId.c_str(), docTypeName.toString().c_str(), rejectMessage.c_str());
+        token->setResult(ResultUP(new ResultType(Result::RESOURCE_EXHAUSTED, message)), false);
+        token->fail(documentapi::DocumentProtocol::ERROR_REJECTED, message);
+    }
+}
+
+void
+notifyFeedOperationRejected(FeedToken *token, const FeedOperation &op,
+                            DocTypeName docTypeName, const vespalib::string &rejectMessage)
+{
+    if (op.getType() == FeedOperation::UPDATE) {
+        vespalib::string docId = (static_cast<const UpdateOperation &>(op)).getUpdate()->getId().toString();
+        feedOperationRejected<UpdateResult>(token, "Update", docId, docTypeName, rejectMessage);
+    } else if (op.getType() == FeedOperation::PUT) {
+        vespalib::string docId = (static_cast<const PutOperation &>(op)).getDocument()->getId().toString();
+        feedOperationRejected<Result>(token, "Put", docId, docTypeName, rejectMessage);
+    } else {
+        feedOperationRejected<Result>(token, "Feed", "", docTypeName, rejectMessage);
+    }
+}
+
+}
+
+bool
+FeedHandler::considerWriteOperationForRejection(FeedToken *token, const FeedOperation &op)
+{
+    if (_owner.isFeedBlockedByRejectedConfig()) {
+        notifyConfigRejected(token, op.getType(), _docTypeName);
+        return true;
+    }
+    if (!_writeFilter.acceptWriteOperation() && isRejectableFeedOperation(op.getType())) {
+        IResourceWriteFilter::State state = _writeFilter.getAcceptState();
+        if (!state.acceptWriteOperation()) {
+            notifyFeedOperationRejected(token, op, _docTypeName, state.message());
+            return true;
+        }
+    }
+    return false;
+}
+
+void
+FeedHandler::performOperation(FeedToken::UP token, FeedOperation::UP op)
+{
+    if (considerWriteOperationForRejection(token.get(), *op)) {
+        return;
+    }
+    switch(op->getType()) {
+    case FeedOperation::PUT:
+        performPut(std::move(token), static_cast<PutOperation &>(*op));
+        return;
+    case FeedOperation::REMOVE:
+        performRemove(std::move(token), static_cast<RemoveOperation &>(*op));
+        return;
+    case FeedOperation::UPDATE:
+        performUpdate(std::move(token), static_cast<UpdateOperation &>(*op));
+        return;
+    case FeedOperation::DELETE_BUCKET:
+        performDeleteBucket(std::move(token), static_cast<DeleteBucketOperation &>(*op));
+        return;
+    case FeedOperation::SPLIT_BUCKET:
+        performSplit(std::move(token), static_cast<SplitBucketOperation &>(*op));
+        return;
+    case FeedOperation::JOIN_BUCKETS:
+        performJoin(std::move(token), static_cast<JoinBucketsOperation &>(*op));
+        return;
+    case FeedOperation::WIPE_HISTORY:
+        performGarbageCollect(std::move(token));
+        return;
+    case FeedOperation::CREATE_BUCKET:
+        performCreateBucket(std::move(token), static_cast<CreateBucketOperation &>(*op));
+        return;
+    default:
+        assert(!"Illegal operation type");
+    }
+}
+
+void
+FeedHandler::handleOperation(FeedToken token, FeedOperation::UP op)
+{
+    _writeService.master().execute(
+            makeTask(makeClosure(this,
+                                 &FeedHandler::doHandleOperation, token, std::move(op))));
+}
+
+void
+FeedHandler::handleMove(MoveOperation &op)
+{
+    assert(_writeService.master().isCurrentThread());
+    _activeFeedView->prepareMove(op);
+    assert(op.getValidDbdId());
+    assert(op.getValidPrevDbdId());
+    assert(op.getSubDbId() != op.getPrevSubDbId());
+    storeOperation(op);
+    _activeFeedView->handleMove(op);
+}
+
+
+void
+FeedHandler::heartBeat(void)
+{
+    assert(_writeService.master().isCurrentThread());
+    if (_owner.isFeedBlockedByRejectedConfig())
+        return;
+    _activeFeedView->heartBeat(_serialNum);
+}
+
+
+void
+FeedHandler::sync()
+{
+    _writeService.master().execute(makeTask(makeClosure(this, &FeedHandler::performSync)));
+    _writeService.sync();
+}
+
+
+FeedHandler::RPC::Result
+FeedHandler::receive(const Packet &packet)
+{
+    // Called directly when replaying transaction log
+    // (by fnet thread).  Called via DocumentDB::recoverPacket() when
+    // recovering from another node.
+    FeedState::SP state = getFeedState();
+    PacketWrapper::SP wrap(new PacketWrapper(packet, _tlsReplayProgress.get()));
+    state->receive(wrap, _writeService.master());
+    wrap->gate.await();
+    return wrap->result;
+}
+
+
+void
+FeedHandler::eof()
+{
+    // Only called by visit, subscription gets one or more inSync() callbacks.
+    _writeService.master().execute(makeTask(makeClosure(this, &FeedHandler::performEof)));
+}
+
+
+void
+FeedHandler::inSync()
+{
+    // Called by visit callback thread, when in sync
+}
+
+
+void
+FeedHandler::
+performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp)
+{
+    const LidVectorContext::LP lids_to_remove = pruneOp.getLidsToRemove();
+    if (lids_to_remove.get() && lids_to_remove->getNumLids() != 0) {
+        storeOperation(pruneOp);
+        _activeFeedView->handlePruneRemovedDocuments(pruneOp);
+    }
+}
+
+
+void
+FeedHandler::syncTls(SerialNum syncTo)
+{
+    {
+        LockGuard guard(_syncLock);
+        if (_syncedSerialNum >= syncTo)
+            return;
+    }
+    if (!_allowSync) {
+        throw vespalib::IllegalStateException(
+                vespalib::make_string(
+                        "Attempted to sync TLS to token %" PRIu64
+                        " at wrong time.",
+                        syncTo));
+    }
+    SerialNum syncedTo(_tlsWriter.sync(syncTo));
+    {
+        LockGuard guard(_syncLock);
+        if (_syncedSerialNum < syncedTo) 
+            _syncedSerialNum = syncedTo;
+    }
+}
+
+void
+FeedHandler::storeRemoteOperation(const FeedOperation &op)
+{
+    SerialNum serialNum(op.getSerialNum()); 
+    assert(serialNum != 0);
+    if (serialNum > _serialNum) {
+        _tlsWriter.storeOperation(op);
+        _serialNum = serialNum;
+    }
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h
new file mode 100644
index 00000000000..536013e0d02
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h
@@ -0,0 +1,336 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "feedstate.h"
+#include "i_operation_storer.h"
+#include "idocumentmovehandler.h"
+#include "ifeedview.h"
+#include "igetserialnum.h"
+#include "iheartbeathandler.h"
+#include "ipruneremoveddocumentshandler.h"
+#include "ireplayconfig.h"
+#include "tlswriter.h"
+#include "transactionlogmanager.h"
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
+#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
+#include <vespa/vespalib/util/executor.h>
+
+namespace proton {
+class ConfigStore;
+class FeedConfigStore;
+class IDocumentDBOwner;
+class CreateBucketOperation;
+class DDBState;
+
+namespace bucketdb
+{
+
+class IBucketDBHandler;
+
+}
+
+/**
+ * Class handling all aspects of feeding for a document database.
+ * In addition to regular feeding this also includes handling the transaction log.
+ */
+class FeedHandler: public boost::noncopyable,
+                   private search::transactionlog::
+                   TransLogClient::Session::Callback,
+                   public IDocumentMoveHandler,
+                   public IPruneRemovedDocumentsHandler,
+                   public IHeartBeatHandler,
+                   public IOperationStorer,
+                   public IGetSerialNum
+{
+private:
+    typedef search::transactionlog::Packet  Packet;
+    typedef search::transactionlog::RPC     RPC;
+    typedef search::SerialNum               SerialNum;
+    typedef storage::spi::Timestamp         Timestamp;
+    typedef document::BucketId              BucketId;
+
+public:
+    /**
+     * Interface defining the communication needed with the owner of the feed handler.
+     */
+    struct IOwner {
+        virtual ~IOwner() {}
+        virtual void performWipeHistory() = 0;
+        virtual void onTransactionLogReplayDone() = 0;
+        virtual void enterRedoReprocessState() = 0;
+        virtual void onPerformPrune(SerialNum oldestSerial) = 0;
+        virtual bool isFeedBlockedByRejectedConfig() = 0;
+        virtual bool getAllowPrune() const = 0;
+    };
+
+private:
+    class TlsMgrWriter : public TlsWriter {
+        TransactionLogManager &_tls_mgr;
+        search::transactionlog::Writer *_tlsDirectWriter;
+    public:
+        TlsMgrWriter(TransactionLogManager &tls_mgr,
+                     search::transactionlog::Writer * tlsDirectWriter) :
+            _tls_mgr(tls_mgr),
+            _tlsDirectWriter(tlsDirectWriter)
+        { }
+        virtual void storeOperation(const FeedOperation &op);
+        virtual bool erase(SerialNum oldest_to_keep);
+
+        virtual SerialNum
+        sync(SerialNum syncTo);
+    };
+    typedef searchcorespi::index::IThreadingService IThreadingService;
+
+    IThreadingService                     &_writeService;
+    DocTypeName                            _docTypeName;
+    DDBState                              &_state;
+    IOwner                                &_owner;
+    const IResourceWriteFilter            &_writeFilter;
+    IReplayConfig                         &_replayConfig;
+    TransactionLogManager                  _tlsMgr;
+    TlsMgrWriter                           _tlsMgrWriter;
+    TlsWriter                             &_tlsWriter;
+    TlsReplayProgress::UP                  _tlsReplayProgress;
+    // the serial num of the last message in the transaction log
+    SerialNum                              _serialNum;
+    SerialNum                              _prunedSerialNum;
+    bool                                   _delayedPrune;
+    vespalib::Lock                         _feedLock;
+    FeedState::SP                          _feedState;
+    // used by master write thread tasks
+    IFeedView                             *_activeFeedView;
+    bucketdb::IBucketDBHandler            *_bucketDBHandler;
+    PerDocTypeFeedMetrics                 &_metrics;
+
+    vespalib::Lock                         _syncLock;
+    SerialNum                              _syncedSerialNum; 
+    bool                                   _allowSync; // Sanity check
+
+    /**
+     * Delayed handling of feed operations, in master write thread.
+     * The current feed state is sampled here.
+     */
+    void doHandleOperation(FeedToken token, FeedOperation::UP op);
+
+    bool considerWriteOperationForRejection(FeedToken *token, const FeedOperation &op);
+
+    /**
+     * Delayed execution of feed operations against feed view, in
+     * master write thread.
+     */
+    void performPut(FeedToken::UP token, PutOperation &op);
+
+    void performUpdate(FeedToken::UP token, UpdateOperation &op);
+    void performInternalUpdate(FeedToken::UP token, UpdateOperation &op);
+    void createNonExistingDocument(FeedToken::UP, const UpdateOperation &op);
+
+    void performRemove(FeedToken::UP token, RemoveOperation &op);
+private:
+    void performGarbageCollect(FeedToken::UP token);
+
+    void
+    performCreateBucket(FeedToken::UP token, CreateBucketOperation &op);
+
+    void performDeleteBucket(FeedToken::UP token, DeleteBucketOperation &op);
+    void performSplit(FeedToken::UP token, SplitBucketOperation &op);
+    void performJoin(FeedToken::UP token, JoinBucketsOperation &op);
+    void performSync(void);
+
+    /**
+     * Used during callback from transaction log.
+     */
+    void
+    handleTransactionLogEntry(const Packet::Entry &entry);
+
+    void
+    performEof(void);
+
+    /**
+     * Used when flushing is done
+     */
+    void
+    performFlushDone(SerialNum oldestSerial);
+
+    void
+    performPrune(SerialNum oldestSerial);
+
+public:
+    void
+    considerDelayedPrune(void);
+
+private:
+    /**
+     * Returns the current feed state of this feed handler.
+     */
+    FeedState::SP
+    getFeedState() const;
+
+    /**
+     * Used to handle feed state transitions.
+     */
+    void
+    changeFeedState(FeedState::SP newState);
+
+    void
+    changeFeedState(FeedState::SP newState,
+                    const vespalib::LockGuard &feedGuard);
+
+public:
+    /**
+     * Create a new feed handler.
+     *
+     * @param writeService  The thread service used for all write tasks.
+     * @param tlsSpec       The spec to connect to the transaction log server.
+     * @param docTypeName   The name and version of the document type we are feed handler for.
+     * @param metrics       Feeding metrics.
+     * @param state         Document db state
+     * @param owner         Reference to the owner of this feed handler.
+     * @param replayConfig  Reference to interface used for replaying config changes.
+     * @param writer        Inject writer for tls, or NULL to use internal.
+     */
+    FeedHandler(IThreadingService &writeService,
+                const vespalib::string &tlsSpec,
+                const DocTypeName &docTypeName,
+                PerDocTypeFeedMetrics &metrics,
+                DDBState &state,
+                IOwner &owner,
+                const IResourceWriteFilter &writerFilter,
+                IReplayConfig &replayConfig,
+                search::transactionlog::Writer *writer,
+                TlsWriter *tlsWriter = NULL);
+
+    virtual
+    ~FeedHandler();
+
+    /**
+     * Init this feed handler.
+     *
+     * @param oldestConfigSerial The serial number of the oldest config snapshot.
+     */
+    void
+    init(SerialNum oldestConfigSerial);
+
+    /**
+     * Close this feed handler and its components.
+     */
+    void
+    close();
+
+    /**
+     * Start replay of the transaction log.
+     *
+     * @param flushedIndexMgrSerial   The flushed serial number of the
+     *                                index manager.
+     * @param flushedSummaryMgrSerial The flushed serial number of the
+     *                                document store.
+     * @param config_store            Reference to the config store.
+     */
+
+    void
+    replayTransactionLog(SerialNum flushedIndexMgrSerial,
+                         SerialNum flushedSummaryMgrSerial,
+                         SerialNum oldestFlushedSerial,
+                         SerialNum newestFlushedSerial,
+                         ConfigStore &config_store);
+
+    /**
+     * Called when a flush is done and allows pruning of the transaction log.
+     *
+     * @param oldestSerial The oldest serial number that is still in use.
+     */
+    void
+    flushDone(SerialNum oldestSerial);
+
+    /**
+     * Used to flip between normal and recovery feed states.
+     */
+    void changeToNormalFeedState();
+
+    /**
+     * Update the active feed view.
+     * Always called by the master write thread so locking is not needed.
+     */
+    void
+    setActiveFeedView(IFeedView *feedView)
+    {
+        _activeFeedView = feedView;
+    }
+
+    void
+    setBucketDBHandler(bucketdb::IBucketDBHandler *bucketDBHandler)
+    {
+        _bucketDBHandler = bucketDBHandler;
+    }
+
+    /**
+     * Wait until transaction log is replayed.
+     */
+    void waitForReplayDone();
+
+    void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; }
+    SerialNum incSerialNum() { return ++_serialNum; }
+    SerialNum getSerialNum() const override { return _serialNum; }
+    SerialNum getPrunedSerialNum() const { return _prunedSerialNum; }
+
+    void setReplayDone();
+    bool getReplayDone() const;
+    bool isDoingReplay() const;
+    float getReplayProgress() const {
+        return _tlsReplayProgress.get() != nullptr ? _tlsReplayProgress->getProgress() : 0;
+    }
+    bool getTransactionLogReplayDone() const;
+    vespalib::string getDocTypeName() const { return _docTypeName.getName(); }
+    void tlsPrune(SerialNum oldest_to_keep);
+
+    void performOperation(FeedToken::UP token, FeedOperation::UP op);
+    void handleOperation(FeedToken token, FeedOperation::UP op);
+
+    /**
+     * Implements IDocumentMoveHandler
+     */
+    virtual void
+    handleMove(MoveOperation &op);
+
+    /**
+     * Implements IHeartBeatHandler
+     */
+    virtual void
+    heartBeat(void);
+
+    virtual void
+    sync(void);
+
+    /**
+     * Implements TransLogClient::Session::Callback.
+     */
+    virtual RPC::Result
+    receive(const Packet &packet);
+
+    virtual void
+    eof(void);
+
+    virtual void
+    inSync(void);
+
+    /**
+     * Implements IPruneRemovedDocumentsHandler
+     */
+    void
+    performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp);
+
+    void
+    syncTls(SerialNum syncTo);
+
+    void
+    storeRemoteOperation(const FeedOperation &op);
+
+    // Implements IOperationStorer
+    virtual void storeOperation(FeedOperation &op);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp
new file mode 100644
index 00000000000..275b8006be2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP(".feedstate");
+#include <vespa/fastos/fastos.h>
+
+#include "feedstate.h"
+#include <vespa/document/bucket/bucketid.h>
+
+using document::BucketId;
+using vespalib::IllegalStateException;
+using vespalib::make_string;
+
+namespace proton {
+
+void FeedState::throwExceptionInReceive(const vespalib::string &docType,
+                                        uint64_t serialRangeFrom,
+                                        uint64_t serialRangeTo,
+                                        size_t packetSize) {
+    throw IllegalStateException
+        (make_string("We should not receive any packets from"
+                     " the transaction log when in '%s' feed state:"
+                     " doctype(%s),"
+                     " packetSerialRange(%" PRIu64 ",%" PRIu64 "),"
+                     " packetSize(%zu)",
+                     getName().c_str(),
+                     docType.c_str(),
+                     serialRangeFrom, serialRangeTo,
+                     packetSize));
+}
+
+void
+FeedState::throwExceptionInHandleOperation(const vespalib::string &docType,
+                                           const FeedOperation &op)
+{
+    throw IllegalStateException
+        (make_string("We should not receive any feed operations"
+                     " when in '%s' feed state:"
+                     " doctype(%s),"
+                     " serial(%" PRIu64 ")",
+                     getName().c_str(),
+                     docType.c_str(),
+                     op.getSerialNum()));
+}
+
+vespalib::string FeedState::getName() const {
+    switch(_type) {
+    case NORMAL:
+        return "NORMAL";
+    case REPLAY_TRANSACTION_LOG:
+        return "REPLAY_TRANSACTION_LOG";
+    case INIT:
+        return "INIT";
+    }
+    return "Unknown";
+}
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.h b/searchcore/src/vespa/searchcore/proton/server/feedstate.h
new file mode 100644
index 00000000000..39eacf206dc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "packetwrapper.h"
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/persistenceengine/resulthandler.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/executor.h>
+
+namespace proton {
+
+
+/**
+ * Class representing the current state of a feed handler.
+ */
+class FeedState {
+public:
+    enum Type { NORMAL,
+                REPLAY_TRANSACTION_LOG,
+                INIT };
+
+private:
+    Type _type;
+
+protected:
+    void throwExceptionInReceive(const vespalib::string &docType,
+                                 uint64_t serialRangeFrom,
+                                 uint64_t serialRangeTo,
+                                 size_t packetSize);
+
+    void throwExceptionInHandleOperation(const vespalib::string &docType,
+                                         const FeedOperation &op);
+
+public:
+    typedef std::shared_ptr<FeedState> SP;
+
+    FeedState(Type type) : _type(type) {}
+    virtual ~FeedState() {}
+
+    Type getType() const { return _type; }
+    vespalib::string getName() const;
+
+    virtual void handleOperation(FeedToken token, FeedOperation::UP op) = 0;
+
+    virtual void receive(const PacketWrapper::SP &wrap,
+                         vespalib::Executor &executor) = 0;
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp
new file mode 100644
index 00000000000..d20fe9f093a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp
@@ -0,0 +1,175 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.feedstates");
+#include <vespa/fastos/fastos.h>
+
+#include "feedstates.h"
+#include "feedconfigstore.h"
+#include "ireplaypackethandler.h"
+#include "replaypacketdispatcher.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h>
+
+using search::transactionlog::Packet;
+using search::transactionlog::RPC;
+using search::SerialNum;
+using vespalib::Executor;
+using vespalib::IllegalStateException;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+using vespalib::make_string;
+using proton::bucketdb::IBucketDBHandler;
+
+namespace proton {
+
+namespace {
+typedef vespalib::Closure1<const Packet::Entry &>::UP EntryHandler;
+
+const search::SerialNum REPLAY_PROGRESS_INTERVAL = 50000;
+
+void
+handleProgress(TlsReplayProgress &progress, SerialNum currentSerial)
+{
+    progress.updateCurrent(currentSerial);
+    if (LOG_WOULD_LOG(event) && (LOG_WOULD_LOG(debug) ||
+            (progress.getCurrent() % REPLAY_PROGRESS_INTERVAL == 0)))
+    {
+        EventLogger::transactionLogReplayProgress(progress.getDomainName(),
+                                                  progress.getProgress(),
+                                                  progress.getFirst(),
+                                                  progress.getLast(),
+                                                  progress.getCurrent());
+    }
+}
+
+void
+handlePacket(PacketWrapper::SP wrap, EntryHandler entryHandler)
+{
+    vespalib::nbostream handle(wrap->packet.getHandle().c_str(),
+                               wrap->packet.getHandle().size(),
+                               true);
+    while (handle.size() > 0) {
+        Packet::Entry entry;
+        entry.deserialize(handle);
+        entryHandler->call(entry);
+        if (wrap->progress != NULL) {
+            handleProgress(*wrap->progress, entry.serial());
+        }
+    }
+    wrap->result = RPC::OK;
+    wrap->gate.countDown();
+}
+
+class TransactionLogReplayPacketHandler : public IReplayPacketHandler {
+    IFeedView *& _feed_view_ptr;  // Pointer can be changed in executor thread.
+    IBucketDBHandler &_bucketDBHandler;
+    IReplayConfig &_replay_config;
+    FeedConfigStore &_config_store;
+
+    void handleTransactionLogEntry(const Packet::Entry &entry);
+
+public:
+    TransactionLogReplayPacketHandler(IFeedView *& feed_view_ptr,
+                                      IBucketDBHandler &bucketDBHandler,
+                                      IReplayConfig &replay_config,
+                                      FeedConfigStore &config_store)
+        : _feed_view_ptr(feed_view_ptr),
+          _bucketDBHandler(bucketDBHandler),
+          _replay_config(replay_config),
+          _config_store(config_store) {
+    }
+
+    virtual void replay(const PutOperation &op) {
+        _feed_view_ptr->handlePut(NULL, op);
+    }
+    virtual void replay(const RemoveOperation &op) {
+        _feed_view_ptr->handleRemove(NULL, op);
+    }
+    virtual void replay(const UpdateOperation &op) {
+        _feed_view_ptr->handleUpdate(NULL, op);
+    }
+    virtual void replay(const NoopOperation &) {} // ignored
+    virtual void replay(const NewConfigOperation &op) {
+        _replay_config.replayConfig(op.getSerialNum());
+    }
+    virtual void replay(const WipeHistoryOperation &op) {
+        _config_store.saveWipeHistoryConfig(op.getSerialNum(),
+                                            op.getWipeTimeLimit());
+        _replay_config.replayWipeHistory(op.getSerialNum(),
+                                         op.getWipeTimeLimit());
+    }
+    virtual void replay(const DeleteBucketOperation &op) {
+        _feed_view_ptr->handleDeleteBucket(op);
+    }
+    virtual void replay(const SplitBucketOperation &op) {
+        _bucketDBHandler.handleSplit(op.getSerialNum(),
+                                     op.getSource(),
+                                     op.getTarget1(),
+                                     op.getTarget2());
+    }
+    virtual void replay(const JoinBucketsOperation &op) {
+        _bucketDBHandler.handleJoin(op.getSerialNum(),
+                                    op.getSource1(),
+                                    op.getSource2(),
+                                    op.getTarget());
+    }
+    virtual void replay(const PruneRemovedDocumentsOperation &op) {
+        _feed_view_ptr->handlePruneRemovedDocuments(op);
+    }
+    virtual void replay(const SpoolerReplayStartOperation &op) {
+        (void) op;
+    }
+    virtual void replay(const SpoolerReplayCompleteOperation &op) {
+        (void) op;
+    }
+    virtual void replay(const MoveOperation &op) {
+        _feed_view_ptr->handleMove(op);
+    }
+    virtual void replay(const CreateBucketOperation &) {
+    }
+    virtual void replay(const CompactLidSpaceOperation &op) {
+        _feed_view_ptr->handleCompactLidSpace(op);
+    }
+    virtual NewConfigOperation::IStreamHandler &getNewConfigStreamHandler() {
+        return _config_store;
+    }
+    virtual document::DocumentTypeRepo &getDeserializeRepo() {
+        return *_feed_view_ptr->getDocumentTypeRepo();
+    }
+};
+
+void startDispatch(IReplayPacketHandler *packet_handler,
+                   const Packet::Entry &entry) {
+    // Called by handlePacket() in executor thread.
+    LOG(spam,
+        "replay packet entry: entrySerial(%" PRIu64 "), entryType(%u)",
+        entry.serial(), entry.type());
+
+    ReplayPacketDispatcher dispatcher(*packet_handler);
+    dispatcher.replayEntry(entry);
+}
+
+}  // namespace
+
+ReplayTransactionLogState::ReplayTransactionLogState(
+        const vespalib::string &name,
+        IFeedView *& feed_view_ptr,
+        IBucketDBHandler &bucketDBHandler,
+        IReplayConfig &replay_config,
+        FeedConfigStore &config_store)
+    : FeedState(REPLAY_TRANSACTION_LOG),
+      _doc_type_name(name),
+      _packet_handler(new TransactionLogReplayPacketHandler(
+                      feed_view_ptr, bucketDBHandler,
+                      replay_config, config_store)) {
+}
+
+void ReplayTransactionLogState::receive(const PacketWrapper::SP &wrap,
+                                        Executor &executor) {
+    EntryHandler closure = makeClosure(&startDispatch, _packet_handler.get());
+    executor.execute(makeTask(makeClosure(&handlePacket, wrap, std::move(closure))));
+}
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.h b/searchcore/src/vespa/searchcore/proton/server/feedstates.h
new file mode 100644
index 00000000000..1bc88627377
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.h
@@ -0,0 +1,97 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/server/feedhandler.h>
+#include <vespa/searchcore/proton/server/feedstate.h>
+#include <vespa/searchcore/proton/server/ireplaypackethandler.h>
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+
+namespace proton {
+
+/**
+ * There are 3 feed states, and one paths through the state machine:
+ *
+ *  INIT -> REPLAY_TRANSACTION_LOG -> NORMAL.
+ */
+
+
+/**
+ * The feed handler owner is initializing components.
+ */
+class InitState : public FeedState {
+    vespalib::string _doc_type_name;
+
+public:
+    InitState(const vespalib::string &name)
+        : FeedState(INIT),
+          _doc_type_name(name)
+    {
+    }
+
+    virtual void handleOperation(FeedToken, FeedOperation::UP op) {
+        throwExceptionInHandleOperation(_doc_type_name, *op);
+    }
+
+    virtual void
+    receive(const PacketWrapper::SP &wrap, vespalib::Executor &) {
+        throwExceptionInReceive(_doc_type_name.c_str(),
+                                wrap->packet.range().from(),
+                                wrap->packet.range().to(),
+                                wrap->packet.size());
+    }
+};
+
+
+/**
+ * The feed handler is replaying the transaction log.
+ * Replayed messages from the transaction log are sent to the active feed view.
+ */
+class ReplayTransactionLogState : public FeedState {
+    vespalib::string _doc_type_name;
+    std::unique_ptr<IReplayPacketHandler> _packet_handler;
+
+public:
+    ReplayTransactionLogState(const vespalib::string &name,
+            IFeedView *& feed_view_ptr,
+            bucketdb::IBucketDBHandler &bucketDBHandler,
+            IReplayConfig &replay_config,
+            FeedConfigStore &config_store);
+
+    virtual void handleOperation(FeedToken, FeedOperation::UP op) {
+        throwExceptionInHandleOperation(_doc_type_name, *op);
+    }
+
+    virtual void receive(const PacketWrapper::SP &wrap,
+                         vespalib::Executor &executor);
+};
+
+
+/**
+ * Normal feed state.
+ * Incoming messages are sent to the active feed view.
+ */
+class NormalState : public FeedState {
+    FeedHandler         &_handler;
+
+public:
+    NormalState(FeedHandler &handler)
+        : FeedState(NORMAL),
+          _handler(handler) {
+    }
+
+    virtual void handleOperation(FeedToken token, FeedOperation::UP op) {
+        _handler.performOperation(FeedToken::UP(new FeedToken(token)), std::move(op));
+    }
+
+    virtual void
+    receive(const PacketWrapper::SP &wrap, vespalib::Executor &)
+    {
+        throwExceptionInReceive(_handler.getDocTypeName().c_str(),
+                                wrap->packet.range().from(),
+                                wrap->packet.range().to(),
+                                wrap->packet.size());
+    }
+};
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
new file mode 100644
index 00000000000..8b5a84d3f9e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
@@ -0,0 +1,594 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "fileconfigmanager.h"
+#include <vespa/config/print/fileconfigwriter.h>
+#include <vespa/config/print/fileconfigsnapshotreader.h>
+#include <vespa/config/print/fileconfigsnapshotwriter.h>
+#include <vespa/config/print/configformatter.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/log/log.h>
+#include <vespa/searchcommon/common/schemaconfigurer.h>
+#include <vespa/searchlib/index/schemautil.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <utime.h>
+#include <fstream>
+LOG_SETUP(".proton.server.fileconfigmanager");
+
+using document::DocumentTypeRepo;
+using document::DocumenttypesConfig;
+using search::IndexMetaInfo;
+using search::SerialNum;
+using search::index::Schema;
+using vespa::config::search::AttributesConfig;
+using vespa::config::search::IndexschemaConfig;
+using vespa::config::search::RankProfilesConfig;
+using vespa::config::search::SummaryConfig;
+using vespa::config::search::SummarymapConfig;
+using vespa::config::search::core::ProtonConfig;
+using vespa::config::search::summary::JuniperrcConfig;
+using vespalib::nbostream;
+
+typedef IndexMetaInfo::SnapshotList SnapshotList;
+
+namespace proton
+{
+
+namespace
+{
+
+vespalib::string
+makeSnapDirBaseName(SerialNum serialNum)
+{
+    std::ostringstream os;
+    os << "config-" << serialNum;
+    return os.str();
+}
+
+vespalib::string
+makeExtraConfigsFileName(const vespalib::string &snapDir)
+{
+    return snapDir + "/extraconfigs.dat";
+}
+
+
+void
+fsyncFile(const vespalib::string &fileName)
+{
+    FastOS_File f;
+    f.OpenReadWrite(fileName.c_str());
+    if (!f.IsOpened()) {
+        LOG(error,
+            "Could not open file '%s' for fsync",
+            fileName.c_str());
+        return;
+    }
+    if (!f.Sync()) {
+        LOG(error,
+            "Could not fsync file '%s'",
+            fileName.c_str());
+    }
+    f.Close();
+}
+
+template <class Config>
+void
+saveHelper(const vespalib::string &snapDir,
+           const vespalib::string &name,
+           const Config &config)
+{
+    vespalib::string fileName(snapDir + "/" + name + ".cfg");
+    config::FileConfigWriter writer(fileName);
+    bool ok = writer.write(config);
+    assert(ok);
+    (void) ok;
+    fsyncFile(fileName);
+}
+
+template <class Config>
+void
+save(const vespalib::string &snapDir,
+     const Config &config)
+{
+    saveHelper(snapDir, config.defName(), config);
+}
+
+void writeExtraConfigs(const vespalib::string &snapDir,
+                       const DocumentDBConfig &snapshot)
+{
+    vespalib::string extraName(makeExtraConfigsFileName(snapDir));
+    config::FileConfigSnapshotWriter writer(extraName);
+    bool extraConfigsWriterResult = writer.write(snapshot.getExtraConfigs());
+    assert(extraConfigsWriterResult);
+    (void) extraConfigsWriterResult;
+    fsyncFile(extraName);
+}
+
+config::ConfigSnapshot
+readExtraConfigs(const vespalib::string &snapDir)
+{
+    vespalib::string fileName = makeExtraConfigsFileName(snapDir);
+    if (vespalib::fileExists(fileName)) {
+        config::FileConfigSnapshotReader reader(fileName);
+        return reader.read();
+    } else {
+        LOG(warning, "Did not find data file for extra configs '%s' during loading of config snapshot. "
+            "Using empty extra configs set.", fileName.c_str());
+    }
+    return config::ConfigSnapshot();
+}
+
+
+class ConfigFile
+{
+    typedef std::shared_ptr<ConfigFile> SP;
+
+    vespalib::string _name;
+    time_t _modTime;
+    std::vector<char> _content;
+
+public:
+    ConfigFile(void);
+
+    ConfigFile(const vespalib::string &name,
+               const vespalib::string &fullName);
+
+    nbostream &
+    serialize(nbostream &stream) const;
+
+    nbostream &
+    deserialize(nbostream &stream);
+
+    void
+    save(const vespalib::string &snapDir) const;
+};
+
+
+ConfigFile::ConfigFile(void)
+    : _name(),
+      _modTime(0),
+      _content()
+{
+}
+
+
+ConfigFile::ConfigFile(const vespalib::string &name,
+                       const vespalib::string &fullName)
+    : _name(name),
+      _modTime(0),
+      _content()
+{
+    FastOS_File file;
+    bool openRes = file.OpenReadOnlyExisting(false, fullName.c_str());
+    if (!openRes)
+        return;
+    int64_t fileSize = file.getSize();
+    _content.resize(fileSize);
+    file.ReadBuf(&_content[0], fileSize);
+    _modTime = file.GetModificationTime();
+    file.Close();
+}
+
+
+nbostream &
+ConfigFile::serialize(nbostream &stream) const
+{
+    assert(strchr(_name.c_str(), '/') == NULL);
+    stream << _name;
+    stream << _modTime;
+    uint32_t sz = _content.size();
+    stream << sz;
+    stream.write(&_content[0], sz);
+    return stream;
+}
+
+
+nbostream &
+ConfigFile::deserialize(nbostream &stream)
+{
+    stream >> _name;
+    assert(strchr(_name.c_str(), '/') == NULL);
+    stream >> _modTime;
+    uint32_t sz;
+    stream >> sz;
+    _content.resize(sz);
+    assert(stream.size() >= sz);
+    memcpy(&_content[0], stream.peek(), sz);
+    stream.adjustReadPos(sz);
+    return stream;
+}
+
+
+void
+ConfigFile::save(const vespalib::string &snapDir) const
+{
+    vespalib::string fullName = snapDir + "/" + _name;
+    FastOS_File file;
+    bool openRes = file.OpenWriteOnlyTruncate(fullName.c_str());
+    assert(openRes);
+    (void) openRes;
+
+    file.WriteBuf(&_content[0], _content.size());
+    bool closeRes = file.Close();
+    assert(closeRes);
+    (void) closeRes;
+
+    fsyncFile(fullName);
+}
+
+
+nbostream &
+operator<<(nbostream &stream, const ConfigFile &configFile)
+{
+    return configFile.serialize(stream);
+}
+
+
+nbostream &
+operator>>(nbostream &stream, ConfigFile &configFile)
+{
+    return configFile.deserialize(stream);
+}
+
+
+std::vector<vespalib::string>
+getFileList(const vespalib::string &snapDir)
+{
+    std::vector<vespalib::string> res;
+    FastOS_DirectoryScan dirScan(snapDir.c_str());
+    while (dirScan.ReadNext()) {
+        if (strcmp(dirScan.GetName(), ".") == 0 ||
+            strcmp(dirScan.GetName(), "..") == 0)
+            continue;
+        res.push_back(dirScan.GetName());
+    }
+    std::sort(res.begin(), res.end());
+    return res;
+}
+
+
+}
+
+FileConfigManager::FileConfigManager(const vespalib::string &baseDir,
+                                     const vespalib::string &configId,
+                                     const vespalib::string &docTypeName)
+    : _baseDir(baseDir),
+      _configId(configId),
+      _docTypeName(docTypeName),
+      _info(baseDir),
+      _protonConfig()
+{
+    vespalib::mkdir(baseDir, false);
+    if (!_info.load())
+        _info.save();
+    removeInvalid();
+    _protonConfig.reset(new ProtonConfig());
+}
+
+
+FileConfigManager::~FileConfigManager(void)
+{
+}
+
+
+SerialNum
+FileConfigManager::getBestSerialNum(void) const
+{
+    Snapshot snap = _info.getBestSnapshot();
+    return snap.valid ? snap.syncToken : UINT64_C(0);
+}
+
+
+SerialNum
+FileConfigManager::getOldestSerialNum(void) const
+{
+    SerialNum res = 0;
+    const SnapshotList &snaps = _info.snapshots();
+    for (const auto &snap : snaps) {
+        if (!snap.valid || snap.syncToken == 0)
+            continue;
+        if (res == 0 || res > snap.syncToken)
+            res = snap.syncToken;
+    }
+    return res;
+}
+
+
+void
+FileConfigManager::saveConfig(const DocumentDBConfig &snapshot,
+                              const search::index::Schema &historySchema,
+                              SerialNum serialNum)
+{
+    if (getBestSerialNum() >= serialNum) {
+        LOG(warning, "Config for serial >= %" PRIu64 " already saved",
+            static_cast<uint64_t>(serialNum));
+        return;
+    }
+    vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum));
+    vespalib::string snapDir(_baseDir + "/" + snapDirBaseName);
+    Snapshot snap(false, serialNum, snapDirBaseName);
+    _info.addSnapshot(snap);
+    bool saveInvalidSnap = _info.save();
+    assert(saveInvalidSnap);
+    (void) saveInvalidSnap;
+    vespalib::mkdir(snapDir, false);
+    save(snapDir, snapshot.getRankProfilesConfig());
+    save(snapDir, snapshot.getIndexschemaConfig());
+    save(snapDir, snapshot.getAttributesConfig());
+    save(snapDir, snapshot.getSummaryConfig());
+    save(snapDir, snapshot.getSummarymapConfig());
+    save(snapDir, snapshot.getJuniperrcConfig());
+    save(snapDir, snapshot.getDocumenttypesConfig());
+
+    bool saveSchemaRes = snapshot.getSchemaSP()->saveToFile(snapDir + "/schema.txt");
+    assert(saveSchemaRes);
+    (void) saveSchemaRes;
+
+    bool saveHistorySchemaRes = historySchema.saveToFile(snapDir + "/historyschema.txt");
+    assert(saveHistorySchemaRes);
+    (void) saveHistorySchemaRes;
+
+    writeExtraConfigs(snapDir, snapshot);
+    _info.validateSnapshot(serialNum);
+
+    bool saveValidSnap = _info.save();
+    assert(saveValidSnap);
+    (void) saveValidSnap;
+}
+
+
+void
+FileConfigManager::loadConfig(const DocumentDBConfig &currentSnapshot,
+                              search::SerialNum serialNum,
+                              DocumentDBConfig::SP &loadedSnapshot,
+                              search::index::Schema::SP &historySchema)
+{
+    vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum));
+    vespalib::string snapDir(_baseDir + "/" + snapDirBaseName);
+    config::DirSpec spec(snapDir);
+
+    DocumentDBConfigHelper dbc(spec, _docTypeName);
+
+    typedef DocumenttypesConfig DTC;
+    typedef DocumentDBConfig::DocumenttypesConfigSP DTCSP;
+    DTCSP docTypesCfg(config::ConfigGetter<DTC>::getConfig("", spec).release());
+    DocumentTypeRepo::SP repo;
+    if (currentSnapshot.getDocumenttypesConfigSP().get() != NULL &&
+        currentSnapshot.getDocumentTypeRepoSP().get() != NULL &&
+        currentSnapshot.getDocumenttypesConfig() == *docTypesCfg) {
+        docTypesCfg = currentSnapshot.getDocumenttypesConfigSP();
+        repo = currentSnapshot.getDocumentTypeRepoSP();
+    } else {
+        repo.reset(new DocumentTypeRepo(*docTypesCfg));
+    }
+
+    /*
+     * XXX: If non-default maintenance config is used then an extra config
+     * snapshot is saved after replaying transaction log due to the use
+     * of default values here instead of the current values from the config
+     * server.
+     */
+    BootstrapConfig::SP bootstrap(
+            new BootstrapConfig(1,
+                                docTypesCfg,
+                                repo,
+                                _protonConfig,
+                                currentSnapshot.getTuneFileDocumentDBSP()));
+    dbc.forwardConfig(bootstrap);
+    dbc.nextGeneration(0);
+
+    Schema::UP newHistorySchema(new Schema);
+    bool loadHistorySchemaRes = newHistorySchema->loadFromFile(snapDir + "/historyschema.txt");
+    assert(loadHistorySchemaRes);
+    (void) loadHistorySchemaRes;
+
+    loadedSnapshot = dbc.getConfig();
+    loadedSnapshot->setConfigId(_configId);
+    loadedSnapshot->setExtraConfigs(readExtraConfigs(snapDir));
+    historySchema.reset(newHistorySchema.release());
+}
+
+
+void
+FileConfigManager::removeInvalid(void)
+{
+    typedef std::vector<SerialNum> RemVec;
+    RemVec toRem;
+
+    const SnapshotList &snaps = _info.snapshots();
+    for (const auto &snap : snaps) {
+        if (!snap.valid)
+            toRem.push_back(snap.syncToken);
+    }
+    if (toRem.empty())
+        return;
+
+    for (const auto &serial : toRem) {
+        vespalib::string snapDirBaseName(makeSnapDirBaseName(serial));
+        vespalib::string snapDir(_baseDir + "/" + snapDirBaseName);
+        try {
+            FastOS_FileInterface::EmptyAndRemoveDirectory(snapDir.c_str());
+        } catch (const std::exception &e) {
+            LOG(warning, "Removing obsolete config directory '%s' failed due to %s", snapDir.c_str(), e.what());
+        }
+    }
+    for (const auto &serial : toRem) {
+        _info.removeSnapshot(serial);
+    }
+    bool saveRemInvalidSnap = _info.save();
+    assert(saveRemInvalidSnap);
+    (void) saveRemInvalidSnap;
+}
+
+
+void
+FileConfigManager::prune(SerialNum serialNum)
+{
+    typedef std::vector<SerialNum> PruneVec;
+    PruneVec toPrune;
+
+    const SnapshotList &snaps = _info.snapshots();
+    for (const auto &snap : snaps) {
+        if (snap.valid && snap.syncToken <= serialNum)
+            toPrune.push_back(snap.syncToken);
+    }
+    std::sort(toPrune.begin(), toPrune.end());
+    if (!toPrune.empty())
+        toPrune.pop_back();	// Keep newest old entry
+    if (toPrune.empty())
+        return;
+    for (const auto &serial : toPrune) {
+        _info.invalidateSnapshot(serial);
+    }
+    bool saveInvalidSnap = _info.save();
+    assert(saveInvalidSnap);
+    (void) saveInvalidSnap;
+    removeInvalid();
+}
+
+
+bool
+FileConfigManager::hasValidSerial(SerialNum serialNum) const
+{
+    IndexMetaInfo::Snapshot snap = _info.getSnapshot(serialNum);
+    return snap.valid;
+}
+
+
+SerialNum
+FileConfigManager::getPrevValidSerial(SerialNum serialNum) const
+{
+    SerialNum res = 0;
+    const SnapshotList &snaps = _info.snapshots();
+    for (const auto &snap : snaps) {
+        if (!snap.valid || snap.syncToken >= serialNum)
+            continue;
+        if (res < snap.syncToken)
+            res = snap.syncToken;
+    }
+    return res;
+}
+
+
+void
+FileConfigManager::saveWipeHistoryConfig(SerialNum serialNum,
+                                         fastos::TimeStamp wipeTimeLimit)
+{
+    vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum));
+    vespalib::string snapDir(_baseDir + "/" + snapDirBaseName);
+
+    if (hasValidSerial(serialNum))
+        return;	// config already saved.
+
+    SerialNum prevSerialNum = getPrevValidSerial(serialNum);
+
+    assert(prevSerialNum > 0);
+
+    Snapshot snap(false, serialNum, snapDirBaseName);
+    _info.addSnapshot(snap);
+
+    bool saveInvalidSnap = _info.save();
+    assert(saveInvalidSnap);
+    (void) saveInvalidSnap;
+
+    vespalib::mkdir(snapDir, false);
+
+    vespalib::string prevSnapDirBaseName(makeSnapDirBaseName(prevSerialNum));
+    vespalib::string prevSnapDir(_baseDir + "/" + prevSnapDirBaseName);
+
+    std::vector<vespalib::string> configs = getFileList(prevSnapDir);
+    for (const auto &config : configs) {
+        if (config == "historyschema.txt") {
+            Schema::UP historySchema(new Schema);
+            if (wipeTimeLimit != 0) {
+                Schema oldHistorySchema;
+                bool loadOldHistorySchemaRes =
+                    oldHistorySchema.loadFromFile(prevSnapDir +
+                                                   "/historyschema.txt");
+                assert(loadOldHistorySchemaRes);
+                (void) loadOldHistorySchemaRes;
+                Schema::UP wipeSchema;
+                wipeSchema = oldHistorySchema.getOldFields(wipeTimeLimit);
+                historySchema = Schema::set_difference(oldHistorySchema,
+                                                       *wipeSchema);
+            }
+            bool saveHistorySchemaRes =
+                historySchema->saveToFile(snapDir + "/historyschema.txt");
+            assert(saveHistorySchemaRes);
+            (void) saveHistorySchemaRes;
+            continue;
+        }
+        ConfigFile file(config,
+                        prevSnapDir + "/" + config);
+        file.save(snapDir);
+    }
+    _info.validateSnapshot(serialNum);
+    bool saveValidSnap = _info.save();
+    assert(saveValidSnap);
+    (void) saveValidSnap;
+}
+
+
+void
+FileConfigManager::serializeConfig(SerialNum serialNum, nbostream &stream)
+{
+    vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum));
+    vespalib::string snapDir(_baseDir + "/" + snapDirBaseName);
+
+    assert(hasValidSerial(serialNum));
+
+    std::vector<vespalib::string> configs = getFileList(snapDir);
+    uint32_t numConfigs = configs.size();
+    stream << numConfigs;
+    for (const auto &config : configs) {
+        ConfigFile file(config,
+                        snapDir + "/" + config);
+        stream << file;
+    }
+}
+
+
+void
+FileConfigManager::deserializeConfig(SerialNum serialNum, nbostream &stream)
+{
+    vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum));
+    vespalib::string snapDir(_baseDir + "/" + snapDirBaseName);
+
+    bool skip = hasValidSerial(serialNum);
+
+    Snapshot snap(false, serialNum, snapDirBaseName);
+    if (!skip) {
+        _info.addSnapshot(snap);
+        bool saveInvalidSnap = _info.save();
+        assert(saveInvalidSnap);
+        (void) saveInvalidSnap;
+        vespalib::mkdir(snapDir, false);
+    }
+
+    uint32_t numConfigs;
+    stream >> numConfigs;
+    for (uint32_t i = 0; i < numConfigs; ++i) {
+        ConfigFile file;
+        stream >> file;
+        if (!skip)
+            file.save(snapDir);
+    }
+    assert(stream.size() == 0);
+    if (!skip) {
+        _info.validateSnapshot(serialNum);
+        bool saveValidSnap = _info.save();
+        assert(saveValidSnap);
+        (void) saveValidSnap;
+    }
+}
+
+
+void
+FileConfigManager::setProtonConfig(const ProtonConfigSP &protonConfig)
+{
+    _protonConfig = protonConfig;
+}
+
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h
new file mode 100644
index 00000000000..8998098dd90
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "configstore.h"
+#include "documentdbconfigmanager.h"
+#include <vespa/searchlib/common/indexmetainfo.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+namespace proton {
+
+class FileConfigManager : public ConfigStore {
+public:
+    typedef std::unique_ptr<FileConfigManager> UP;
+    typedef std::shared_ptr<FileConfigManager> SP;
+    typedef search::IndexMetaInfo::Snapshot Snapshot;
+
+private:
+    vespalib::string _baseDir;
+    vespalib::string _configId;
+    vespalib::string _docTypeName;
+    search::IndexMetaInfo _info;
+    ProtonConfigSP _protonConfig;
+
+public:
+    /**
+     * Creates a new file config manager.
+     *
+     * @param baseDir the directory in which config snapshots are saved and loaded.
+     * @param configId the configId that was used to subscribe to config that is later handled by this manager.
+     */
+    FileConfigManager(const vespalib::string &baseDir,
+                      const vespalib::string &configId,
+                      const vespalib::string &docTypeName);
+
+    virtual
+    ~FileConfigManager(void);
+
+    virtual SerialNum getBestSerialNum() const;
+    virtual SerialNum getOldestSerialNum() const;
+
+    virtual void saveConfig(const DocumentDBConfig &snapshot,
+                            const search::index::Schema &historySchema,
+                            SerialNum serialNum);
+
+    /**
+     * Load a config snapshot from disk corresponding to the given
+     * serial number.  The config id of this manager is set on the
+     * loaded config snapshot.
+     *
+     * @param currentSnapshot the current snapshot, for reusing
+     *                        unchanged parts .
+     * @param serialNum the serial number of the config snapshot to load.
+     * @param loadedSnapshot the shared pointer in which to store the
+     *                       resulting config snapshot.
+     * @param historySchema the shared pointer in which to store the
+     *                      resulting history schema.
+     */
+    virtual void loadConfig(const DocumentDBConfig &currentSnapshot,
+                            SerialNum serialNum,
+                            DocumentDBConfig::SP &loadedSnapshot,
+                            search::index::Schema::SP &historySchema);
+
+    virtual void removeInvalid();
+    virtual void prune(SerialNum serialNum);
+    virtual bool hasValidSerial(SerialNum serialNum) const;
+
+    virtual SerialNum getPrevValidSerial(SerialNum serialNum) const;
+
+    /**
+     * Clone config except for history schema.
+     * Used when wiping history.
+     */
+    virtual void saveWipeHistoryConfig(SerialNum serialNum,
+                                       fastos::TimeStamp wipeTimeLimit);
+
+
+    /**
+     * Serialize config files.
+     *
+     * Used for serializing config into transaction log.
+     */
+    virtual void
+    serializeConfig(SerialNum serialNum, vespalib::nbostream &stream);
+
+
+    /**
+     * Deserialize config files.
+     *
+     * Used for deserializing config from transaction log when it is
+     * not already present on disk.  Config files on disk
+     * takes precedence over the serialized config files in the
+     * transaction log.
+     */
+    virtual void
+    deserializeConfig(SerialNum serialNum, vespalib::nbostream &stream);
+
+    virtual void setProtonConfig(const ProtonConfigSP &protonConfig) override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp
new file mode 100644
index 00000000000..7736161ddbc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.flushhandlerproxy");
+#include "flushhandlerproxy.h"
+
+namespace proton {
+
+FlushHandlerProxy::FlushHandlerProxy(const DocumentDB::SP &documentDB)
+    : IFlushHandler(documentDB->getDocTypeName().toString()),
+      _documentDB(documentDB)
+{
+    _documentDB->retain();
+}
+
+
+FlushHandlerProxy::~FlushHandlerProxy(void)
+{
+    _documentDB->release();
+}
+
+
+std::vector<IFlushTarget::SP>
+FlushHandlerProxy::getFlushTargets(void)
+{
+    return _documentDB->getFlushTargets();
+}
+
+
+IFlushHandler::SerialNum
+FlushHandlerProxy::getCurrentSerialNumber(void) const
+{
+    return _documentDB->getCurrentSerialNumber();
+}
+
+
+void
+FlushHandlerProxy::flushDone(SerialNum oldestSerial)
+{
+    _documentDB->flushDone(oldestSerial);
+}
+
+
+void
+FlushHandlerProxy::syncTls(SerialNum syncTo)
+{
+    _documentDB->sync(syncTo);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h
new file mode 100644
index 00000000000..7e4cd1f3176
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/flushengine/iflushhandler.h>
+#include "documentdb.h"
+
+namespace proton {
+
+class FlushHandlerProxy : public boost::noncopyable,
+                          public IFlushHandler
+{
+private:
+    DocumentDB::SP _documentDB;
+public:
+    FlushHandlerProxy(const DocumentDB::SP &documentDB);
+
+    virtual
+    ~FlushHandlerProxy(void);
+
+    /**
+     * Implements IFlushHandler.
+     */
+    virtual std::vector<IFlushTarget::SP>
+    getFlushTargets(void);
+
+    virtual SerialNum
+    getCurrentSerialNumber(void) const;
+
+    virtual void
+    flushDone(SerialNum oldestSerial);
+
+    virtual void
+    syncTls(SerialNum syncTo);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp
new file mode 100644
index 00000000000..d382a76faeb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "forcecommitcontext.h"
+#include "forcecommitdonetask.h"
+#include <vespa/searchcore/proton/common/docid_limit.h>
+
+namespace proton
+{
+
+
+ForceCommitContext::ForceCommitContext(vespalib::Executor &executor,
+                                       IDocumentMetaStore &documentMetaStore)
+    : _executor(executor),
+      _task(std::make_unique<ForceCommitDoneTask>(documentMetaStore)),
+      _committedDocIdLimit(0u),
+      _docIdLimit(nullptr)
+{
+}
+
+
+ForceCommitContext::~ForceCommitContext()
+{
+    if (_docIdLimit != nullptr) {
+        _docIdLimit->bumpUpLimit(_committedDocIdLimit);
+    }
+    if (!_task->empty()) {
+        vespalib::Executor::Task::UP res = _executor.execute(std::move(_task));
+        assert(!res);
+    }
+}
+
+
+void
+ForceCommitContext::reuseLids(std::vector<uint32_t> &&lids)
+{
+    _task->reuseLids(std::move(lids));
+}
+
+
+void
+ForceCommitContext::holdUnblockShrinkLidSpace()
+{
+    _task->holdUnblockShrinkLidSpace();
+}
+
+
+void
+ForceCommitContext::registerCommittedDocIdLimit(uint32_t committedDocIdLimit,
+                                                DocIdLimit *docIdLimit)
+{
+    _committedDocIdLimit = committedDocIdLimit;
+    _docIdLimit = docIdLimit;
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h
new file mode 100644
index 00000000000..ecf50f1402a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/idestructorcallback.h>
+
+namespace vespalib
+{
+
+class Executor;
+
+}
+
+
+namespace proton
+{
+
+class ForceCommitDoneTask;
+class IDocumentMetaStore;
+class DocIdLimit;
+
+/**
+ * Context class for forced commits that schedules a task when
+ * instance is destroyed. Typically a shared pointer to an instance is
+ * passed around to multiple worker threads that performs portions of
+ * a larger task before dropping the shared pointer, triggering the
+ * callback when all worker threads have completed.
+ */
+class ForceCommitContext : public search::IDestructorCallback
+{
+    vespalib::Executor &_executor;
+    std::unique_ptr<ForceCommitDoneTask> _task;
+    uint32_t _committedDocIdLimit;
+    DocIdLimit *_docIdLimit;
+
+public:
+    ForceCommitContext(vespalib::Executor &executor,
+                       IDocumentMetaStore &documentMetaStore);
+
+    virtual ~ForceCommitContext();
+
+    void reuseLids(std::vector<uint32_t> &&lids);
+
+    void holdUnblockShrinkLidSpace();
+
+    void registerCommittedDocIdLimit(uint32_t committedDocIdLimit,
+                                     DocIdLimit *docIdLimit);
+};
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp
new file mode 100644
index 00000000000..f78eed0164e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "forcecommitdonetask.h"
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+
+namespace proton
+{
+
+
+ForceCommitDoneTask::ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore)
+    : _lidsToReuse(),
+      _holdUnblockShrinkLidSpace(false),
+      _documentMetaStore(documentMetaStore)
+{
+}
+
+
+ForceCommitDoneTask::~ForceCommitDoneTask()
+{
+}
+
+
+void
+ForceCommitDoneTask::reuseLids(std::vector<uint32_t> &&lids)
+{
+    assert(_lidsToReuse.empty());
+    _lidsToReuse = std::move(lids);
+}
+
+
+void
+ForceCommitDoneTask::run()
+{
+    if (!_lidsToReuse.empty()) {
+        if (_lidsToReuse.size() == 1) {
+            _documentMetaStore.removeComplete(_lidsToReuse[0]);
+        } else {
+            _documentMetaStore.removeBatchComplete(_lidsToReuse);
+        }
+    }
+    if (_holdUnblockShrinkLidSpace) {
+        _documentMetaStore.holdUnblockShrinkLidSpace();
+    }
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h
new file mode 100644
index 00000000000..73cb2ee1b47
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/executor.h>
+
+namespace proton
+{
+
+class IDocumentMetaStore;
+
+/**
+ * Class for task to be executed when a forced commit has completed and
+ * memory index and attributes have been updated.
+ *
+ * The task handles two things:
+ *
+ * 1.  Passing on lids that can be reused do document meta store.
+ *     They have to go through a hold cycle in order for searches that
+ *     might have posting lists referencing the lids in context of
+ *     their old identity.
+ *
+ * 2.  Shrinking of document meta store lid space.  This also goes through
+ *     a hold cycle, since it must be handled after any lids to be reused.
+ */
+class ForceCommitDoneTask : public vespalib::Executor::Task
+{
+    std::vector<uint32_t> _lidsToReuse;
+    bool _holdUnblockShrinkLidSpace;
+    IDocumentMetaStore &_documentMetaStore;
+
+public:
+    ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore);
+
+    virtual ~ForceCommitDoneTask();
+
+    void reuseLids(std::vector<uint32_t> &&lids);
+
+    void holdUnblockShrinkLidSpace() {
+        _holdUnblockShrinkLidSpace = true;
+    }
+
+    virtual void run() override;
+
+    bool empty() const {
+        return _lidsToReuse.empty() && !_holdUnblockShrinkLidSpace;
+    }
+};
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/frozenbuckets.cpp b/searchcore/src/vespa/searchcore/proton/server/frozenbuckets.cpp
new file mode 100644
index 00000000000..096cd802185
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/frozenbuckets.cpp
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.frozenbuckets");
+
+#include <vespa/searchcorespi/index/i_thread_service.h>
+#include "frozenbuckets.h"
+#include "ibucketfreezelistener.h"
+#include <vespa/vespalib/util/closuretask.h>
+#include <algorithm>
+
+using searchcorespi::index::IThreadService;
+using document::BucketId;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+using vespalib::MonitorGuard;
+
+namespace proton
+{
+
+FrozenBucketsMap::FrozenBucketsMap() :
+    _lock(),
+    _map()
+{ }
+
+FrozenBucketsMap::~FrozenBucketsMap() {
+    assert(_map.empty());
+}
+
+void
+FrozenBucketsMap::freezeBucket(BucketId bucket) {
+
+    MonitorGuard guard(_lock);
+    std::pair<BucketId, FrozenBucket> tryVal(std::make_pair(bucket, FrozenBucket(FrozenBucket::Reader)));
+
+    std::pair<Map::iterator, bool> res;
+    for (res = _map.insert(tryVal); !res.second && (res.first->second.isExclusive()); res = _map.insert(tryVal)) {
+        guard.wait();
+    }
+
+    if (!res.second) {
+        res.first->second.addReader();
+    }
+}
+
+
+bool
+FrozenBucketsMap::thawBucket(BucketId bucket)
+{
+    MonitorGuard guard(_lock);
+    Map::iterator it(_map.find(bucket));
+    assert(it != _map.end());
+    assert(it->second.hasReaders());
+    bool isLastAndContended(false);
+    if (it->second.isLast()) {
+        if (it->second.getNotifyWriter()) {
+            isLastAndContended = true;
+        }
+        _map.erase(it);
+        guard.broadcast();
+    } else {
+        it->second.removeReader();
+    }
+    return isLastAndContended;
+}
+
+
+IFrozenBucketHandler::ExclusiveBucketGuard::UP
+FrozenBucketsMap::acquireExclusiveBucket(document::BucketId bucket)
+{
+    MonitorGuard guard(_lock);
+    Map::iterator it(_map.find(bucket));
+    if (it != _map.end()) {
+        assert(it->second.hasReaders());
+        it->second.setNotifyWriter();
+        return ExclusiveBucketGuard::UP();
+    }
+    _map[bucket] = FrozenBucket(FrozenBucket::Writer);
+    return std::make_unique<ExclusiveBucketGuard>(*this, bucket);
+}
+
+void
+FrozenBucketsMap::releaseExclusiveBucket(document::BucketId bucket)
+{
+    MonitorGuard guard(_lock);
+    Map::const_iterator it(_map.find(bucket));
+    assert ((it != _map.end()) && (it->second.isExclusive()));
+    _map.erase(it);
+    guard.broadcast();
+}
+
+FrozenBuckets::FrozenBuckets(IThreadService &masterThread) :
+    _frozen(),
+    _masterThread(masterThread),
+    _listeners()
+{
+}
+
+FrozenBuckets::~FrozenBuckets()
+{
+    assert(_listeners.empty());
+}
+
+IFrozenBucketHandler::ExclusiveBucketGuard::UP
+FrozenBuckets::acquireExclusiveBucket(document::BucketId bucket) {
+    return _frozen.acquireExclusiveBucket(bucket);
+}
+
+void
+FrozenBuckets::notifyThawed(document::BucketId bucket) {
+    assert(_masterThread.isCurrentThread());
+    for (auto &listener : _listeners) {
+        listener->notifyThawedBucket(bucket);
+    }
+}
+
+void
+FrozenBuckets::freezeBucket(BucketId bucket)
+{
+    _frozen.freezeBucket(bucket);
+}
+
+void
+FrozenBuckets::thawBucket(BucketId bucket)
+{
+    if (_frozen.thawBucket(bucket)) {
+        _masterThread.execute(makeTask(makeClosure(this, &FrozenBuckets::notifyThawed, bucket)));
+    }
+}
+
+void
+FrozenBuckets::addListener(IBucketFreezeListener *listener)
+{
+    // assert(_masterThread.isCurrentThread());
+    _listeners.push_back(listener);
+}
+
+void
+FrozenBuckets::removeListener(IBucketFreezeListener *listener)
+{
+    // assert(_masterThread.isCurrentThread());
+    auto it = std::find(_listeners.begin(), _listeners.end(), listener);
+    if (it != _listeners.end()) {
+        _listeners.erase(it);
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/frozenbuckets.h b/searchcore/src/vespa/searchcore/proton/server/frozenbuckets.h
new file mode 100644
index 00000000000..e3ac69f0fed
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/frozenbuckets.h
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcorespi/index/i_thread_service.h>
+#include <vespa/document/bucket/bucketid.h>
+#include "ifrozenbuckethandler.h"
+#include "ibucketfreezer.h"
+#include <map>
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton
+{
+
+class IBucketFreezeListener;
+
+/**
+ * Controls read and write access to buckets.
+ */
+class FrozenBucketsMap {
+public:
+    FrozenBucketsMap();
+    ~FrozenBucketsMap();
+    IFrozenBucketHandler::ExclusiveBucketGuard::UP acquireExclusiveBucket(document::BucketId bucket);
+    void freezeBucket(document::BucketId bucket);
+    // Returns true if it was the last one and it was contended.
+    bool thawBucket(document::BucketId bucket);
+    class ExclusiveBucketGuard : public IFrozenBucketHandler::ExclusiveBucketGuard {
+    public:
+        ExclusiveBucketGuard(const ExclusiveBucketGuard &) = delete;
+        ExclusiveBucketGuard(ExclusiveBucketGuard &&) = delete;
+        ExclusiveBucketGuard & operator=(const ExclusiveBucketGuard &) = delete;
+        ExclusiveBucketGuard & operator=(ExclusiveBucketGuard &&) = delete;
+
+        ExclusiveBucketGuard(FrozenBucketsMap & handler, document::BucketId & bucketId)
+                : IFrozenBucketHandler::ExclusiveBucketGuard(bucketId),
+                  _handler(handler)
+        { }
+        ~ExclusiveBucketGuard() { _handler.releaseExclusiveBucket(getBucket());}
+    private:
+        FrozenBucketsMap      & _handler;
+    };
+private:
+    void releaseExclusiveBucket(document::BucketId bucket);
+    class FrozenBucket {
+    public:
+        enum Type {Reader, Writer};
+        explicit FrozenBucket(Type type=Reader) : _refCount((type==Reader) ? 1 : -1), _notifyWriter(false) { }
+        ~FrozenBucket() { assert((_refCount == -1) || (_refCount == 1));}
+        void setNotifyWriter() { _notifyWriter = true; }
+        bool getNotifyWriter() const { return _notifyWriter; }
+        bool isLast() const { return _refCount == 1; }
+        bool isExclusive() const { return _refCount == -1; }
+        bool hasReaders() const { return _refCount >= 1; }
+        void addReader() {
+            assert(_refCount >= 1);
+            _refCount++;
+        }
+        void removeReader() {
+            assert(_refCount > 1);
+            _refCount--;
+        }
+    private:
+        int32_t _refCount;
+        bool    _notifyWriter;
+    };
+    typedef std::map<document::BucketId, FrozenBucket> Map;
+    vespalib::Monitor  _lock;
+    Map                _map;
+};
+
+/**
+ * Class that remembers which buckets are frozen and notifies all
+ * registered listeners on bucket frozenness changes.
+ */
+class FrozenBuckets : public IFrozenBucketHandler,
+                      public IBucketFreezer
+{
+    FrozenBucketsMap                       _frozen;
+    searchcorespi::index::IThreadService  &_masterThread;
+    std::vector<IBucketFreezeListener *>   _listeners;
+
+    void notifyThawed(document::BucketId bucket);
+public:
+    FrozenBuckets(searchcorespi::index::IThreadService &masterThread);
+    virtual ~FrozenBuckets();
+
+    virtual ExclusiveBucketGuard::UP acquireExclusiveBucket(document::BucketId bucket) override;
+    virtual void freezeBucket(document::BucketId bucket) override;
+    virtual void thawBucket(document::BucketId bucket) override;
+    virtual void addListener(IBucketFreezeListener *listener) override;
+    virtual void removeListener(IBucketFreezeListener *listener) override;
+};
+    
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/health_adapter.cpp b/searchcore/src/vespa/searchcore/proton/server/health_adapter.cpp
new file mode 100644
index 00000000000..be5ed968851
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/health_adapter.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.health_adapter");
+#include <vespa/vespalib/util/stringfmt.h>
+#include "health_adapter.h"
+
+namespace proton {
+
+HealthAdapter::HealthAdapter(const StatusProducer &sp)
+    : _statusProducer(sp)
+{
+}
+
+vespalib::HealthProducer::Health
+HealthAdapter::getHealth() const
+{
+    bool ok = true;
+    vespalib::string msg;
+    StatusReport::List reports(_statusProducer.getStatusReports());
+    for (size_t i = 0; i < reports.size(); ++i) {
+        const StatusReport &r = *reports[i];
+        if (r.getState() != StatusReport::UPOK) {
+            ok = false;
+            if (msg.size() > 0) {
+                msg.append(", ");
+            }
+            msg.append(vespalib::make_string("%s: %s",
+                                             r.getComponent().c_str(),
+                                             r.getMessage().c_str()));
+        }
+    }
+    if (ok) {
+        return Health(true, "All OK");
+    } else {
+        return Health(false, msg);
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/health_adapter.h b/searchcore/src/vespa/searchcore/proton/server/health_adapter.h
new file mode 100644
index 00000000000..9cc87c63eed
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/health_adapter.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/common/statusreport.h>
+#include <vespa/vespalib/net/health_producer.h>
+
+namespace proton {
+
+class HealthAdapter : public vespalib::HealthProducer
+{
+private:
+    const StatusProducer &_statusProducer;
+
+public:
+    HealthAdapter(const StatusProducer &sp);
+    virtual Health getHealth() const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.cpp b/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.cpp
new file mode 100644
index 00000000000..b9dcf94c105
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.cpp
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "heart_beat_job.h"
+
+namespace proton {
+
+HeartBeatJob::HeartBeatJob(IHeartBeatHandler &handler,
+                           const DocumentDBHeartBeatConfig &config)
+    : IMaintenanceJob("heart_beat", config.getInterval(), config.getInterval()),
+      _handler(handler)
+{
+}
+
+bool
+HeartBeatJob::run()
+{
+    _handler.heartBeat();
+    return true;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.h b/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.h
new file mode 100644
index 00000000000..0741594c567
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "document_db_maintenance_config.h"
+#include "i_maintenance_job.h"
+#include "iheartbeathandler.h"
+
+namespace proton {
+
+/**
+ * Job that regularly does heart beating on a given handler.
+ *
+ * The FeedHandler is typically acting as a handler to do
+ * heart beating on its underlying components.
+ */
+class HeartBeatJob : public IMaintenanceJob
+{
+private:
+    IHeartBeatHandler &_handler;
+
+public:
+    HeartBeatJob(IHeartBeatHandler &handler,
+                 const DocumentDBHeartBeatConfig &config);
+
+    // Implements IMaintenanceJob
+    virtual bool run();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_document_scan_iterator.h b/searchcore/src/vespa/searchcore/proton/server/i_document_scan_iterator.h
new file mode 100644
index 00000000000..6f9bae77242
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/i_document_scan_iterator.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/idocumentmetastore.h>
+
+namespace proton {
+
+/**
+ * Iterator for scanning all documents in a document sub db to find candidates
+ * for moving as part of lid space compaction.
+ */
+struct IDocumentScanIterator
+{
+    typedef std::unique_ptr<IDocumentScanIterator> UP;
+
+    virtual ~IDocumentScanIterator() {}
+
+    /**
+     * Returns false if we are certain there are no more documents to scan, true otherwise.
+     * Returning false should only happen after a call to next() has returned an invalid document.
+     */
+    virtual bool valid() const = 0;
+
+    /**
+     * Returns the next document that has lid > compactLidLimit to be moved.
+     * Returns an invalid document if no documents satisfy the limit or
+     * if max documents are scanned.
+     *
+     * @param compactLidLimit The returned document must have lid larger than this limit.
+     * @param maxDocsToScan The maximum documents to scan before returning.
+     * @param retry Whether we should start the scan with the previous returned document.
+     */
+    virtual search::DocumentMetaData next(uint32_t compactLidLimit,
+                                          uint32_t maxDocsToScan,
+                                          bool retry) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_lid_space_compaction_handler.h b/searchcore/src/vespa/searchcore/proton/server/i_lid_space_compaction_handler.h
new file mode 100644
index 00000000000..022079aa87f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/i_lid_space_compaction_handler.h
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_document_scan_iterator.h"
+#include "ifrozenbuckethandler.h"
+#include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include <vespa/searchlib/common/lid_usage_stats.h>
+
+namespace proton {
+
+/**
+ * Interface for handling of lid space compaction, used by a LidSpaceCompactionJob.
+ *
+ * An implementation of this interface is typically working over a single document sub db.
+ */
+struct ILidSpaceCompactionHandler
+{
+    typedef std::unique_ptr<ILidSpaceCompactionHandler> UP;
+    typedef std::vector<UP> Vector;
+
+    virtual ~ILidSpaceCompactionHandler() {}
+
+    /**
+     * Returns the name of this handler.
+     */
+    virtual vespalib::string getName() const = 0;
+
+    /**
+     * Returns the id of the sub database this handler is operating over.
+     */
+    virtual uint32_t getSubDbId() const = 0;
+
+    /**
+     * Returns the current lid status of the underlying components.
+     */
+    virtual search::LidUsageStats getLidStatus() const = 0;
+
+    /**
+     * Returns an iterator for scanning documents.
+     */
+    virtual IDocumentScanIterator::UP getIterator() const = 0;
+
+    /**
+     * Creates a move operation for moving the given document to the given lid.
+     */
+    virtual MoveOperation::UP createMoveOperation(const search::DocumentMetaData &document, uint32_t moveToLid) const = 0;
+
+    /**
+     * Performs the actual move operation.
+     */
+    virtual void handleMove(const MoveOperation &op) = 0;
+
+    /**
+     * Compacts the underlying lid space by starting using the new lid limit.
+     */
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h
new file mode 100644
index 00000000000..fd7b2b17b46
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+class IMaintenanceJobRunner;
+
+/**
+ * Interface for a maintenance job that is executed after "delay" seconds and
+ * then every "interval" seconds.
+ */
+class IMaintenanceJob
+{
+private:
+    const vespalib::string _name;
+    const double           _delay;
+    const double           _interval;
+    bool                   _blocked;
+
+protected:
+    void setBlocked(bool v) { _blocked = v; }
+
+public:
+    typedef std::unique_ptr<IMaintenanceJob> UP;
+
+    IMaintenanceJob(const vespalib::string &name,
+                    double delay,
+                    double interval)
+        : _name(name),
+          _delay(delay),
+          _interval(interval),
+          _blocked(false)
+    {
+    }
+
+    virtual ~IMaintenanceJob() {}
+
+    virtual const vespalib::string &getName() const { return _name; }
+
+    virtual double getDelay() const { return _delay; }
+
+    virtual double getInterval() const { return _interval; }
+
+    virtual bool isBlocked() const { return _blocked; }
+
+    virtual void unBlock() { setBlocked(false); }
+
+    /**
+     * Register maintenance job runner, in case event passed to the
+     * job causes it to want to be run again.
+     */
+    virtual void
+    registerRunner(IMaintenanceJobRunner *runner)
+    {
+        (void) runner;
+    }
+
+    /**
+     * Run this maintenance job every "interval" seconds in an external executor thread.
+     * It is first executed after "delay" seconds.
+     *
+     * Return true if the job was finished (it will be executed again in "interval" seconds).
+     * Return false if the job was not finished and need to be executed again immediately. This
+     * should be used to split up a large job to avoid starvation of other tasks that also are
+     * executed on the external executor thread.
+     */
+    virtual bool run() = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h
new file mode 100644
index 00000000000..3d75244a188
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/feedoperation/feedoperation.h>
+
+namespace proton {
+
+/**
+ * Interface for a component assigning serial numbers and storing feed operations.
+ */
+struct IOperationStorer
+{
+    virtual ~IOperationStorer() {}
+
+    /**
+     * Assign serial number to (if not set) and store the given operation.
+     */
+    virtual void storeOperation(FeedOperation &op) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/iattributeadapterfactory.h b/searchcore/src/vespa/searchcore/proton/server/iattributeadapterfactory.h
new file mode 100644
index 00000000000..e977d816f28
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/iattributeadapterfactory.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/attribute/attribute_collection_spec.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_writer.h>
+
+namespace proton {
+
+struct IAttributeAdapterFactory
+{
+    typedef std::unique_ptr<IAttributeAdapterFactory> UP;
+    virtual ~IAttributeAdapterFactory() {}
+    virtual IAttributeWriter::SP create(const IAttributeWriter::SP &old,
+            const AttributeCollectionSpec &attrSpec) const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ibucketfreezelistener.h b/searchcore/src/vespa/searchcore/proton/server/ibucketfreezelistener.h
new file mode 100644
index 00000000000..0401983710b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ibucketfreezelistener.h
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document
+{
+
+class BucketId;
+
+}
+
+namespace proton
+{
+
+
+/**
+ * Interface class used by a registered listener to get notifications about
+ * bucket frozenness changes.
+ */
+class IBucketFreezeListener
+{
+public:
+    virtual ~IBucketFreezeListener() {}
+    virtual void notifyThawedBucket(const document::BucketId &bucket) = 0;
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ibucketfreezer.h b/searchcore/src/vespa/searchcore/proton/server/ibucketfreezer.h
new file mode 100644
index 00000000000..16f127b1822
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ibucketfreezer.h
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+
+namespace proton
+{
+
+class IBucketFreezer
+{
+public:
+    virtual ~IBucketFreezer() {}
+    virtual void freezeBucket(document::BucketId bucket) = 0;
+    virtual void thawBucket(document::BucketId bucket) = 0;
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h b/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h
new file mode 100644
index 00000000000..aa7ceafb470
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document
+{
+
+class BucketId;
+
+}
+
+namespace proton
+{
+
+
+class IBucketModifiedHandler
+{
+public:
+    virtual void notifyBucketModified(const document::BucketId &bucket) = 0;
+    virtual ~IBucketModifiedHandler() {}
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ibucketstatecalculator.h b/searchcore/src/vespa/searchcore/proton/server/ibucketstatecalculator.h
new file mode 100644
index 00000000000..0fb3c773471
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ibucketstatecalculator.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document
+{
+
+class BucketId;
+
+}
+
+namespace proton
+{
+
+struct IBucketStateCalculator
+{
+    typedef std::shared_ptr<IBucketStateCalculator> SP;
+    virtual bool shouldBeReady(const document::BucketId &bucket) const = 0;
+
+    virtual bool
+    clusterUp(void) const = 0;
+
+    virtual bool
+    nodeUp(void) const = 0;
+
+    virtual bool
+    nodeInitializing() const = 0;
+
+    virtual ~IBucketStateCalculator() {}
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ibucketstatechangedhandler.h b/searchcore/src/vespa/searchcore/proton/server/ibucketstatechangedhandler.h
new file mode 100644
index 00000000000..51e19cd6571
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ibucketstatechangedhandler.h
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/persistence/spi/bucketinfo.h>
+
+namespace document
+{
+
+class BucketId;
+
+}
+
+namespace proton
+{
+
+
+/**
+ * Interface used to notify when bucket state has changed.
+ */
+class IBucketStateChangedHandler
+{
+public:
+    virtual void notifyBucketStateChanged(const document::BucketId &bucketId,
+                                          storage::spi::BucketInfo::ActiveState newState) = 0;
+
+    virtual ~IBucketStateChangedHandler() {}
+};
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/ibucketstatechangednotifier.h b/searchcore/src/vespa/searchcore/proton/server/ibucketstatechangednotifier.h
new file mode 100644
index 00000000000..057c0ce8c33
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ibucketstatechangednotifier.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+class IBucketStateChangedHandler;
+
+/**
+ * Interface used to request notification when bucket state has changed.
+ */
+class IBucketStateChangedNotifier
+{
+public:
+    virtual void
+    addBucketStateChangedHandler(IBucketStateChangedHandler *handler) = 0;
+
+    virtual void
+    removeBucketStateChangedHandler(IBucketStateChangedHandler *handler) = 0;
+    
+    virtual ~IBucketStateChangedNotifier() {}
+};
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/iclusterstatechangedhandler.h b/searchcore/src/vespa/searchcore/proton/server/iclusterstatechangedhandler.h
new file mode 100644
index 00000000000..6761960f9b1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/iclusterstatechangedhandler.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "ibucketstatecalculator.h"
+
+namespace proton
+{
+
+/**
+ * Interface used to notify when cluster state has changed.
+ */
+class IClusterStateChangedHandler
+{
+public:
+    virtual ~IClusterStateChangedHandler() { }
+
+    virtual void
+    notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) = 0;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/iclusterstatechangednotifier.h b/searchcore/src/vespa/searchcore/proton/server/iclusterstatechangednotifier.h
new file mode 100644
index 00000000000..ba8586fc0aa
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/iclusterstatechangednotifier.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+class IClusterStateChangedHandler;
+
+/**
+ * Interface used to request notification when cluster state has changed.
+ */
+class IClusterStateChangedNotifier
+{
+public:
+    virtual ~IClusterStateChangedNotifier() { }
+
+    virtual void
+    addClusterStateChangedHandler(IClusterStateChangedHandler *handler) = 0;
+
+    virtual void
+    removeClusterStateChangedHandler(IClusterStateChangedHandler *handler) = 0;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/icommitable.h b/searchcore/src/vespa/searchcore/proton/server/icommitable.h
new file mode 100644
index 00000000000..90f7b9c71d1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/icommitable.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton {
+
+/**
+ * Interface for anyone that needs to commit.
+ **/
+class ICommitable {
+public:
+    virtual void commit() = 0;
+    virtual void commitAndWait() = 0;
+protected:
+    virtual ~ICommitable() { }
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.cpp b/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.cpp
new file mode 100644
index 00000000000..0fcf29b50c6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.cpp
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "idocumentdbowner.h"
+
+namespace proton
+{
+
+IDocumentDBOwner::~IDocumentDBOwner(void)
+{
+}
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.h b/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.h
new file mode 100644
index 00000000000..b26b80b579b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.h
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h>
+
+namespace proton
+{
+
+class IDocumentDBOwner
+{
+public:
+    virtual ~IDocumentDBOwner(void);
+
+    virtual bool isInitializing() const = 0;
+
+    virtual searchcorespi::IIndexManagerFactory::SP
+    getIndexManagerFactory(const vespalib::stringref & name) const = 0;
+    virtual uint32_t getDistributionKey() const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentmovehandler.h b/searchcore/src/vespa/searchcore/proton/server/idocumentmovehandler.h
new file mode 100644
index 00000000000..543dc565ede
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentmovehandler.h
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+class MoveOperation;
+
+
+struct IDocumentMoveHandler
+{
+    virtual void handleMove(MoveOperation &op) = 0;
+    virtual ~IDocumentMoveHandler() {}
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
new file mode 100644
index 00000000000..da49aa2a725
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
@@ -0,0 +1,180 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "document_subdb_initializer.h"
+#include "ifeedview.h"
+#include "searchable_doc_subdb_configurer.h"
+#include <vespa/searchcore/config/config-proton.h>
+#include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include <vespa/searchcore/proton/docsummary/isummarymanager.h>
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+#include <vespa/searchcore/proton/index/i_index_writer.h>
+#include <vespa/searchcore/proton/matchengine/imatchhandler.h>
+#include <vespa/searchcore/proton/matching/matching_stats.h>
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
+#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h>
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+#include <vespa/searchcorespi/index/iindexmanager.h>
+#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/searchlib/util/searchable_stats.h>
+
+namespace document
+{
+
+class DocumentId;
+
+}
+
+namespace proton
+{
+
+class FeedHandler;
+class DocumentDBConfig;
+class FileConfigManager;
+class IReplayConfig;
+
+/**
+ * Interface for a document sub database that handles a subset of the documents that belong to a
+ * DocumentDB.
+ *
+ * Documents can be inserted/updated/removed to a sub database via a feed view,
+ * searched via a search view and retrieved via a document retriever.
+ * A sub database is separate and independent from other sub databases.
+ */
+class IDocumentSubDB
+{
+public:
+    class IOwner
+    {
+    public:
+        virtual ~IOwner() {}
+        virtual void syncFeedView() = 0;
+        virtual searchcorespi::IIndexManagerFactory::SP
+        getIndexManagerFactory(const vespalib::stringref &name) const = 0;
+        virtual vespalib::string getName() const = 0;
+        virtual uint32_t getDistributionKey() const = 0;
+    };
+
+    typedef std::unique_ptr<IDocumentSubDB>              UP;
+    typedef search::SerialNum      SerialNum;
+
+public:
+    IDocumentSubDB() { }
+
+    virtual ~IDocumentSubDB() { }
+
+    virtual uint32_t getSubDbId() const = 0;
+
+    virtual vespalib::string getName() const = 0;
+
+    virtual DocumentSubDbInitializer::UP
+    createInitializer(const DocumentDBConfig &configSnapshot,
+                      SerialNum configSerialNum,
+                      const search::index::Schema::SP &unionSchema,
+                      const vespa::config::search::core::
+                      ProtonConfig::Summary &protonSummaryCfg,
+                      const vespa::config::search::core::
+                      ProtonConfig::Index &indexCfg) const = 0;
+
+    // Called by master thread
+    virtual void setup(const DocumentSubDbInitializerResult &initResult) = 0;
+
+    virtual void
+    initViews(const DocumentDBConfig &configSnapshot,
+              const proton::matching::SessionManager::SP &sessionManager) = 0;
+
+    virtual IReprocessingTask::List
+    applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                const DocumentDBConfig &oldConfigSnapshot,
+                SerialNum serialNum,
+                const ReconfigParams params) = 0;
+
+    virtual ISearchHandler::SP
+    getSearchView(void) const = 0;
+
+    virtual IFeedView::SP
+    getFeedView(void) const = 0;
+
+    virtual void
+    clearViews(void) = 0;
+
+    virtual const ISummaryManager::SP &
+    getSummaryManager() const = 0;
+
+    virtual proton::IAttributeManager::SP
+    getAttributeManager(void) const = 0;
+
+    virtual const IIndexManager::SP &
+    getIndexManager(void) const = 0;
+
+    virtual const ISummaryAdapter::SP &
+    getSummaryAdapter(void) const = 0;
+
+    virtual const IIndexWriter::SP &
+    getIndexWriter(void) const = 0;
+
+    virtual IDocumentMetaStoreContext &
+    getDocumentMetaStoreContext() = 0;
+
+    virtual IFlushTarget::List
+    getFlushTargets(void) = 0;
+
+    virtual size_t
+    getNumDocs(void) const = 0;
+
+    virtual size_t
+    getNumActiveDocs(void) const = 0;
+
+    /**
+     * Needed by FeedRouter::handleRemove().
+     * TODO: remove together with FeedEngine.
+     **/
+    virtual bool
+    hasDocument(const document::DocumentId &id) = 0;
+
+    virtual void
+    onReplayDone(void) = 0;
+
+    virtual void
+    onReprocessDone(SerialNum serialNum) = 0;
+
+    /*
+     * Get oldest flushed serial for components.
+     */
+    virtual SerialNum
+    getOldestFlushedSerial(void) = 0;
+
+    /*
+     * Get newest flushed serial.  Used to validate that we've not lost
+     * last part of transaction log.
+     */
+    virtual SerialNum
+    getNewestFlushedSerial()  = 0;
+
+    virtual void
+    wipeHistory(SerialNum wipeSerial,
+                const search::index::Schema &newHistorySchema,
+                const search::index::Schema &wipeSchema) = 0;
+
+    virtual void
+    setIndexSchema(const search::index::Schema::SP &schema,
+                   const search::index::Schema::SP &fusionSchema) = 0;
+
+    virtual search::SearchableStats
+    getSearchableStats(void) const = 0;
+
+    virtual IDocumentRetriever::UP
+    getDocumentRetriever(void) = 0;
+
+    virtual matching::MatchingStats
+    getMatcherStats(const vespalib::string &rankProfile) const = 0;
+
+    virtual void close() = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
new file mode 100644
index 00000000000..eda592ec0e4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/proton/documentmetastore/i_simple_document_meta_store.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h>
+#include <vespa/searchcore/proton/feedoperation/deletebucketoperation.h>
+#include <vespa/searchcore/proton/feedoperation/joinbucketsoperation.h>
+#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h>
+#include <vespa/searchcore/proton/feedoperation/putoperation.h>
+#include <vespa/searchcore/proton/feedoperation/removeoperation.h>
+#include <vespa/searchcore/proton/feedoperation/splitbucketoperation.h>
+#include <vespa/searchcore/proton/feedoperation/updateoperation.h>
+#include <vespa/searchlib/transactionlog/common.h>
+
+namespace proton
+{
+
+class MoveOperation;
+
+/**
+ * Interface for a feed view as seen from a feed handler.
+ */
+class IFeedView : public boost::noncopyable
+{
+protected:
+    typedef search::transactionlog::Packet Packet;
+public:
+    typedef std::shared_ptr<IFeedView> SP;
+
+    virtual~IFeedView() { }
+
+    virtual const document::DocumentTypeRepo::SP &getDocumentTypeRepo() const = 0;
+
+    /**
+     * Access to const version of document meta store.
+     * Should only be used by the writer thread.
+     */
+    virtual const ISimpleDocumentMetaStore * getDocumentMetaStorePtr() const = 0;
+
+    /**
+     * Similar to IPersistenceHandler functions.
+     */
+
+    virtual void preparePut(PutOperation &putOp) = 0;
+    virtual void handlePut(FeedToken *token, const PutOperation &putOp) = 0;
+    virtual void prepareUpdate(UpdateOperation &updOp) = 0;
+    virtual void handleUpdate(FeedToken *token, const UpdateOperation &updOp) = 0;
+    virtual void prepareRemove(RemoveOperation &rmOp) = 0;
+    virtual void handleRemove(FeedToken *token, const RemoveOperation &rmOp) = 0;
+    virtual void prepareDeleteBucket(DeleteBucketOperation &delOp) = 0;
+    virtual void handleDeleteBucket(const DeleteBucketOperation &delOp) = 0;
+    virtual void prepareMove(MoveOperation &putOp) = 0;
+    virtual void handleMove(const MoveOperation &putOp) = 0;
+    virtual void heartBeat(search::SerialNum serialNum) = 0;
+    virtual void sync() = 0;
+    virtual void forceCommit(search::SerialNum serialNum) = 0;
+    virtual void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation & pruneOp) = 0;
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ifrozenbuckethandler.h b/searchcore/src/vespa/searchcore/proton/server/ifrozenbuckethandler.h
new file mode 100644
index 00000000000..705f7ec8408
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ifrozenbuckethandler.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+
+namespace proton
+{
+
+class IBucketFreezeListener;
+
+class IFrozenBucketHandler
+{
+public:
+    class ExclusiveBucketGuard {
+    public:
+        typedef std::unique_ptr<ExclusiveBucketGuard> UP;
+        ExclusiveBucketGuard(document::BucketId bucketId) : _bucketId(bucketId) { }
+        virtual ~ExclusiveBucketGuard() { }
+        document::BucketId getBucket() const { return _bucketId; }
+    private:
+        document::BucketId _bucketId;
+    };
+
+    virtual ~IFrozenBucketHandler(void) { }
+    virtual ExclusiveBucketGuard::UP acquireExclusiveBucket(document::BucketId bucket) = 0;
+    virtual void addListener(IBucketFreezeListener *listener) = 0;
+    virtual void removeListener(IBucketFreezeListener *listener) = 0;
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/igetserialnum.h b/searchcore/src/vespa/searchcore/proton/server/igetserialnum.h
new file mode 100644
index 00000000000..ea547d2f4b3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/igetserialnum.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/serialnum.h>
+
+namespace proton {
+
+/**
+ * A simple interface for getting a serial number
+ **/
+class IGetSerialNum {
+public:
+    virtual ~IGetSerialNum() { }
+    virtual search::SerialNum getSerialNum() const = 0;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/iheartbeathandler.h b/searchcore/src/vespa/searchcore/proton/server/iheartbeathandler.h
new file mode 100644
index 00000000000..f863bcc7b4a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/iheartbeathandler.h
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+class FeedToken;
+
+class IHeartBeatHandler
+{
+public:
+    virtual void
+    heartBeat(void) = 0;
+
+    virtual
+    ~IHeartBeatHandler(void)
+    {
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h b/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h
new file mode 100644
index 00000000000..921753555fa
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+/**
+ * Interface for running maintenance jobs (cf. IMaintenanceJob).
+ */
+class IMaintenanceJobRunner
+{
+public:
+    /*
+     * Schedule job to be run in the future.
+     */
+    virtual void run() = 0;
+    virtual ~IMaintenanceJobRunner() { }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ipruneremoveddocumentshandler.h b/searchcore/src/vespa/searchcore/proton/server/ipruneremoveddocumentshandler.h
new file mode 100644
index 00000000000..3ff8dfcab2e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ipruneremoveddocumentshandler.h
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+class PruneRemovedDocumentsOperation;
+
+class IPruneRemovedDocumentsHandler
+{
+public:
+    virtual void
+    performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp) = 0;
+
+    virtual
+    ~IPruneRemovedDocumentsHandler(void)
+    {
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp
new file mode 100644
index 00000000000..16ed662b310
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "ireplayconfig.h"
+
+namespace proton
+{
+
+IReplayConfig::~IReplayConfig(void)
+{
+}
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h
new file mode 100644
index 00000000000..866eeba4bea
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/transactionlog/translogclient.h>
+
+namespace proton
+{
+
+class IReplayConfig
+{
+public:
+    virtual
+    ~IReplayConfig(void);
+
+    virtual void
+    replayConfig(search::SerialNum serialNum) = 0;
+
+    virtual void
+    replayWipeHistory(search::SerialNum serialNum,
+                      fastos::TimeStamp wipeTimeLimit) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h b/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h
new file mode 100644
index 00000000000..e3a588654b4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/feedoperation/operations.h>
+
+namespace proton {
+
+/**
+ * Interface used to handle the various feed operations during
+ * replay of the transaction log.
+ */
+struct IReplayPacketHandler
+{
+    virtual ~IReplayPacketHandler() {}
+    virtual void replay(const PutOperation &op) = 0;
+    virtual void replay(const RemoveOperation &op) = 0;
+    virtual void replay(const UpdateOperation &op) = 0;
+    virtual void replay(const NoopOperation &op) = 0;
+    virtual void replay(const NewConfigOperation &op) = 0;
+    virtual void replay(const WipeHistoryOperation &op) = 0;
+    virtual void replay(const DeleteBucketOperation &op) = 0;
+    virtual void replay(const SplitBucketOperation &op) = 0;
+    virtual void replay(const JoinBucketsOperation &op) = 0;
+    virtual void replay(const PruneRemovedDocumentsOperation &op) = 0;
+    virtual void replay(const SpoolerReplayStartOperation &op) = 0;
+    virtual void replay(const SpoolerReplayCompleteOperation &op) = 0;
+    virtual void replay(const MoveOperation &op) = 0;
+    virtual void replay(const CreateBucketOperation &op) = 0;
+    virtual void replay(const CompactLidSpaceOperation &op) = 0;
+    
+    virtual NewConfigOperation::IStreamHandler &getNewConfigStreamHandler() = 0;
+    virtual document::DocumentTypeRepo &getDeserializeRepo() = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h b/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h
new file mode 100644
index 00000000000..0560dc334ab
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/searchcore/proton/docsummary/isummarymanager.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/searchlib/common/serialnum.h>
+
+namespace proton {
+
+/**
+ * Interface for a summary adapter.
+ **/
+class ISummaryAdapter {
+public:
+    typedef std::unique_ptr<ISummaryAdapter> UP;
+    typedef std::shared_ptr<ISummaryAdapter> SP;
+
+    virtual ~ISummaryAdapter() {}
+
+    // feed interface
+    virtual void put(search::SerialNum serialNum,
+                     const document::Document &doc,
+                     const search::DocumentIdT lid) = 0;
+    virtual void remove(search::SerialNum serialNum,
+                        const search::DocumentIdT lid) = 0;
+    virtual void update(search::SerialNum serialNum,
+                        const document::DocumentUpdate &upd,
+                        const search::DocumentIdT lid,
+                        const document::DocumentTypeRepo &repo) = 0;
+
+    virtual void
+    heartBeat(search::SerialNum serialNum) = 0;
+
+    virtual const search::IDocumentStore &
+    getDocumentStore(void) const = 0;
+
+    virtual std::unique_ptr<document::Document>
+    get(const search::DocumentIdT lid,
+        const document::DocumentTypeRepo &repo) = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/itlssyncer.h b/searchcore/src/vespa/searchcore/proton/server/itlssyncer.h
new file mode 100644
index 00000000000..f8c79d6485d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/itlssyncer.h
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton
+{
+
+
+/*
+ * Interface class for syncing transaction log server in a safe manner.
+ */
+class ITlsSyncer
+{
+public:
+    virtual ~ITlsSyncer(void) = default;
+    virtual void sync() = 0;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h b/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h
new file mode 100644
index 00000000000..95f70ca308c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton {
+
+class IWipeOldRemovedFieldsHandler
+{
+public:
+    virtual void wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit) = 0;
+
+    virtual ~IWipeOldRemovedFieldsHandler() {}
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp
new file mode 100644
index 00000000000..d8c197b6c87
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.job_tracked_maintenance_job");
+#include "job_tracked_maintenance_job.h"
+
+namespace proton {
+
+JobTrackedMaintenanceJob::JobTrackedMaintenanceJob(const IJobTracker::SP &tracker,
+                                                   IMaintenanceJob::UP job)
+    : IMaintenanceJob(job->getName(), job->getDelay(), job->getInterval()),
+      _tracker(tracker),
+      _job(std::move(job)),
+      _running(false)
+{
+}
+
+JobTrackedMaintenanceJob::~JobTrackedMaintenanceJob()
+{
+    if (_running) {
+        _tracker->end();
+    }
+}
+
+bool
+JobTrackedMaintenanceJob::run()
+{
+    if (!_running) {
+        _running = true;
+        _tracker->start();
+    }
+    bool finished = _job->run();
+    if (finished) {
+        _running = false;
+        _tracker->end();
+    }
+    return finished;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.h
new file mode 100644
index 00000000000..6bd227ae193
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_maintenance_job.h"
+#include <vespa/searchcore/proton/metrics/i_job_tracker.h>
+
+namespace proton {
+
+/**
+ * Class for tracking the start and end of a maintenance job.
+ */
+class JobTrackedMaintenanceJob : public IMaintenanceJob
+{
+private:
+    IJobTracker::SP     _tracker;
+    IMaintenanceJob::UP _job;
+    bool                _running;
+
+public:
+    JobTrackedMaintenanceJob(const IJobTracker::SP &tracker,
+                             IMaintenanceJob::UP job);
+    ~JobTrackedMaintenanceJob();
+
+    // Implements IMaintenanceJob
+    virtual bool isBlocked() const { return _job->isBlocked(); }
+    virtual void unBlock() { _job->unBlock(); }
+    virtual void registerRunner(IMaintenanceJobRunner *runner) {
+        _job->registerRunner(runner);
+    }
+    virtual bool run();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp
new file mode 100644
index 00000000000..09cd3770919
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.lid_space_compaction_handler");
+
+#include "lid_space_compaction_handler.h"
+#include "document_scan_iterator.h"
+
+using document::BucketId;
+using document::Document;
+using search::LidUsageStats;
+using storage::spi::Timestamp;
+
+namespace proton {
+
+LidSpaceCompactionHandler::LidSpaceCompactionHandler(IDocumentSubDB &subDb,
+                                                     const vespalib::string &docTypeName)
+    : _subDb(subDb),
+      _docTypeName(docTypeName)
+{
+}
+
+LidUsageStats
+LidSpaceCompactionHandler::getLidStatus() const
+{
+    return _subDb.getDocumentMetaStoreContext().get().getLidUsageStats();
+}
+
+IDocumentScanIterator::UP
+LidSpaceCompactionHandler::getIterator() const
+{
+    return IDocumentScanIterator::UP(new DocumentScanIterator(
+            _subDb.getDocumentMetaStoreContext().get()));
+}
+
+MoveOperation::UP
+LidSpaceCompactionHandler::createMoveOperation(const search::DocumentMetaData &document, uint32_t moveToLid) const
+{
+    IFeedView::SP feedView = _subDb.getFeedView();
+    const ISummaryManager::SP &summaryMan = _subDb.getSummaryManager();
+    const uint32_t moveFromLid = document.lid;
+    Document::UP doc = summaryMan->getBackingStore().read(moveFromLid,
+            *feedView->getDocumentTypeRepo());
+    MoveOperation::UP op(new MoveOperation(document.bucketId, document.timestamp,
+                                           Document::SP(doc.release()),
+                                           DbDocumentId(_subDb.getSubDbId(), moveFromLid),
+                                           _subDb.getSubDbId()));
+    op->setTargetLid(moveToLid);
+    return op;
+}
+
+void
+LidSpaceCompactionHandler::handleMove(const MoveOperation& op)
+{
+    _subDb.getFeedView()->handleMove(op);
+}
+
+void
+LidSpaceCompactionHandler::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+{
+    assert(_subDb.getSubDbId() == op.getSubDbId());
+    _subDb.getFeedView()->handleCompactLidSpace(op);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.h
new file mode 100644
index 00000000000..19ca603a972
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_lid_space_compaction_handler.h"
+#include "idocumentsubdb.h"
+
+namespace proton {
+
+/**
+ * Class that handles lid space compaction over a single document sub db.
+ */
+class LidSpaceCompactionHandler : public ILidSpaceCompactionHandler
+{
+private:
+    IDocumentSubDB  &_subDb;
+    vespalib::string _docTypeName;
+
+public:
+    LidSpaceCompactionHandler(IDocumentSubDB &subDb,
+                              const vespalib::string &docTypeName);
+
+    // Implements ILidSpaceCompactionHandler
+    virtual vespalib::string getName() const {
+        return _docTypeName + "." + _subDb.getName();
+    }
+    virtual uint32_t getSubDbId() const { return _subDb.getSubDbId(); }
+    virtual search::LidUsageStats getLidStatus() const;
+    virtual IDocumentScanIterator::UP getIterator() const;
+    virtual MoveOperation::UP createMoveOperation(const search::DocumentMetaData &document, uint32_t moveToLid) const;
+    virtual void handleMove(const MoveOperation &op);
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp
new file mode 100644
index 00000000000..838f36675c5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.lid_space_compaction_job");
+
+#include "lid_space_compaction_job.h"
+#include "ifrozenbuckethandler.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+
+using search::DocumentMetaData;
+using search::LidUsageStats;
+
+namespace proton {
+
+bool
+LidSpaceCompactionJob::hasTooMuchLidBloat(const LidUsageStats &stats) const
+{
+    return (stats.getLidBloat() >= _cfg.getAllowedLidBloat() &&
+            stats.getLidBloatFactor() >= _cfg.getAllowedLidBloatFactor() &&
+            stats.getLidLimit() > stats.getLowestFreeLid());
+}
+
+bool
+LidSpaceCompactionJob::shouldRestartScanDocuments(const LidUsageStats &stats) const
+{
+    return (stats.getUsedLids() + _cfg.getAllowedLidBloat()) < stats.getHighestUsedLid() &&
+        stats.getLowestFreeLid() < stats.getHighestUsedLid();
+}
+
+DocumentMetaData
+LidSpaceCompactionJob::getNextDocument(const LidUsageStats &stats)
+{
+    DocumentMetaData document =
+        _scanItr->next(std::max(stats.getLowestFreeLid(), stats.getUsedLids()),
+                       _cfg.getMaxDocsToScan(), _retryFrozenDocument);
+    _retryFrozenDocument = false;
+    return document;
+}
+
+bool
+LidSpaceCompactionJob::scanDocuments(const LidUsageStats &stats)
+{
+    if (_scanItr->valid()) {
+        DocumentMetaData document = getNextDocument(stats);
+        if (document.valid()) {
+            IFrozenBucketHandler::ExclusiveBucketGuard::UP bucketGuard = _frozenHandler.acquireExclusiveBucket(document.bucketId);
+            if ( ! bucketGuard ) {
+                // the job is blocked until the bucket for this document is thawed
+                setBlocked(true);
+                _retryFrozenDocument = true;
+                return true;
+            } else {
+                MoveOperation::UP op = _handler.createMoveOperation(document, stats.getLowestFreeLid());
+                _opStorer.storeOperation(*op);
+                _handler.handleMove(*op);
+            }
+        }
+    }
+    if (!_scanItr->valid()){
+        if (shouldRestartScanDocuments(_handler.getLidStatus())) {
+            _scanItr = _handler.getIterator();
+        } else {
+            _scanItr = IDocumentScanIterator::UP();
+            _shouldCompactLidSpace = true;
+        }
+    }
+    return false; // more work to do (scan documents or compact lid space)
+}
+
+void
+LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats)
+{
+    uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1;
+    CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit);
+    _opStorer.storeOperation(op);
+    _handler.handleCompactLidSpace(op);
+    if (LOG_WOULD_LOG(event)) {
+        EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit);
+    }
+    _shouldCompactLidSpace = false;
+}
+
+LidSpaceCompactionJob::LidSpaceCompactionJob(const DocumentDBLidSpaceCompactionConfig &config,
+                                             ILidSpaceCompactionHandler &handler,
+                                             IOperationStorer &opStorer,
+                                             IFrozenBucketHandler &frozenHandler)
+    : IMaintenanceJob("lid_space_compaction." + handler.getName(),
+            config.getInterval(), config.getInterval()),
+      _cfg(config),
+      _handler(handler),
+      _opStorer(opStorer),
+      _frozenHandler(frozenHandler),
+      _scanItr(),
+      _retryFrozenDocument(false),
+      _shouldCompactLidSpace(false)
+{
+}
+
+bool
+LidSpaceCompactionJob::run()
+{
+    LidUsageStats stats = _handler.getLidStatus();
+    if (_scanItr) {
+        return scanDocuments(stats);
+    } else if (_shouldCompactLidSpace) {
+        compactLidSpace(stats);
+    } else if (hasTooMuchLidBloat(stats)) {
+        assert(!_scanItr);
+        _scanItr = _handler.getIterator();
+        return scanDocuments(stats);
+    }
+    return true;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h
new file mode 100644
index 00000000000..36d01355cd6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "document_db_maintenance_config.h"
+#include "i_lid_space_compaction_handler.h"
+#include "i_maintenance_job.h"
+#include "i_operation_storer.h"
+
+namespace proton {
+
+class IFrozenBucketHandler;
+
+/**
+ * Job that regularly checks whether lid space compaction should be performed
+ * for the given handler.
+ *
+ * Compaction is handled by moving documents from high lids to low free lids.
+ * A handler is typically working over a single document sub db.
+ */
+class LidSpaceCompactionJob : public IMaintenanceJob
+{
+private:
+    const DocumentDBLidSpaceCompactionConfig _cfg;
+    ILidSpaceCompactionHandler &_handler;
+    IOperationStorer           &_opStorer;
+    IFrozenBucketHandler       &_frozenHandler;
+    IDocumentScanIterator::UP   _scanItr;
+    bool                        _retryFrozenDocument;
+    bool                        _shouldCompactLidSpace;
+
+    bool hasTooMuchLidBloat(const search::LidUsageStats &stats) const;
+    bool shouldRestartScanDocuments(const search::LidUsageStats &stats) const;
+    search::DocumentMetaData getNextDocument(const search::LidUsageStats &stats);
+    bool scanDocuments(const search::LidUsageStats &stats);
+    void compactLidSpace(const search::LidUsageStats &stats);
+
+public:
+    LidSpaceCompactionJob(const DocumentDBLidSpaceCompactionConfig &config,
+                          ILidSpaceCompactionHandler &handler,
+                          IOperationStorer &opStorer,
+                          IFrozenBucketHandler &frozenHandler);
+
+    // Implements IMaintenanceJob
+    virtual bool run();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp
new file mode 100644
index 00000000000..3bc7ae7603e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.maintenance_controller_explorer");
+#include "maintenance_controller_explorer.h"
+
+#include <vespa/vespalib/data/slime/cursor.h>
+
+using vespalib::slime::Cursor;
+using vespalib::slime::Inserter;
+
+namespace proton {
+
+namespace {
+
+void
+convertRunningJobsToSlime(const std::vector<MaintenanceJobRunner::SP> &jobs, Cursor &array)
+{
+    for (const auto &jobRunner : jobs) {
+        if (jobRunner->isRunning()) {
+            Cursor &object = array.addObject();
+            object.setString("name", jobRunner->getJob().getName());
+        }
+    }
+}
+
+void
+convertAllJobsToSlime(const std::vector<MaintenanceJobRunner::SP> &jobs, Cursor &array)
+{
+    for (const auto &jobRunner : jobs) {
+        Cursor &object = array.addObject();
+        const IMaintenanceJob &job = jobRunner->getJob();
+        object.setString("name", job.getName());
+        object.setDouble("delay", job.getDelay());
+        object.setDouble("interval", job.getInterval());
+        object.setBool("blocked", job.isBlocked());
+    }
+}
+
+}
+
+MaintenanceControllerExplorer::
+MaintenanceControllerExplorer(std::vector<MaintenanceJobRunner::SP> jobs)
+    : _jobs(std::move(jobs))
+{
+}
+
+void
+MaintenanceControllerExplorer::get_state(const Inserter &inserter, bool full) const
+{
+    Cursor &object = inserter.insertObject();
+    if (full) {
+        convertRunningJobsToSlime(_jobs, object.setArray("runningJobs"));
+        convertAllJobsToSlime(_jobs, object.setArray("allJobs"));
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h
new file mode 100644
index 00000000000..4caa86941e0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "maintenancejobrunner.h"
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of a maintenance controller and its jobs.
+ */
+class MaintenanceControllerExplorer : public vespalib::StateExplorer
+{
+private:
+    std::vector<MaintenanceJobRunner::SP> _jobs;
+
+public:
+    MaintenanceControllerExplorer(std::vector<MaintenanceJobRunner::SP> jobs);
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp
new file mode 100644
index 00000000000..d56fff31bee
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.maintenance_jobs_injector");
+
+#include "heart_beat_job.h"
+#include "job_tracked_maintenance_job.h"
+#include "lid_space_compaction_job.h"
+#include "maintenance_jobs_injector.h"
+#include "prune_session_cache_job.h"
+#include "wipe_old_removed_fields_job.h"
+#include <vespa/fastos/timestamp.h>
+#include "pruneremoveddocumentsjob.h"
+#include "documentdb_commit_job.h"
+#include "documentbucketmover.h"
+#include "bucketmovejob.h"
+#include "sample_attribute_usage_job.h"
+
+using fastos::ClockSystem;
+using fastos::TimeStamp;
+
+namespace proton {
+
+namespace {
+
+IMaintenanceJob::UP
+trackJob(const IJobTracker::SP &tracker,
+         IMaintenanceJob::UP job)
+{
+    return IMaintenanceJob::UP(new JobTrackedMaintenanceJob(tracker, std::move(job)));
+}
+
+void
+injectLidSpaceCompactionJobs(MaintenanceController &controller,
+                             const DocumentDBMaintenanceConfig &config,
+                             const ILidSpaceCompactionHandler::Vector &lscHandlers,
+                             IOperationStorer &opStorer,
+                             IFrozenBucketHandler &fbHandler,
+                             const IJobTracker::SP &tracker)
+{
+    for (auto &lidHandler : lscHandlers) {
+        IMaintenanceJob::UP job = IMaintenanceJob::UP
+                (new LidSpaceCompactionJob(config.getLidSpaceCompactionConfig(),
+                        *lidHandler, opStorer, fbHandler));
+        controller.registerJob(std::move(trackJob(tracker,
+                std::move(job))));
+    }
+}
+
+
+void
+injectBucketMoveJob(MaintenanceController &controller,
+                    IFrozenBucketHandler &fbHandler,
+                    const vespalib::string &docTypeName,
+                    IDocumentMoveHandler &moveHandler,
+                    IBucketModifiedHandler &bucketModifiedHandler,
+                    IClusterStateChangedNotifier &clusterStateChangedNotifier,
+                    IBucketStateChangedNotifier &bucketStateChangedNotifier,
+                    const std::shared_ptr<IBucketStateCalculator> &calc,
+                    DocumentDBJobTrackers &jobTrackers)
+{
+    IMaintenanceJob::UP bmj;
+    bmj.reset(new BucketMoveJob(calc,
+                                moveHandler,
+                                bucketModifiedHandler,
+                                controller.getReadySubDB(),
+                                controller.getNotReadySubDB(),
+                                fbHandler,
+                                clusterStateChangedNotifier,
+                                bucketStateChangedNotifier,
+                                docTypeName));
+    controller.registerJob(std::move(trackJob(jobTrackers.getBucketMove(),
+                                              std::move(bmj))));
+}
+
+}
+
+void
+MaintenanceJobsInjector::injectJobs(MaintenanceController &controller,
+                                    const DocumentDBMaintenanceConfig &config,
+                                    IHeartBeatHandler &hbHandler,
+                                    matching::ISessionCachePruner &scPruner,
+                                    IWipeOldRemovedFieldsHandler &worfHandler,
+                                    const ILidSpaceCompactionHandler::Vector &lscHandlers,
+                                    IOperationStorer &opStorer,
+                                    IFrozenBucketHandler &fbHandler,
+                                    const vespalib::string &docTypeName,
+                                    IPruneRemovedDocumentsHandler &prdHandler,
+                                    IDocumentMoveHandler &moveHandler,
+                                    IBucketModifiedHandler &
+                                    bucketModifiedHandler,
+                                    IClusterStateChangedNotifier &
+                                    clusterStateChangedNotifier,
+                                    IBucketStateChangedNotifier &
+                                    bucketStateChangedNotifier,
+                                    const std::shared_ptr<IBucketStateCalculator> & calc,
+                                    DocumentDBJobTrackers &jobTrackers,
+                                    ICommitable & commit,
+                                    IAttributeManagerSP readyAttributeManager,
+                                    IAttributeManagerSP
+                                    notReadyAttributeManager,
+                                    AttributeUsageFilter &attributeUsageFilter)
+{
+    typedef IMaintenanceJob::UP MUP;
+    controller.registerJob(MUP(new HeartBeatJob(hbHandler, config.getHeartBeatConfig())));
+    controller.registerJob(MUP(new PruneSessionCacheJob(scPruner, config.getSessionCachePruneInterval())));
+    if (config.getVisibilityDelay() > 0) {
+        controller.registerJob(MUP(new DocumentDBCommitJob(commit, config.getVisibilityDelay())));
+    }
+    controller.registerJob(MUP(new WipeOldRemovedFieldsJob(worfHandler, config.getWipeOldRemovedFieldsConfig())));
+    const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB());
+    MUP pruneRDjob(new PruneRemovedDocumentsJob(config.getPruneRemovedDocumentsConfig(), *mRemSubDB._metaStore,
+                                                mRemSubDB._subDbId, docTypeName, prdHandler, fbHandler));
+    controller.registerJob(std::move(trackJob(jobTrackers.getRemovedDocumentsPrune(), std::move(pruneRDjob))));
+    injectLidSpaceCompactionJobs(controller, config, lscHandlers, opStorer,
+                                 fbHandler, jobTrackers.getLidSpaceCompact());
+    injectBucketMoveJob(controller, fbHandler, docTypeName, moveHandler, bucketModifiedHandler,
+                        clusterStateChangedNotifier, bucketStateChangedNotifier, calc, jobTrackers);
+    controller.registerJob(std::make_unique<SampleAttributeUsageJob>
+                           (readyAttributeManager,
+                            notReadyAttributeManager,
+                            attributeUsageFilter,
+                            docTypeName,
+                            config.getAttributeUsageSampleInterval()));
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h
new file mode 100644
index 00000000000..423459af162
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "document_db_maintenance_config.h"
+#include "maintenancecontroller.h"
+#include "i_lid_space_compaction_handler.h"
+#include "i_operation_storer.h"
+#include "iheartbeathandler.h"
+#include "iwipeoldremovedfieldshandler.h"
+#include "icommitable.h"
+#include <vespa/searchcore/proton/matching/isessioncachepruner.h>
+#include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h>
+
+namespace proton {
+
+class IPruneRemovedDocumentsHandler;
+class IDocumentMoveHandler;
+class IBucketModifiedHandler;
+class IClusterStateChangedNotifier;
+class IBucketStateChangedNotifier;
+class IBucketStateCalculator;
+class IAttributeManager;
+class AttributeUsageFilter;
+
+/**
+ * Class that injects all concrete maintenance jobs used in document db
+ * into a MaintenanceController.
+ */
+struct MaintenanceJobsInjector
+{
+    using IAttributeManagerSP = std::shared_ptr<IAttributeManager>;
+    static void injectJobs(MaintenanceController &controller,
+                           const DocumentDBMaintenanceConfig &config,
+                           IHeartBeatHandler &hbHandler,
+                           matching::ISessionCachePruner &scPruner,
+                           IWipeOldRemovedFieldsHandler &worfHandler,
+                           const ILidSpaceCompactionHandler::Vector &lscHandlers,
+                           IOperationStorer &opStorer,
+                           IFrozenBucketHandler &fbHandler,
+                           const vespalib::string &docTypeName,
+                           IPruneRemovedDocumentsHandler &prdHandler,
+                           IDocumentMoveHandler &moveHandler,
+                           IBucketModifiedHandler &bucketModifiedHandler,
+                           IClusterStateChangedNotifier &
+                           clusterStateChangedNotifier,
+                           IBucketStateChangedNotifier &
+                           bucketStateChangedNotifier,
+                           const std::shared_ptr<IBucketStateCalculator> &calc,
+                           DocumentDBJobTrackers &jobTrackers,
+                           ICommitable & commit,
+                           IAttributeManagerSP readyAttributeManager,
+                           IAttributeManagerSP notReadyAttributeManager,
+                           AttributeUsageFilter &attributeUsageFilter);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp
new file mode 100644
index 00000000000..bdd1a46eed4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp
@@ -0,0 +1,225 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.maintenancecontroller");
+
+#include "maintenancecontroller.h"
+#include <vespa/vespalib/util/closuretask.h>
+#include "maintenancejobrunner.h"
+
+using document::BucketId;
+using searchcorespi::index::IThreadService;
+using vespalib::Executor;
+using vespalib::ThreadExecutor;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+
+namespace proton {
+
+namespace {
+
+class JobWrapperTask : public Executor::Task
+{
+private:
+    MaintenanceJobRunner *_job;
+public:
+    JobWrapperTask(MaintenanceJobRunner *job) : _job(job) {}
+    virtual void run() { _job->run(); }
+};
+
+
+}
+
+MaintenanceController::MaintenanceController(IThreadService &masterThread,
+                                             const DocTypeName &docTypeName)
+    : IBucketFreezeListener(),
+      _masterThread(masterThread),
+      _readySubDB(),
+      _remSubDB(),
+      _notReadySubDB(),
+      _periodicTimer(),
+      _config(),
+      _frozenBuckets(masterThread),
+      _started(false),
+      _stopping(false),
+      _docTypeName(docTypeName),
+      _jobs(),
+      _jobsLock()
+{
+    _frozenBuckets.addListener(this); // forward freeze/thaw to bmc
+}
+
+MaintenanceController::~MaintenanceController(void)
+{
+    kill();
+    _frozenBuckets.removeListener(this);
+}
+
+void
+MaintenanceController::registerJob(IMaintenanceJob::UP job)
+{
+    // Called by master write thread
+    Guard guard(_jobsLock);
+    _jobs.push_back(std::make_shared<MaintenanceJobRunner>(_masterThread,
+                                                           std::move(job)));
+}
+
+void
+MaintenanceController::killJobs()
+{
+    // Called by master write thread during start/reconfig
+    // Called by other thread during stop
+    LOG(debug, "killJobs(): threadId=%" PRIu64 "",
+            (uint64_t)FastOS_Thread::GetCurrentThreadId());
+    _periodicTimer.reset();
+    // No need to take _jobsLock as modification of _jobs also happens in master write thread.
+    for (auto &job : _jobs) {
+        job->stop(); // Make sure no more tasks are added to the executor
+    }
+    if (_masterThread.isCurrentThread()) {
+        JobList tmpJobs = _jobs;
+        {
+            Guard guard(_jobsLock);
+            _jobs.clear();
+        }
+        // Hold jobs until existing tasks have been drained
+        _masterThread.execute(makeTask(makeClosure(this,
+                                                   &MaintenanceController::
+                                                   performHoldJobs,
+                                                   tmpJobs)));
+    } else {
+        // Wait for all tasks to be finished.
+        // NOTE: We must sync 2 times as a task currently being executed can add a new
+        // task to the executor as it might not see the new value of the stopped flag.
+        _masterThread.sync();
+        _masterThread.sync();
+        // Clear jobs in master write thread, to avoid races
+        _masterThread.execute(makeTask(makeClosure(this,
+                                                   &MaintenanceController::
+                                                   performClearJobs)));
+        _masterThread.sync();
+    }
+}
+
+void
+MaintenanceController::performHoldJobs(JobList jobs)
+{
+    // Called by master write thread
+    LOG(debug, "performHoldJobs(): threadId=%" PRIu64 "",
+            (uint64_t)FastOS_Thread::GetCurrentThreadId());
+    (void) jobs;
+}
+
+void
+MaintenanceController::performClearJobs()
+{
+    // Called by master write thread
+    LOG(debug, "performClearJobs(): threadId=%" PRIu64 "",
+        (uint64_t)FastOS_Thread::GetCurrentThreadId());
+    Guard guard(_jobsLock);
+    _jobs.clear();
+}
+
+
+void
+MaintenanceController::stop(void)
+{
+    assert(!_masterThread.isCurrentThread());
+    _stopping = true;
+    killJobs();
+}
+
+void
+MaintenanceController::kill()
+{
+    stop();
+    _readySubDB.clear();
+    _remSubDB.clear();
+    _notReadySubDB.clear();
+}
+
+void
+MaintenanceController::start(const DocumentDBMaintenanceConfig::SP &config)
+{
+    // Called by master write thread
+    assert(!_started);
+    _config = config;
+    _started = true;
+    restart();
+}
+
+
+void
+MaintenanceController::restart(void)
+{
+    // Called by master write thread
+    if (!_started || _stopping || !_readySubDB.valid()) {
+        return;
+    }
+    _periodicTimer.reset(new vespalib::Timer());
+
+    addJobsToPeriodicTimer();
+}
+
+void
+MaintenanceController::addJobsToPeriodicTimer()
+{
+    // No need to take _jobsLock as modification of _jobs also happens in master write thread.
+    for (const auto &jw : _jobs) {
+        const IMaintenanceJob &job = jw->getJob();
+        LOG(debug, "addJobsToPeriodicTimer(): docType='%s', job.name='%s', "
+                "job.delay=%f, job.interval=%f",
+                _docTypeName.getName().c_str(), job.getName().c_str(),
+                job.getDelay(), job.getInterval());
+        if (job.getInterval() == 0.0) {
+            jw->run();
+            continue;
+        }
+        _periodicTimer->scheduleAtFixedRate(Executor::Task::UP(new JobWrapperTask(jw.get())),
+                job.getDelay(), job.getInterval());
+    }
+}
+
+void
+MaintenanceController::newConfig(const DocumentDBMaintenanceConfig::SP &config)
+{
+    // Called by master write thread
+    _config = config;
+    restart();
+}
+
+void
+MaintenanceController::syncSubDBs(const MaintenanceDocumentSubDB &readySubDB,
+                                  const MaintenanceDocumentSubDB &remSubDB,
+                                  const MaintenanceDocumentSubDB &
+                                  notReadySubDB)
+{
+    // Called by master write thread
+    bool oldValid = _readySubDB.valid();
+    assert(readySubDB.valid());
+    assert(remSubDB.valid());
+    _readySubDB = readySubDB;
+    _remSubDB = remSubDB;
+    _notReadySubDB = notReadySubDB;
+    if (!oldValid && _started)
+        restart();
+}
+
+
+void
+MaintenanceController::notifyThawedBucket(const BucketId &bucket)
+{
+    (void) bucket;
+    // No need to take _jobsLock as modification of _jobs also happens in master write thread.
+    for (const auto &jw : _jobs) {
+        IMaintenanceJob &job = jw->getJob();
+        if (job.isBlocked()) {
+            job.unBlock();
+            jw->run();
+        }
+    }
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h
new file mode 100644
index 00000000000..d24de856cca
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h
@@ -0,0 +1,97 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "maintenancedocumentsubdb.h"
+#include "documentdbconfig.h"
+#include "i_maintenance_job.h"
+#include <vespa/searchcorespi/index/i_thread_service.h>
+#include <vespa/vespalib/util/random.h>
+#include <vespa/vespalib/util/timer.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
+#include "frozenbuckets.h"
+#include "ibucketfreezelistener.h"
+#include <mutex>
+
+namespace proton
+{
+
+class MaintenanceJobRunner;
+
+
+/**
+ * Class that controls the bucket moving between ready and notready sub databases
+ * and a set of maintenance jobs for a document db.
+ * The maintenance jobs are independent of the controller.
+ */
+class MaintenanceController : public IBucketFreezeListener
+{
+public:
+
+    typedef std::vector<std::shared_ptr<MaintenanceJobRunner>> JobList;
+
+private:
+    using Mutex = std::mutex;
+    using Guard = std::lock_guard<Mutex>;
+
+    searchcorespi::index::IThreadService &_masterThread;
+    MaintenanceDocumentSubDB            _readySubDB;
+    MaintenanceDocumentSubDB            _remSubDB;
+    MaintenanceDocumentSubDB            _notReadySubDB;
+    std::unique_ptr<vespalib::Timer>    _periodicTimer;
+    DocumentDBMaintenanceConfig::SP     _config;
+    FrozenBuckets                       _frozenBuckets;
+    bool                                _started;
+    bool                                _stopping;
+    const DocTypeName                  &_docTypeName;
+    JobList                             _jobs;
+    mutable Mutex                       _jobsLock;
+
+    void addJobsToPeriodicTimer();
+    void restart(void);
+    virtual void notifyThawedBucket(const document::BucketId &bucket) override;
+    void performClearJobs();
+    void performHoldJobs(JobList jobs);
+
+public:
+    typedef std::unique_ptr<MaintenanceController> UP;
+
+    MaintenanceController(searchcorespi::index::IThreadService &masterThread,
+                          const DocTypeName &docTypeName);
+
+    virtual ~MaintenanceController(void);
+    void registerJob(IMaintenanceJob::UP job);
+    void killJobs();
+
+    JobList getJobList() const {
+        Guard guard(_jobsLock);
+        return _jobs;
+    }
+
+    void stop(void);
+    void start(const DocumentDBMaintenanceConfig::SP &config);
+    void newConfig(const DocumentDBMaintenanceConfig::SP &config);
+
+    void
+    syncSubDBs(const MaintenanceDocumentSubDB &readySubDB,
+               const MaintenanceDocumentSubDB &remSubdB,
+               const MaintenanceDocumentSubDB &notReadySubDB);
+
+    void kill();
+
+    operator IBucketFreezer &() { return _frozenBuckets; }
+    operator const IFrozenBucketHandler &() const { return _frozenBuckets; }
+    operator IFrozenBucketHandler &() { return _frozenBuckets; }
+
+    bool  getStarted() const { return _started; }
+    bool getStopping() const { return _stopping; }
+
+    const MaintenanceDocumentSubDB &    getReadySubDB() const { return _readySubDB; }
+    const MaintenanceDocumentSubDB &      getRemSubDB() const { return _remSubDB; }
+    const MaintenanceDocumentSubDB & getNotReadySubDB() const { return _notReadySubDB; }
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancedocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/maintenancedocumentsubdb.h
new file mode 100644
index 00000000000..9154299b6f5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancedocumentsubdb.h
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
+
+namespace proton
+{
+
+/**
+ * The view of a document sub db as seen from the maintenance controller
+ * and various maintenance jobs.
+ */
+class MaintenanceDocumentSubDB
+{
+public:
+    IDocumentMetaStore::SP   _metaStore;
+    IDocumentRetriever::SP   _retriever;
+    uint32_t                 _subDbId;
+
+    MaintenanceDocumentSubDB(void)
+        : _metaStore(),
+          _retriever(),
+          _subDbId(0u)
+    {
+    }
+
+    MaintenanceDocumentSubDB(const IDocumentMetaStore::SP & metaStore,
+                             const IDocumentRetriever::SP & retriever,
+                             uint32_t subDbId)
+        : _metaStore(metaStore),
+          _retriever(retriever),
+          _subDbId(subDbId)
+    {
+    }
+
+    bool valid() const { return bool(_metaStore); }
+
+    void clear() {
+        _metaStore.reset();
+        _retriever.reset();
+        _subDbId = 0u;
+    } 
+};
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp
new file mode 100644
index 00000000000..2f30a96df2a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.maintenancejobrunner");
+#include "maintenancejobrunner.h"
+#include <vespa/vespalib/util/closuretask.h>
+
+
+using vespalib::Executor;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+
+namespace proton
+{
+
+
+void
+MaintenanceJobRunner::run()
+{
+    addExecutorTask();
+}
+
+
+void
+MaintenanceJobRunner::addExecutorTask()
+{
+    if (!_stopped && !_job->isBlocked()) {
+        Guard guard(_lock);
+        if (!_queued) {
+            _queued = true;
+            _executor.execute(makeTask(makeClosure(this,
+                                                   &MaintenanceJobRunner::
+                                                   runJobInExecutor)));
+        }
+    }
+}
+
+void
+MaintenanceJobRunner::runJobInExecutor()
+{
+    {
+        Guard guard(_lock);
+        _queued = false;
+        _running = true;
+    }
+    bool finished = _job->run();
+    if (LOG_WOULD_LOG(debug)) {
+        FastOS_ThreadId threadId = FastOS_Thread::GetCurrentThreadId();
+        LOG(debug,
+            "runJobInExecutor(): job='%s', task='%p', threadId=%" PRIu64 "",
+            _job->getName().c_str(), this, (uint64_t)threadId);
+    }
+    if (!finished) {
+        addExecutorTask();
+    }
+    {
+        Guard guard(_lock);
+        _running = false;
+    }
+}
+
+MaintenanceJobRunner::MaintenanceJobRunner(Executor &executor,
+                                           IMaintenanceJob::UP job)
+    : _executor(executor),
+      _job(std::move(job)),
+      _stopped(false),
+      _queued(false),
+      _running(false),
+      _lock()
+{
+    _job->registerRunner(this);
+}
+
+bool
+MaintenanceJobRunner::isRunning() const
+{
+    Guard guard(_lock);
+    return _running || _queued;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h
new file mode 100644
index 00000000000..a7e50cc3c79
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/threadstackexecutorbase.h>
+#include "i_maintenance_job.h"
+#include "imaintenancejobrunner.h"
+#include <mutex>
+
+namespace proton
+{
+
+class MaintenanceJobRunner : public IMaintenanceJobRunner
+{
+private:
+    using Mutex = std::mutex;
+    using Guard = std::lock_guard<Mutex>;
+
+    vespalib::Executor  &_executor;
+    IMaintenanceJob::UP  _job;
+    bool                 _stopped;
+    bool                 _queued;
+    bool                 _running;
+    mutable Mutex        _lock;
+
+    void addExecutorTask();
+    void runJobInExecutor();
+    
+public:
+    typedef std::shared_ptr<MaintenanceJobRunner> SP;
+
+    MaintenanceJobRunner(vespalib::Executor &executor,
+                         IMaintenanceJob::UP job);
+    virtual void run();
+    void stop() { _stopped = true; }
+    bool isRunning() const;
+    const IMaintenanceJob &getJob() const { return *_job; }
+    IMaintenanceJob &getJob() { return *_job; }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp
new file mode 100644
index 00000000000..0d6f6c95cab
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.matchers");
+#include "matchers.h"
+
+namespace proton {
+
+Matchers::Matchers(const vespalib::Clock & clock, matching::QueryLimiter & queryLimiter) :
+    _rpmap(),
+    _fallback(new matching::Matcher(search::index::Schema(), search::fef::Properties(), clock, queryLimiter, -1)),
+    _default()
+{
+}
+
+void
+Matchers::add(const vespalib::string &name, matching::Matcher::SP matcher)
+{
+    _rpmap[name] = matcher;
+    if (name == "default" || _default.get() == 0) {
+        _default = matcher;
+    }
+}
+
+matching::MatchingStats
+Matchers::getStats() const
+{
+    matching::MatchingStats stats;
+    for (Map::const_iterator it(_rpmap.begin()), mt(_rpmap.end()); it != mt; it++) {
+        stats.add(it->second->getStats());
+    }
+    return stats;
+}
+
+matching::MatchingStats
+Matchers::getStats(const vespalib::string &name) const
+{
+    auto it = _rpmap.find(name);
+    return it != _rpmap.end() ? it->second->getStats() :
+        matching::MatchingStats();
+}
+
+matching::Matcher::SP
+Matchers::lookup(const vespalib::string &name) const
+{
+    Map::const_iterator found(_rpmap.find(name));
+    return (found != _rpmap.end()) ? found->second : _default;
+    //TODO add warning log message when not found, may want to use "_fallback" in most cases here
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.h b/searchcore/src/vespa/searchcore/proton/server/matchers.h
new file mode 100644
index 00000000000..1cc83bf3988
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/matchers.h
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/matching/matcher.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+
+namespace proton {
+
+class Matchers {
+private:
+    typedef vespalib::hash_map<vespalib::string, matching::Matcher::SP> Map;
+    Map                   _rpmap;
+    matching::Matcher::SP _fallback;
+    matching::Matcher::SP _default;
+public:
+    typedef std::shared_ptr<Matchers> SP;
+    typedef std::unique_ptr<Matchers> UP;
+    Matchers(const vespalib::Clock & clock, matching::QueryLimiter & queryLimiter);
+    void add(const vespalib::string &name, matching::Matcher::SP matcher);
+    matching::MatchingStats getStats() const;
+    matching::MatchingStats getStats(const vespalib::string &name) const;
+    matching::Matcher::SP lookup(const vespalib::string &name) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/matchhandlerproxy.cpp
new file mode 100644
index 00000000000..071286c8556
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/matchhandlerproxy.cpp
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.matchhandlerproxy");
+#include "matchhandlerproxy.h"
+
+namespace proton {
+
+MatchHandlerProxy::MatchHandlerProxy(const DocumentDB::SP &documentDB)
+    : _documentDB(documentDB)
+{
+    _documentDB->retain();
+}
+
+
+MatchHandlerProxy::~MatchHandlerProxy(void)
+{
+    _documentDB->release();
+}
+
+
+search::engine::SearchReply::UP
+MatchHandlerProxy::match(const ISearchHandler::SP &searchHandler,
+                         const search::engine::SearchRequest &req,
+                         vespalib::ThreadBundle &threadBundle) const
+{
+    return _documentDB->match(searchHandler, req, threadBundle);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/matchhandlerproxy.h
new file mode 100644
index 00000000000..cde2ded80aa
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/matchhandlerproxy.h
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/matchengine/imatchhandler.h>
+#include "documentdb.h"
+
+namespace proton {
+
+class MatchHandlerProxy : public boost::noncopyable,
+                          public IMatchHandler
+{
+private:
+    DocumentDB::SP _documentDB;
+public:
+    MatchHandlerProxy(const DocumentDB::SP &documentDB);
+
+    virtual
+    ~MatchHandlerProxy(void);
+
+    /**
+     * Implements IMatchHandler.
+     */
+    virtual search::engine::SearchReply::UP
+    match(const ISearchHandler::SP &searchHandler,
+          const search::engine::SearchRequest &req,
+          vespalib::ThreadBundle &threadBundle) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
new file mode 100644
index 00000000000..7873a54cc0c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.matchview");
+#include "matchview.h"
+#include "searchcontext.h"
+#include <vespa/searchcommon/attribute/iattributecontext.h>
+#include <vespa/searchcore/proton/common/indexsearchabletosearchableadapter.h>
+#include <vespa/searchcore/proton/matching/search_session.h>
+#include <vespa/searchlib/queryeval/field_spec.h>
+#include <vespa/searchlib/queryeval/searchable.h>
+
+using proton::matching::MatchContext;
+using proton::matching::SearchSession;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::attribute::IAttributeContext;
+using search::engine::SearchReply;
+using search::engine::SearchRequest;
+using search::queryeval::Blueprint;
+using search::queryeval::FieldSpec;
+using search::queryeval::FieldSpecList;
+using search::queryeval::Searchable;
+using searchcorespi::IndexSearchable;
+
+namespace proton
+{
+
+using matching::ISearchContext;
+using matching::Matcher;
+using matching::SessionManager;
+
+MatchView::MatchView(const Matchers::SP &matchers,
+                     const IndexSearchable::SP &indexSearchable,
+                     const IAttributeManager::SP &attrMgr,
+                     const SessionManager::SP &sessionMgr,
+                     const IDocumentMetaStoreContext::SP &metaStore,
+                     DocIdLimit &docIdLimit)
+    : boost::noncopyable(),
+      _matchers(matchers),
+      _indexSearchable(indexSearchable),
+      _attrMgr(attrMgr),
+      _sessionMgr(sessionMgr),
+      _metaStore(metaStore),
+      _docIdLimit(docIdLimit)
+{
+}
+
+
+Matcher::SP
+MatchView::getMatcher(const vespalib::string & rankProfile) const
+{
+    Matcher::SP retval = _matchers->lookup(rankProfile);
+    if (retval.get() == NULL) {
+        throw std::runtime_error(vespalib::make_string(
+                        "Failed locating Matcher for rank profile '%s'",
+                        rankProfile.c_str()));
+    }
+    LOG(debug, "Rankprofile = %s has termwise_limit=%f", rankProfile.c_str(), retval->get_termwise_limit());
+    return retval;
+}
+
+
+MatchContext::UP MatchView::createContext() const {
+    IAttributeContext::UP attrCtx = _attrMgr->createContext();
+    Searchable::SP indexSearchable(
+            new IndexSearchableToSearchableAdapter(
+                    _indexSearchable, *attrCtx));
+    ISearchContext::UP searchCtx(new SearchContext(indexSearchable,
+                                                   _docIdLimit.get()));
+    return MatchContext::UP(new MatchContext(std::move(attrCtx),
+                                             std::move(searchCtx)));
+}
+
+
+SearchReply::UP
+MatchView::match(const ISearchHandler::SP &searchHandler,
+                 const SearchRequest &req,
+                 vespalib::ThreadBundle &threadBundle) const
+{
+    Matcher::SP matcher = getMatcher(req.ranking);
+    IDocumentMetaStoreContext::IReadGuard::UP guard = _metaStore->getReadGuard();
+    SearchSession::OwnershipBundle owned_objects;
+    owned_objects.search_handler = searchHandler;
+    owned_objects.context = createContext();
+    MatchContext *ctx = owned_objects.context.get();
+    return matcher->match(req,
+                          threadBundle,
+                          ctx->getSearchContext(),
+                          ctx->getAttributeContext(),
+                          *_sessionMgr,
+                          guard->get(),
+                          std::move(owned_objects));
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.h b/searchcore/src/vespa/searchcore/proton/server/matchview.h
new file mode 100644
index 00000000000..45a809dfe14
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.h
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "matchers.h"
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/common/docid_limit.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
+#include <vespa/searchcore/proton/matching/match_context.h>
+#include <vespa/searchcore/proton/matching/matcher.h>
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
+#include <vespa/searchcorespi/index/indexsearchable.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+
+namespace proton {
+
+class MatchView : public boost::noncopyable {
+    Matchers::SP                         _matchers;
+    searchcorespi::IndexSearchable::SP   _indexSearchable;
+    IAttributeManager::SP                _attrMgr;
+    matching::SessionManager::SP         _sessionMgr;
+    IDocumentMetaStoreContext::SP        _metaStore;
+    DocIdLimit                          &_docIdLimit;
+
+    size_t getNumDocs() const {
+        return _metaStore->get().getNumActiveLids();
+    }
+
+public:
+    typedef std::shared_ptr<MatchView> SP;
+
+    MatchView(const Matchers::SP &matchers,
+              const searchcorespi::IndexSearchable::SP &indexSearchable,
+              const IAttributeManager::SP &attrMgr,
+              const matching::SessionManager::SP &sessionMgr,
+              const IDocumentMetaStoreContext::SP &metaStore,
+              DocIdLimit &docIdLimit);
+
+    const Matchers::SP &
+    getMatchers() const
+    {
+        return _matchers;
+    }
+
+    const searchcorespi::IndexSearchable::SP &
+    getIndexSearchable() const
+    {
+        return _indexSearchable;
+    }
+
+    const IAttributeManager::SP &
+    getAttributeManager() const
+    {
+        return _attrMgr;
+    }
+
+    const matching::SessionManager::SP &
+    getSessionManager() const
+    {
+        return _sessionMgr;
+    }
+
+    const IDocumentMetaStoreContext::SP &
+    getDocumentMetaStore() const
+    {
+        return _metaStore;
+    }
+
+    DocIdLimit &
+    getDocIdLimit(void) const
+    {
+        return _docIdLimit;
+    }
+
+    // Throws on error.
+    matching::Matcher::SP
+    getMatcher(const vespalib::string & rankProfile) const;
+
+    matching::MatchingStats
+    getMatcherStats(const vespalib::string &rankProfile) const
+    {
+        return _matchers->getStats(rankProfile);
+    }
+
+    matching::MatchContext::UP createContext() const;
+
+    search::engine::SearchReply::UP
+    match(const ISearchHandler::SP &searchHandler,
+          const search::engine::SearchRequest &req,
+          vespalib::ThreadBundle &threadBundle) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h
new file mode 100644
index 00000000000..c0c0f5aa6ca
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "configstore.h"
+#include "documentdbconfig.h"
+#include <vespa/searchcommon/common/schema.h>
+#include <map>
+
+namespace proton {
+
+struct ConfigMaps {
+    typedef std::shared_ptr<ConfigMaps> SP;
+    std::map<search::SerialNum, DocumentDBConfig::SP> configs;
+    std::map<search::SerialNum,
+             search::index::Schema::SP> histories;
+    std::set<search::SerialNum> _valid;
+};
+
+class MemoryConfigStore : public ConfigStore {
+    ConfigMaps::SP _maps;
+
+public:
+    MemoryConfigStore() : _maps(new ConfigMaps) {}
+    MemoryConfigStore(ConfigMaps::SP maps) : _maps(maps) {}
+
+    virtual SerialNum getBestSerialNum() const {
+        return _maps->_valid.empty() ? 0 : *_maps->_valid.rbegin();
+    }
+    virtual SerialNum getOldestSerialNum() const {
+        return _maps->_valid.empty() ? 0 : *_maps->_valid.begin();
+    }
+    virtual bool hasValidSerial(SerialNum serial) const {
+        return _maps->_valid.find(serial) != _maps->_valid.end();
+    }
+    virtual SerialNum getPrevValidSerial(SerialNum serial) const {
+        if (_maps->_valid.empty() ||
+            *_maps->_valid.begin() >= serial) {
+            return 0;
+        }
+        return *(--(_maps->_valid.lower_bound(serial)));
+    }
+    virtual void saveConfig(const DocumentDBConfig &config,
+                            const search::index::Schema &history,
+                            SerialNum serial) {
+        _maps->configs[serial].reset(new DocumentDBConfig(config));
+        _maps->histories[serial].reset(new search::index::Schema(history));
+        _maps->_valid.insert(serial);
+    }
+    virtual void loadConfig(const DocumentDBConfig &, SerialNum serial,
+                            DocumentDBConfig::SP &loaded_config,
+                            search::index::Schema::SP &history_schema) {
+        assert(hasValidSerial(serial));
+        loaded_config = _maps->configs[serial];
+        history_schema = _maps->histories[serial];
+    }
+    virtual void removeInvalid()
+    {
+        // Note: Depends on C++11 semantics for erase
+        for (auto it = _maps->configs.begin(); it != _maps->configs.end();) {
+            if (!hasValidSerial(it->first)) {
+                it = _maps->configs.erase(it);
+                continue;
+            }
+            ++it;
+        }
+        for (auto it = _maps->histories.begin();
+             it != _maps->histories.end();) {
+            if (!hasValidSerial(it->first)) {
+                it = _maps->histories.erase(it);
+                continue;
+            }
+            ++it;
+        }
+    }
+    void prune(SerialNum serial) {
+        _maps->configs.erase(_maps->configs.begin(),
+                             _maps->configs.upper_bound(serial));
+        _maps->histories.erase(_maps->histories.begin(),
+                               _maps->histories.upper_bound(serial));
+        _maps->_valid.erase(_maps->_valid.begin(),
+                            _maps->_valid.upper_bound(serial));
+    }
+    virtual void saveWipeHistoryConfig(SerialNum serial,
+                                       fastos::TimeStamp wipeTimeLimit)
+    {
+        if (hasValidSerial(serial)) {
+            return;
+        }
+        SerialNum prev = getPrevValidSerial(serial);
+        search::index::Schema::SP schema(new search::index::Schema);
+        if (wipeTimeLimit != 0) {
+            search::index::Schema::SP oldHistorySchema(_maps->histories[prev]);
+            search::index::Schema::UP wipeSchema;
+            wipeSchema = oldHistorySchema->getOldFields(wipeTimeLimit);
+            schema.reset(search::index::Schema::
+                         set_difference(*oldHistorySchema,
+                                        *wipeSchema).release());
+            
+        }
+        _maps->histories[serial] = schema;
+        _maps->configs[serial] = _maps->configs[prev];
+        _maps->_valid.insert(serial);
+    }
+    virtual void serializeConfig(SerialNum, vespalib::nbostream &) {
+        LOG(info, "Serialization of config not implemented.");
+    }
+    virtual void deserializeConfig(SerialNum, vespalib::nbostream &) {
+        assert(!"Not implemented");
+    }
+    virtual void setProtonConfig(const ProtonConfigSP &) override { }
+};
+
+// Holds the state of a set of MemoryConfigStore objects, making stored
+// state available between different instantiations.
+class MemoryConfigStores {
+    std::map<std::string, ConfigMaps::SP> _config_maps;
+
+public:
+    ConfigStore::UP getConfigStore(const std::string &type) {
+        if (!_config_maps[type].get()) {
+            _config_maps[type].reset(new ConfigMaps);
+        }
+        return ConfigStore::UP(new MemoryConfigStore(_config_maps[type]));
+    }
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp
new file mode 100644
index 00000000000..2f26c1a2601
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp
@@ -0,0 +1,249 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.memoryflush");
+
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/searchcore/proton/flushengine/tls_stats_map.h>
+#include "memoryflush.h"
+#include <algorithm>
+
+using search::SerialNum;
+using proton::flushengine::TlsStats;
+
+namespace proton {
+
+namespace {
+
+int64_t
+getSerialDiff(SerialNum localLastSerial, const IFlushTarget &target)
+{
+    return localLastSerial - target.getFlushedSerialNum();
+}
+
+vespalib::string
+getName(const IFlushHandler & handler, const IFlushTarget & target)
+{
+    return (handler.getName() + "." + target.getName());
+}
+
+static constexpr uint64_t gibi = UINT64_C(1024) * UINT64_C(1024) * UINT64_C(1024);
+
+uint64_t
+estimateNeededTlsSizeForFlushTarget(const TlsStats &tlsStats,
+                                    uint32_t flushedSerialNum)
+{
+    if (flushedSerialNum < tlsStats.getFirstSerial()) {
+        return tlsStats.getNumBytes();
+    }
+    int64_t numEntries = tlsStats.getLastSerial() -
+                         tlsStats.getFirstSerial() + 1;
+    if (numEntries <= 0) {
+        return 0u;
+    }
+    if (flushedSerialNum >= tlsStats.getLastSerial()) {
+        return 0u;
+    }
+    double bytesPerEntry = static_cast<double>(tlsStats.getNumBytes()) /
+                           numEntries;
+    return bytesPerEntry * (tlsStats.getLastSerial() - flushedSerialNum);
+}
+
+}
+
+MemoryFlush::Config::Config()
+    : maxGlobalMemory(4000*1024*1024ul),
+      maxGlobalTlsSize(20 * gibi),
+      globalDiskBloatFactor(0.2),
+      maxMemoryGain(1000*1024*1024ul),
+      diskBloatFactor(0.2),
+      maxSerialGain(1*1000*1000ul),
+      maxTimeGain(fastos::TimeStamp::MINUTE*60*24)
+{
+}
+
+
+MemoryFlush::Config::Config(uint64_t maxGlobalMemory_in,
+                            uint64_t maxGlobalTlsSize_in,
+                            double globalDiskBloatFactor_in,
+                            uint64_t maxMemoryGain_in,
+                            double diskBloatFactor_in,
+                            uint64_t maxSerialGain_in,
+                            fastos::TimeStamp maxTimeGain_in)
+    : maxGlobalMemory(maxGlobalMemory_in),
+      maxGlobalTlsSize(maxGlobalTlsSize_in),
+      globalDiskBloatFactor(globalDiskBloatFactor_in),
+      maxMemoryGain(maxMemoryGain_in),
+      diskBloatFactor(diskBloatFactor_in),
+      maxSerialGain(maxSerialGain_in),
+      maxTimeGain(maxTimeGain_in)
+{
+}
+
+MemoryFlush::MemoryFlush(const Config &config,
+                         fastos::TimeStamp startTime)
+    : _lock(),
+      _globalMaxMemory(config.maxGlobalMemory),
+      _maxGlobalTlsSize(config.maxGlobalTlsSize),
+      _globalDiskBloatFactor(config.globalDiskBloatFactor),
+      _maxMemoryGain(config.maxMemoryGain),
+      _diskBloatFactor(config.diskBloatFactor),
+      _maxSerialGain(config.maxSerialGain),
+      _maxTimeGain(config.maxTimeGain),
+      _startTime(startTime)
+{
+}
+
+
+MemoryFlush::MemoryFlush()
+    : MemoryFlush(Config(), fastos::ClockSystem::now())
+{
+    // empty
+}
+
+FlushContext::List
+MemoryFlush::getFlushTargets(const FlushContext::List &targetList,
+                             const flushengine::TlsStatsMap &
+                             tlsStatsMap) const
+{
+    OrderType order(DEFAULT);
+    uint64_t totalMemory(0);
+    IFlushTarget::DiskGain totalDisk;
+    uint64_t totalTlsSize(0);
+    vespalib::hash_set<const void *> visitedHandlers;
+    fastos::TimeStamp now(fastos::ClockSystem::now());
+    LOG(debug,
+        "getFlushTargets(): globalMaxMemory(%" PRIu64 "), globalDiskBloatFactor(%f), "
+        "maxMemoryGain(%" PRIu64 "), diskBloatFactor(%f), maxSerialGain(%" PRIu64 "), maxTimeGain(%f), startTime(%f)",
+        _globalMaxMemory, _globalDiskBloatFactor, _maxMemoryGain, _diskBloatFactor,
+        _maxSerialGain, _maxTimeGain.sec(), _startTime.sec());
+    for (size_t i(0), m(targetList.size()); i < m; i++) {
+        const IFlushTarget & target(*targetList[i]->getTarget());
+        const IFlushHandler & handler(*targetList[i]->getHandler());
+        int64_t mgain(std::max(0l, target.getApproxMemoryGain().gain()));
+        const IFlushTarget::DiskGain dgain(target.getApproxDiskGain());
+        totalDisk += dgain;
+        SerialNum localLastSerial = targetList[i]->getLastSerial();
+        int64_t serialDiff = getSerialDiff(localLastSerial, target);
+        vespalib::string name(getName(handler, target));
+        fastos::TimeStamp lastFlushTime = target.getLastFlushTime();
+        fastos::TimeStamp timeDiff(now - (lastFlushTime.val() > 0 ? lastFlushTime : _startTime));
+        totalMemory += mgain;
+        const flushengine::TlsStats &tlsStats =
+            tlsStatsMap.getTlsStats(handler.getName());
+        if (visitedHandlers.insert(&handler).second) {
+            totalTlsSize += tlsStats.getNumBytes();
+            if ((totalTlsSize > _maxGlobalTlsSize) && (order < TLSSIZE)) {
+                order = TLSSIZE;
+            }
+        }
+        if (((totalMemory >= _globalMaxMemory) ||
+             (mgain >= _maxMemoryGain)) && (order < MEMORY)) {
+            order = MEMORY;
+        } else if (((totalDisk.gain() >
+                     _globalDiskBloatFactor * std::max(100000000l,
+                             std::max(totalDisk.getBefore(),
+                                      totalDisk.getAfter())))
+                      || dgain.gain() > _diskBloatFactor *
+                    std::max(10000000l,
+                             std::max(dgain.getBefore(), dgain.getAfter())))
+                   && (order < DISKBLOAT)
+                  ) {
+            order = DISKBLOAT;
+        } else if ((serialDiff >= _maxSerialGain) && (order < MAXSERIAL)) {
+            order = MAXSERIAL;
+        } else if ((timeDiff >= _maxTimeGain) && (order < MAXAGE)) {
+            order = MAXAGE;
+        }
+        LOG(debug,
+            "getFlushTargets(): target(%s), totalMemoryGain(%" PRIu64 "), memoryGain(%" PRIu64 "), "
+            "totalDiskGain(%" PRId64 "), diskGain(%" PRId64 "), "
+            "tlsSize(%" PRIu64 "), tlsSizeNeeded(%" PRIu64 "), "
+            "flushedSerial(%" PRIu64 "), localLastSerial(%" PRIu64 "), serialDiff(%ld), "
+            "lastFlushTime(%fs), nowTime(%fs), timeDiff(%fs), order(%s)",
+            targetList[i]->getName().c_str(),
+            totalMemory,
+            mgain,
+            totalDisk.gain(),
+            dgain.gain(),
+            tlsStats.getNumBytes(),
+            estimateNeededTlsSizeForFlushTarget(tlsStats,
+                                                target.getFlushedSerialNum()),
+            target.getFlushedSerialNum(),
+            localLastSerial,
+            serialDiff,
+            lastFlushTime.sec(),
+            now.sec(),
+            timeDiff.sec(),
+            (order == DEFAULT ? "DEFAULT" :
+             (order == MEMORY ? "MEMORY" :
+              (order == MAXAGE ? "MAXAGE" :
+               (order == MAXSERIAL ? "MAXSERIAL" : "DISKBLOAT")))));
+    }
+    FlushContext::List fv(targetList);
+    std::sort(fv.begin(), fv.end(), CompareTarget(order, tlsStatsMap));
+    // No desired order and no urgent needs; no flush required at this moment.
+    if (order == DEFAULT &&
+        !fv.empty() &&
+        !fv[0]->getTarget()->needUrgentFlush()) {
+        LOG(debug, "getFlushTargets(): empty list");
+        return FlushContext::List();
+    }
+    if (LOG_WOULD_LOG(debug)) {
+        std::ostringstream oss;
+        for (size_t i = 0; i < fv.size(); ++i) {
+            if (i > 0) {
+                oss << ",";
+            }
+            oss << fv[i]->getName().c_str();
+        }
+        LOG(debug, "getFlushTargets(): %zu sorted targets: [%s]", fv.size(), oss.str().c_str());
+    }
+    return fv;
+}
+
+
+bool
+MemoryFlush::CompareTarget::operator()(const FlushContext::SP &lfc,
+                                       const FlushContext::SP &rfc) const
+{
+    const IFlushTarget &lhs = *lfc->getTarget();
+    const IFlushTarget &rhs = *rfc->getTarget();
+    if (lhs.needUrgentFlush() != rhs.needUrgentFlush()) {
+        return lhs.needUrgentFlush();
+    }
+
+    switch (_order) {
+    case MEMORY:
+        return (lhs.getApproxMemoryGain().gain() >
+                rhs.getApproxMemoryGain().gain());
+    case TLSSIZE: {
+        const flushengine::TlsStats &lhsTlsStats =
+            _tlsStatsMap.getTlsStats(lfc->getHandler()->getName());
+        const flushengine::TlsStats &rhsTlsStats =
+            _tlsStatsMap.getTlsStats(rfc->getHandler()->getName());
+        SerialNum lhsFlushedSerialNum(lhs.getFlushedSerialNum());
+        SerialNum rhsFlushedSerialNum(rhs.getFlushedSerialNum());
+        uint64_t lhsNeededTlsSize =
+            estimateNeededTlsSizeForFlushTarget(lhsTlsStats,
+                                                lhsFlushedSerialNum);
+        uint64_t rhsNeededTlsSize =
+            estimateNeededTlsSizeForFlushTarget(rhsTlsStats,
+                                                rhsFlushedSerialNum);
+        if (lhsNeededTlsSize != rhsNeededTlsSize) {
+            return (lhsNeededTlsSize > rhsNeededTlsSize);
+        }
+        return (lhs.getLastFlushTime() < rhs.getLastFlushTime());
+    }
+    case DISKBLOAT:
+        return (lhs.getApproxDiskGain().gain() >
+                rhs.getApproxDiskGain().gain());
+    case MAXAGE:
+        return (lhs.getLastFlushTime() < rhs.getLastFlushTime());
+    default:
+        return (getSerialDiff(lfc->getLastSerial(), lhs) >
+                getSerialDiff(rfc->getLastSerial(), rhs));
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.h b/searchcore/src/vespa/searchcore/proton/server/memoryflush.h
new file mode 100644
index 00000000000..cb0c84b6364
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.h
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/utility.hpp>
+#include <vespa/searchcore/proton/flushengine/iflushstrategy.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <map>
+
+namespace proton {
+
+class MemoryFlush : public boost::noncopyable,
+                    public IFlushStrategy
+{
+public:
+    struct Config
+    {
+        uint64_t          maxGlobalMemory;
+        uint64_t          maxGlobalTlsSize;
+        double            globalDiskBloatFactor;
+        int64_t           maxMemoryGain;
+        double            diskBloatFactor;
+        int64_t           maxSerialGain;
+        fastos::TimeStamp maxTimeGain;
+        Config();
+        Config(uint64_t maxGlobalMemory_in,
+               uint64_t maxGlobalTlsSize_in,
+               double globalDiskBloatFactor_in,
+               uint64_t maxMemoryGain_in,
+               double diskBloatFactor_in,
+               uint64_t maxSerialGain_in,
+               fastos::TimeStamp maxTimeGain_in);
+    };
+
+private:
+    /// Needed as flushDone is called in different context from the rest
+    vespalib::Lock     _lock;
+    /// Global maxMemory
+    uint64_t           _globalMaxMemory;
+    /// Maximum global tls size.
+    uint64_t           _maxGlobalTlsSize;
+    /// Maximum global disk bloat factor. When this limit is reached
+    /// flush is forced.
+    double             _globalDiskBloatFactor;
+    /// Maximum memory saved. When this limit is reached flush is forced.
+    int64_t            _maxMemoryGain;
+    /// Maximum disk bloat factor. When this limit is reached flush is forced.
+    double             _diskBloatFactor;
+    /// Maximum count of what a target can have outstanding in the TLS.
+    int64_t            _maxSerialGain;
+    /// Maximum age of unflushed data.
+    fastos::TimeStamp  _maxTimeGain;
+    /// The time when the strategy was started.
+    fastos::TimeStamp  _startTime;
+
+    enum OrderType { DEFAULT, MAXAGE, MAXSERIAL, DISKBLOAT, TLSSIZE, MEMORY };
+    class CompareTarget
+    {
+    public:
+        CompareTarget(OrderType order,
+                      const flushengine::TlsStatsMap &tlsStatsMap)
+            : _order(order),
+              _tlsStatsMap(tlsStatsMap)
+        {
+        }
+
+        bool
+        operator ()(const FlushContext::SP &lfc,
+                    const FlushContext::SP &rfc) const;
+    private:
+
+        OrderType     _order;
+        const flushengine::TlsStatsMap &_tlsStatsMap;
+    };
+
+public:
+    MemoryFlush();
+
+    MemoryFlush(const Config &config,
+                fastos::TimeStamp startTime = fastos::TimeStamp(fastos::ClockSystem::now()));
+
+    // Implements IFlushStrategy
+    virtual FlushContext::List
+    getFlushTargets(const FlushContext::List &targetList,
+                    const flushengine::TlsStatsMap &
+                    tlsStatsMap) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp
new file mode 100644
index 00000000000..6c8ef4c30d1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".minimal_document_retriever");
+
+#include "minimal_document_retriever.h"
+#include <vespa/searchlib/docstore/idocumentstore.h>
+
+using document::Document;
+using document::DocumentTypeRepo;
+using search::DocumentIdT;
+using search::IDocumentStore;
+
+namespace proton {
+
+MinimalDocumentRetriever::MinimalDocumentRetriever(
+        const DocTypeName &docTypeName,
+        const DocumentTypeRepo::SP repo,
+        const IDocumentMetaStoreContext &meta_store,
+        const IDocumentStore &doc_store,
+        bool hasFields)
+    : DocumentRetrieverBase(docTypeName, *repo, meta_store, hasFields),
+      _repo(repo),
+      _doc_store(doc_store) {
+}
+
+Document::UP MinimalDocumentRetriever::getDocument(DocumentIdT lid) const {
+    return _doc_store.read(lid, *_repo);
+}
+
+void MinimalDocumentRetriever::visitDocuments(const LidVector & lids, search::IDocumentVisitor & visitor, ReadConsistency) const {
+    _doc_store.visit(lids, getDocumentTypeRepo(), visitor);
+}
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.h
new file mode 100644
index 00000000000..3c0c87b2e57
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentretrieverbase.h"
+#include <vespa/document/repo/documenttyperepo.h>
+
+namespace search { class IDocumentStore; }
+
+namespace proton {
+
+/**
+ * The document retriever used by the store-only sub database.
+ *
+ * Handles retrieving of documents from the underlying document store.
+ */
+class MinimalDocumentRetriever : public DocumentRetrieverBase
+{
+    const document::DocumentTypeRepo::SP _repo;
+    const search::IDocumentStore &_doc_store;
+
+public:
+    // meta_store and doc_store must out-live the MinimalDocumentRetriever.
+    MinimalDocumentRetriever(const DocTypeName &docTypeName,
+                             const document::DocumentTypeRepo::SP repo,
+                             const IDocumentMetaStoreContext &meta_store,
+                             const search::IDocumentStore &doc_store,
+                             bool hasFields);
+
+    document::Document::UP getDocument(search::DocumentIdT lid) const override;
+    void visitDocuments(const LidVector & lids, search::IDocumentVisitor & visitor, ReadConsistency) const override;
+};
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/ooscli.cpp b/searchcore/src/vespa/searchcore/proton/server/ooscli.cpp
new file mode 100644
index 00000000000..1531b90d77e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ooscli.cpp
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.ooscli");
+
+#include "ooscli.h"
+#include "proton.h"
+
+namespace proton {
+
+OosCli::OosCli(const OosParams &params, FRT_Supervisor &orb)
+  : FNET_Task(orb.GetScheduler()),
+    _orb(orb),
+    _params(params),
+    _sbmirror(_orb, params.slobrok_config),
+    _oosmanager(_orb, _sbmirror, params.oos_server_pattern),
+    _curState(-1)
+{
+    Schedule(0.1);
+}
+
+
+OosCli::~OosCli() {
+    Kill(); // unschedule task
+}
+
+void
+OosCli::PerformTask()
+{
+    int old = _curState;
+    if (_oosmanager.isOOS(_params.my_oos_name)) {
+        _params.proton.getMatchEngine().setOutOfService();
+        _curState = 0;
+        if (_curState != old) {
+            LOG(warning, "this search engine (messagebus name '%s') is Out Of Service",
+                _params.my_oos_name.c_str());
+        }
+    } else {
+        if (_params.proton.isReplayDone()) {
+            _params.proton.getMatchEngine().setInService();
+            _curState = 1;
+        }
+        if (_curState != old) {
+            LOG(info, "search engine is In Service, setting online");
+        }
+    }
+    Schedule(1.0);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/ooscli.h b/searchcore/src/vespa/searchcore/proton/server/ooscli.h
new file mode 100644
index 00000000000..0b68049e134
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ooscli.h
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fnet/fnet.h>
+#include <vespa/slobrok/sbmirror.h>
+#include <vespa/slobrok/cfg.h>
+#include <vespa/messagebus/network/oosmanager.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+class Proton;
+
+class OosCli : public FNET_Task
+{
+public:
+    struct OosParams {
+        Proton           &proton;
+        vespalib::string  oos_server_pattern;
+        vespalib::string  my_oos_name;
+        config::ConfigUri slobrok_config;
+
+        OosParams(Proton &p)
+            : proton(p),
+              oos_server_pattern("search/cluster.*/rtx/*/*"),
+              my_oos_name(),
+              slobrok_config(config::ConfigUri("admin/slobrok.0"))
+        {}
+    };
+private:
+    FRT_Supervisor        & _orb;
+    OosParams               _params;
+    slobrok::api::MirrorAPI _sbmirror;
+    mbus::OOSManager        _oosmanager;
+    int                     _curState;
+public:
+    OosCli(const OosParams &params, FRT_Supervisor &orb);
+    virtual ~OosCli();
+    virtual void PerformTask();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp
new file mode 100644
index 00000000000..215f21b66a8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "operationdonecontext.h"
+#include <vespa/searchcore/proton/common/feedtoken.h>
+
+namespace proton
+{
+
+
+OperationDoneContext::OperationDoneContext(std::unique_ptr<FeedToken> token,
+                                           const FeedOperation::Type opType,
+                                           PerDocTypeFeedMetrics &metrics)
+    : _token(std::move(token)),
+      _opType(opType),
+      _metrics(metrics)
+{
+}
+
+
+OperationDoneContext::~OperationDoneContext()
+{
+    ack();
+}
+
+
+void
+OperationDoneContext::ack()
+{
+    if (_token) {
+        std::unique_ptr<FeedToken> token(std::move(_token));
+        token->ack(_opType, _metrics);
+    }
+}
+
+
+bool
+OperationDoneContext::shouldTrace(uint32_t traceLevel)
+{
+    return _token ? _token->shouldTrace(traceLevel) : false;
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.h b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.h
new file mode 100644
index 00000000000..d02bff24615
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.h
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/idestructorcallback.h>
+#include <vespa/searchcore/proton/feedoperation/feedoperation.h>
+
+namespace proton
+{
+
+class PerDocTypeFeedMetrics;
+class FeedToken;
+
+/**
+ * Context class for document operations that acks operation when
+ * instance is destroyed. Typically a shared pointer to an instance is
+ * passed around to multiple worker threads that performs portions of
+ * a larger task before dropping the shared pointer, triggering the
+ * ack when all worker threads have completed.
+ */
+class OperationDoneContext : public search::IDestructorCallback
+{
+    std::unique_ptr<FeedToken> _token;
+    const FeedOperation::Type _opType;
+    PerDocTypeFeedMetrics &_metrics;
+
+protected:
+    void ack();
+
+public:
+    OperationDoneContext(std::unique_ptr<FeedToken> token,
+                         const FeedOperation::Type opType,
+                         PerDocTypeFeedMetrics &metrics);
+
+    virtual ~OperationDoneContext();
+
+    FeedToken *getToken() { return _token.get(); }
+
+    bool shouldTrace(uint32_t traceLevel);
+};
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h b/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h
new file mode 100644
index 00000000000..ff41a1d3328
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "tls_replay_progress.h"
+#include <vespa/searchlib/transactionlog/common.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton {
+/**
+ * Wrapper of transaction log packet to use when handing over to
+ * executor thread.
+ */
+struct PacketWrapper {
+    typedef std::shared_ptr<PacketWrapper> SP;
+
+    const search::transactionlog::Packet &packet;
+    TlsReplayProgress *progress;
+    search::transactionlog::RPC::Result result;
+    vespalib::Gate gate;
+
+    PacketWrapper(const search::transactionlog::Packet &p,
+                  TlsReplayProgress *progress_)
+        : packet(p),
+          progress(progress_),
+          result(search::transactionlog::RPC::ERROR),
+          gate()
+    {
+    }
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp
new file mode 100644
index 00000000000..706e738d8af
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp
@@ -0,0 +1,199 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.persistencehandlerproxy");
+#include "documentretriever.h"
+#include "persistencehandlerproxy.h"
+#include <vespa/persistence/spi/result.h>
+
+using storage::spi::Bucket;
+using storage::spi::Timestamp;
+
+namespace proton {
+
+PersistenceHandlerProxy::PersistenceHandlerProxy(const DocumentDB::SP &documentDB)
+    : _documentDB(documentDB),
+      _feedHandler(_documentDB->getFeedHandler()),
+      _bucketHandler(_documentDB->getBucketHandler()),
+      _clusterStateHandler(_documentDB->getClusterStateHandler())
+{
+    _documentDB->retain();
+}
+
+
+PersistenceHandlerProxy::~PersistenceHandlerProxy()
+{
+    _documentDB->release();
+}
+
+
+void
+PersistenceHandlerProxy::initialize()
+{
+    _documentDB->waitForOnlineState();
+}
+
+
+void
+PersistenceHandlerProxy::handlePut(FeedToken token,
+                                   const Bucket &bucket,
+                                   Timestamp timestamp,
+                                   const document::Document::SP &doc)
+{
+    FeedOperation::UP op(new PutOperation(bucket.getBucketId().stripUnused(),
+                                          timestamp, doc));
+    _feedHandler.handleOperation(token, std::move(op));
+}
+
+
+void
+PersistenceHandlerProxy::handleUpdate(FeedToken token,
+                                      const Bucket &bucket,
+                                      Timestamp timestamp,
+                                      const document::DocumentUpdate::SP &upd)
+{
+    FeedOperation::UP op(new UpdateOperation(bucket.getBucketId().
+                                             stripUnused(),
+                                             timestamp, upd));
+    _feedHandler.handleOperation(token, std::move(op));
+}
+
+
+void
+PersistenceHandlerProxy::handleRemove(FeedToken token,
+                                      const Bucket &bucket,
+                                      Timestamp timestamp,
+                                      const document::DocumentId &id)
+{
+    FeedOperation::UP op(new RemoveOperation(bucket.getBucketId().
+                                             stripUnused(),
+                                             timestamp, id));
+    _feedHandler.handleOperation(token, std::move(op));
+}
+
+
+void
+PersistenceHandlerProxy::handleListBuckets(IBucketIdListResultHandler &resultHandler)
+{
+    _bucketHandler.handleListBuckets(resultHandler);
+}
+
+
+void
+PersistenceHandlerProxy::handleSetClusterState(const storage::spi::ClusterState &calc,
+                                               IGenericResultHandler &resultHandler)
+{
+    _clusterStateHandler.handleSetClusterState(calc, resultHandler);
+}
+
+
+void
+PersistenceHandlerProxy::handleSetActiveState(
+        const storage::spi::Bucket &bucket,
+        storage::spi::BucketInfo::ActiveState newState,
+        IGenericResultHandler &resultHandler)
+{
+    _bucketHandler.handleSetCurrentState(bucket.getBucketId().stripUnused(),
+                                         newState, resultHandler);
+}
+
+
+void
+PersistenceHandlerProxy::handleGetBucketInfo(const Bucket &bucket,
+                                             IBucketInfoResultHandler &resultHandler)
+{
+    _bucketHandler.handleGetBucketInfo(bucket, resultHandler);
+}
+
+
+void
+PersistenceHandlerProxy::handleCreateBucket(FeedToken token,
+                                            const Bucket &bucket)
+{
+    FeedOperation::UP op(new CreateBucketOperation(bucket.getBucketId().
+                                                   stripUnused()));
+    _feedHandler.handleOperation(token, std::move(op));
+}
+
+
+void
+PersistenceHandlerProxy::handleDeleteBucket(FeedToken token,
+                                            const Bucket &bucket)
+{
+    FeedOperation::UP op(new DeleteBucketOperation(bucket.getBucketId().
+                                                   stripUnused()));
+    _feedHandler.handleOperation(token, std::move(op));
+}
+
+
+void
+PersistenceHandlerProxy::handleGetModifiedBuckets(IBucketIdListResultHandler &resultHandler)
+{
+    _clusterStateHandler.handleGetModifiedBuckets(resultHandler);
+}
+
+
+void
+PersistenceHandlerProxy::handleSplit(FeedToken token,
+                                     const Bucket &source,
+                                     const Bucket &target1,
+                                     const Bucket &target2)
+{
+    FeedOperation::UP op(new SplitBucketOperation(source.getBucketId().
+                                                  stripUnused(),
+                                                  target1.getBucketId().
+                                                  stripUnused(),
+                                                  target2.getBucketId().
+                                                  stripUnused()));
+    _feedHandler.handleOperation(token, std::move(op));
+}
+
+
+void
+PersistenceHandlerProxy::handleJoin(FeedToken token,
+                                    const Bucket &source1,
+                                    const Bucket &source2,
+                                    const Bucket &target)
+{
+    FeedOperation::UP op(new JoinBucketsOperation(source1.getBucketId().
+                                                  stripUnused(),
+                                                  source2.getBucketId().
+                                                  stripUnused(),
+                                                  target.getBucketId().
+                                                  stripUnused()));
+    _feedHandler.handleOperation(token, std::move(op));
+}
+
+
+IPersistenceHandler::RetrieversSP
+PersistenceHandlerProxy::getDocumentRetrievers()
+{
+    return _documentDB->getDocumentRetrievers();
+}
+
+BucketGuard::UP
+PersistenceHandlerProxy::lockBucket(const storage::spi::Bucket &bucket)
+{
+    return _documentDB->lockBucket(bucket.getBucketId().stripUnused());
+}
+
+
+void
+PersistenceHandlerProxy::handleListActiveBuckets(
+        IBucketIdListResultHandler &resultHandler)
+{
+    _bucketHandler.handleListActiveBuckets(resultHandler);
+}
+
+
+void
+PersistenceHandlerProxy::handlePopulateActiveBuckets(
+        document::BucketId::List &buckets,
+        IGenericResultHandler &resultHandler)
+{
+    _bucketHandler.handlePopulateActiveBuckets(buckets, resultHandler);
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h
new file mode 100644
index 00000000000..56d16a3144a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/persistenceengine/ipersistencehandler.h>
+#include "documentdb.h"
+
+namespace proton {
+
+class PersistenceHandlerProxy : public boost::noncopyable,
+                                public IPersistenceHandler
+{
+private:
+    DocumentDB::SP       _documentDB;
+    FeedHandler         &_feedHandler;
+    BucketHandler       &_bucketHandler;
+    ClusterStateHandler &_clusterStateHandler;
+public:
+    PersistenceHandlerProxy(const DocumentDB::SP &documentDB);
+
+    virtual ~PersistenceHandlerProxy();
+
+    /**
+     * Implements IPersistenceHandler.
+     */
+    virtual void initialize() override;
+
+    virtual void handlePut(FeedToken token,
+                           const storage::spi::Bucket &bucket,
+                           storage::spi::Timestamp timestamp,
+                           const document::Document::SP &doc) override;
+
+    virtual void handleUpdate(FeedToken token,
+                              const storage::spi::Bucket &bucket,
+                              storage::spi::Timestamp timestamp,
+                              const document::DocumentUpdate::SP &upd) override;
+
+    virtual void handleRemove(FeedToken token,
+                              const storage::spi::Bucket &bucket,
+                              storage::spi::Timestamp timestamp,
+                              const document::DocumentId &id) override;
+
+    virtual void handleListBuckets(IBucketIdListResultHandler &resultHandler) override;
+
+    virtual void handleSetClusterState(const storage::spi::ClusterState &calc,
+                                       IGenericResultHandler &resultHandler) override;
+
+    virtual void handleSetActiveState(const storage::spi::Bucket &bucket,
+                                      storage::spi::BucketInfo::ActiveState newState,
+                                      IGenericResultHandler &resultHandler) override;
+
+    virtual void handleGetBucketInfo(const storage::spi::Bucket &bucket,
+                                     IBucketInfoResultHandler &resultHandler) override;
+
+    virtual void
+    handleCreateBucket(FeedToken token,
+                       const storage::spi::Bucket &bucket) override;
+
+    virtual void handleDeleteBucket(FeedToken token,
+                                    const storage::spi::Bucket &bucket) override;
+
+    virtual void handleGetModifiedBuckets(IBucketIdListResultHandler &resultHandler) override;
+
+    virtual void
+    handleSplit(FeedToken token,
+                const storage::spi::Bucket &source,
+                const storage::spi::Bucket &target1,
+                const storage::spi::Bucket &target2) override;
+
+    virtual void
+    handleJoin(FeedToken token,
+               const storage::spi::Bucket &source,
+               const storage::spi::Bucket &target1,
+               const storage::spi::Bucket &target2) override;
+
+    virtual RetrieversSP getDocumentRetrievers() override;
+    virtual BucketGuard::UP lockBucket(const storage::spi::Bucket &bucket) override;
+
+    virtual void
+    handleListActiveBuckets(IBucketIdListResultHandler &resultHandler) override;
+
+    virtual void
+    handlePopulateActiveBuckets(document::BucketId::List &buckets,
+                                IGenericResultHandler &resultHandler) override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.cpp
new file mode 100644
index 00000000000..32177a99564
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.cpp
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.persistencproviderproxy");
+
+#include "persistenceproviderproxy.h"
+
+using storage::spi::PersistenceProvider;
+
+namespace proton {
+
+PersistenceProviderProxy::PersistenceProviderProxy(PersistenceProvider &pp)
+    : _pp(pp)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.h b/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.h
new file mode 100644
index 00000000000..fab92066456
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.h
@@ -0,0 +1,165 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/persistence/spi/persistenceprovider.h>
+
+namespace proton {
+
+using storage::spi::Bucket;
+using storage::spi::BucketIdListResult;
+using storage::spi::BucketInfoResult;
+using storage::spi::ClusterState;
+using storage::spi::Context;
+using storage::spi::CreateIteratorResult;
+using storage::spi::GetResult;
+using storage::spi::IncludedVersions;
+using storage::spi::IterateResult;
+using storage::spi::IteratorId;
+using storage::spi::PartitionId;
+using storage::spi::PartitionStateListResult;
+using storage::spi::RemoveResult;
+using storage::spi::Result;
+using storage::spi::Selection;
+using storage::spi::Timestamp;
+using storage::spi::UpdateResult;
+
+class PersistenceProviderProxy : public storage::spi::PersistenceProvider
+{
+private:
+    storage::spi::PersistenceProvider &_pp;
+
+public:
+    PersistenceProviderProxy(storage::spi::PersistenceProvider &pp);
+
+    virtual ~PersistenceProviderProxy() {}
+
+    virtual Result initialize() {
+        return _pp.initialize();
+    }
+
+    // Implements PersistenceProvider
+    virtual PartitionStateListResult getPartitionStates() const {
+        return _pp.getPartitionStates();
+    }
+
+    virtual BucketIdListResult listBuckets(PartitionId partId) const {
+        return _pp.listBuckets(partId);
+    }
+
+    virtual Result setClusterState(const ClusterState &state) {
+        return _pp.setClusterState(state);
+    }
+
+    virtual Result setActiveState(const Bucket &bucket,
+                                  storage::spi::BucketInfo::ActiveState newState) {
+        return _pp.setActiveState(bucket, newState);
+    }
+
+    virtual BucketInfoResult getBucketInfo(const Bucket &bucket) const {
+        return _pp.getBucketInfo(bucket);
+    }
+
+    virtual Result put(const Bucket &bucket,
+                       Timestamp timestamp,
+                       const document::Document::SP& doc,
+                       Context& context) {
+        return _pp.put(bucket, timestamp, doc, context);
+    }
+
+    virtual RemoveResult remove(const Bucket &bucket,
+                                Timestamp timestamp,
+                                const document::DocumentId &docId,
+                                Context& context) {
+        return _pp.remove(bucket, timestamp, docId, context);
+    }
+
+    virtual RemoveResult removeIfFound(const Bucket &bucket,
+                                       Timestamp timestamp,
+                                       const document::DocumentId &docId,
+                                       Context& context) {
+        return _pp.removeIfFound(bucket, timestamp, docId, context);
+    }
+
+    virtual UpdateResult update(const Bucket &bucket,
+                                Timestamp timestamp,
+                                const document::DocumentUpdate::SP& docUpd,
+                                Context& context) {
+        return _pp.update(bucket, timestamp, docUpd, context);
+    }
+
+    virtual Result flush(const Bucket &bucket, Context& context) {
+        return _pp.flush(bucket, context);
+    }
+
+    virtual GetResult get(const Bucket &bucket,
+                          const document::FieldSet& fieldSet,
+                          const document::DocumentId &docId,
+                          Context& context) const {
+        return _pp.get(bucket, fieldSet, docId, context);
+    }
+
+    virtual CreateIteratorResult createIterator(const Bucket &bucket,
+                                                const document::FieldSet& fieldSet,
+                                                const Selection &selection,
+                                                IncludedVersions versions,
+                                                Context& context) {
+        return _pp.createIterator(bucket, fieldSet, selection, versions,
+                                  context);
+    }
+
+    virtual IterateResult iterate(IteratorId itrId,
+                                  uint64_t maxByteSize,
+                                  Context& context) const {
+        return _pp.iterate(itrId, maxByteSize, context);
+    }
+
+    virtual Result destroyIterator(IteratorId itrId, Context& context) {
+        return _pp.destroyIterator(itrId, context);
+    }
+
+    virtual Result createBucket(const Bucket &bucket, Context& context) {
+        return _pp.createBucket(bucket, context);
+    }
+
+    virtual Result deleteBucket(const Bucket &bucket, Context& context) {
+        return _pp.deleteBucket(bucket, context);
+    }
+
+    virtual BucketIdListResult getModifiedBuckets() const {
+        return _pp.getModifiedBuckets();
+    }
+
+    virtual Result maintain(const Bucket &bucket,
+                            storage::spi::MaintenanceLevel level) {
+        return _pp.maintain(bucket, level);
+    }
+
+    virtual Result split(const Bucket &source,
+                         const Bucket &target1,
+                         const Bucket &target2,
+                         Context& context) {
+        return _pp.split(source, target1, target2, context);
+    }
+
+    virtual Result join(const Bucket &source1,
+                        const Bucket &source2,
+                        const Bucket &target,
+                        Context& context) {
+        return _pp.join(source1, source2, target, context);
+    }
+
+    virtual Result move(const Bucket &source,
+                        storage::spi::PartitionId target,
+                        Context& context) {
+        return _pp.move(source, target, context);
+    }
+
+    virtual Result removeEntry(const Bucket &bucket,
+                               Timestamp timestamp,
+                               Context& context) {
+        return _pp.removeEntry(bucket, timestamp, context);
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
new file mode 100644
index 00000000000..60c54bf1ef3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -0,0 +1,1108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.proton");
+
+#include "data_directory_upgrader.h"
+#include "disk_mem_usage_sampler.h"
+#include "document_db_explorer.h"
+#include "flushhandlerproxy.h"
+#include "matchhandlerproxy.h"
+#include "memoryflush.h"
+#include "persistencehandlerproxy.h"
+#include "persistenceproviderproxy.h"
+#include "proton.h"
+#include "protonconfigurer.h"
+#include "resource_usage_explorer.h"
+#include "searchhandlerproxy.h"
+#include "simpleflush.h"
+
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/messagebus/emptyreply.h>
+#include <vespa/searchcommon/common/schemaconfigurer.h>
+#include <vespa/searchcore/proton/flushengine/flush_engine_explorer.h>
+#include <vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h>
+#include <vespa/searchcore/proton/flushengine/tls_stats_factory.h>
+#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h>
+#include <vespa/searchlib/aggregation/forcelink.hpp>
+#include <vespa/searchlib/common/packets.h>
+#include <vespa/searchlib/expression/forcelink.hpp>
+#include <vespa/searchlib/transactionlog/trans_log_server_explorer.h>
+#include <vespa/searchlib/util/fileheadertk.h>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/util/random.h>
+
+using document::DocumentTypeRepo;
+using vespalib::FileHeader;
+using vespalib::IllegalStateException;
+using vespalib::LockGuard;
+using vespalib::MonitorGuard;
+using vespalib::RWLockReader;
+using vespalib::RWLockWriter;
+using vespalib::Slime;
+using vespalib::slime::ArrayInserter;
+using vespalib::slime::Cursor;
+using vespalib::slime::ObjectInserter;
+
+using search::TuneFileDocumentDB;
+using search::index::Schema;
+using search::index::SchemaBuilder;
+using search::transactionlog::DomainStats;
+using vespa::config::search::core::ProtonConfig;
+using vespa::config::search::core::internal::InternalProtonType;
+using document::CompressionConfig;
+using searchcorespi::IIndexManagerFactory;
+
+namespace proton {
+
+namespace {
+
+using search::fs4transport::FS4PersistentPacketStreamer;
+
+CompressionConfig::Type
+convert(InternalProtonType::Packetcompresstype type)
+{
+    switch (type) {
+      case InternalProtonType::LZ4: return CompressionConfig::LZ4;
+      default: return CompressionConfig::LZ4;
+    }
+}
+
+void
+setFS4Compression(const ProtonConfig & proton)
+{
+    FS4PersistentPacketStreamer & fs4(FS4PersistentPacketStreamer::Instance);
+    fs4.SetCompressionLimit(proton.packetcompresslimit);
+    fs4.SetCompressionLevel(proton.packetcompresslevel);
+    fs4.SetCompressionType(convert(proton.packetcompresstype));
+}
+
+DiskMemUsageSampler::Config
+diskMemUsageSamplerConfig(const ProtonConfig &proton)
+{
+    return DiskMemUsageSampler::Config(
+            proton.writefilter.memorylimit,
+            proton.writefilter.disklimit,
+            proton.writefilter.sampleinterval);
+}
+
+}
+
+static const size_t TOTAL_HARD_MEMORY_LIMIT=16*1024*1024*1024ul;
+static const size_t EACH_HARD_MEMORY_LIMIT=12*1024*1024*1024ul;
+static const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component";
+
+Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const Proton &proton_,
+        const vespalib::string &creator)
+    : _proton(proton_),
+      _hostName(),
+      _creator(creator),
+      _cluster(),
+      _pid(getpid())
+{
+    _hostName = FastOS_Socket::getHostName();
+    assert(!_hostName.empty());
+}
+
+
+void
+Proton::ProtonFileHeaderContext::addTags(vespalib::GenericHeader &header,
+        const vespalib::string &name) const
+{
+    typedef vespalib::GenericHeader::Tag Tag;
+
+    search::FileHeaderTk::addVersionTags(header);
+    header.putTag(Tag("fileName", name));
+    addCreateAndFreezeTime(header);
+    header.putTag(Tag("hostName", _hostName));
+    header.putTag(Tag("pid", _pid));
+    header.putTag(Tag("creator", _creator));
+    if (!_cluster.empty()) {
+        header.putTag(Tag("cluster", _cluster));
+    }
+}
+
+
+void
+Proton::ProtonFileHeaderContext::setClusterName(const vespalib::string &
+                                                clusterName,
+                                                const vespalib::string &
+                                                baseDir)
+{
+    if (!clusterName.empty()) {
+        _cluster = clusterName;
+        return;
+    }
+    // Derive cluster name from base dir.
+    size_t cpos(baseDir.rfind('/'));
+    if (cpos == vespalib::string::npos)
+        return;
+    size_t rpos(baseDir.rfind('/', cpos - 1));
+    if (rpos == vespalib::string::npos)
+        return;
+    size_t clpos(baseDir.rfind('/', rpos - 1));
+    if (clpos == vespalib::string::npos)
+        return;
+    if (baseDir.substr(clpos + 1, 8) != "cluster.")
+        return;
+    _cluster = baseDir.substr(clpos + 9, rpos - clpos - 9);
+}
+
+
+Proton::Proton(const config::ConfigUri & configUri,
+               const vespalib::string &progName,
+               uint64_t subscribeTimeout)
+    : IBootstrapOwner(),
+      search::engine::MonitorServer(),
+      IDocumentDBOwner(),
+      StatusProducer(),
+      PersistenceProviderFactory(),
+      IPersistenceEngineOwner(),
+      ComponentConfigProducer(),
+      _protonConfigurer(configUri, this, subscribeTimeout),
+      _configUri(configUri),
+      _lock(),
+      _metricsHook(*this),
+      _metricsEngine(),
+      _fileHeaderContext(*this, progName),
+      _tls(),
+      _diskMemUsageSampler(),
+      _persistenceEngine(),
+      _persistenceProxy(),
+      _documentDBMap(),
+      _matchEngine(),
+      _summaryEngine(),
+      _docsumBySlime(),
+      _flushEngine(),
+      _rpcHooks(),
+      _healthAdapter(*this),
+      _componentConfig(),
+      _genericStateHandler(CUSTOM_COMPONENT_API_PATH, *this),
+      _customComponentBindToken(),
+      _customComponentRootToken(),
+      _stateServer(),
+      _fs4Server(),
+      // This executor can only have 1 thread as it is used for
+      // serializing startup.
+      _executor(1, 128 * 1024),
+      _warmupExecutor(),
+      _summaryExecutor(),
+      _allowReconfig(false),
+      _initialProtonConfig(),
+      _activeConfigSnapshot(),
+      _activeConfigSnapshotGeneration(0),
+      _pendingConfigSnapshot(),
+      _configLock(),
+      _queryLimiter(),
+      _clock(0.010),
+      _threadPool(128 * 1024),
+      _libraries(),
+      _indexManagerFactoryRegistry(),
+      _configGenMonitor(),
+      _configGen(0),
+      _distributionKey(-1),
+      _isInitializing(true),
+      _isReplayDone(false),
+      _abortInit(false),
+      _initStarted(false),
+      _initComplete(false),
+      _initDocumentDbsInSequence(false)
+{
+}
+
+BootstrapConfig::SP
+Proton::init()
+{
+    assert( ! _initStarted && ! _initComplete );
+    _initStarted = true;
+    if (_threadPool.NewThread(&_clock, NULL) == NULL) {
+        throw IllegalStateException("Failed starting thread for the cheap clock");
+    }
+    _protonConfigurer.start();
+    BootstrapConfig::SP configSnapshot = _pendingConfigSnapshot.get();
+    assert(configSnapshot.get() != NULL);
+
+    const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
+
+    if (!performDataDirectoryUpgrade(protonConfig.basedir)) {
+        _abortInit = true;
+    }
+    return configSnapshot;
+}
+
+void
+Proton::init(const BootstrapConfig::SP & configSnapshot)
+{
+    assert( _initStarted && ! _initComplete );
+    const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
+    setFS4Compression(protonConfig);
+    _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>
+                           (protonConfig.basedir,
+                            diskMemUsageSamplerConfig(protonConfig));
+
+    _initialProtonConfig.reset(new ProtonConfig(protonConfig));
+    _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("proton",
+                                       configSnapshot->getGeneration(),
+                                       "config obtained at startup"));
+
+    _metricsEngine.reset(new MetricsEngine());
+    _metricsEngine->addMetricsHook(_metricsHook);
+    _fileHeaderContext.setClusterName(protonConfig.clustername,
+                                      protonConfig.basedir);
+    _tls.reset(new TLS(_configUri.createWithNewId(protonConfig.tlsconfigid), _fileHeaderContext));
+    _matchEngine.reset(new MatchEngine(protonConfig.numsearcherthreads,
+                                       protonConfig.numthreadspersearch,
+                                       protonConfig.distributionkey));
+    _distributionKey = protonConfig.distributionkey;
+    _summaryEngine.reset(new SummaryEngine(protonConfig.numsummarythreads));
+    _docsumBySlime.reset(new DocsumBySlime(*_summaryEngine));
+    IFlushStrategy::SP strategy;
+    const ProtonConfig::Flush & flush(protonConfig.flush);
+    switch (flush.strategy) {
+    case ProtonConfig::Flush::MEMORY: {
+            size_t totalMaxMemory = flush.memory.maxmemory;
+            if (totalMaxMemory > TOTAL_HARD_MEMORY_LIMIT) {
+                LOG(warning, "flush.memory.maxmemory=%ld can not"
+                             " be set above the hard limit of %ld so we cap it",
+                             flush.memory.maxmemory,
+                             TOTAL_HARD_MEMORY_LIMIT);
+                totalMaxMemory = TOTAL_HARD_MEMORY_LIMIT;
+            }
+            size_t eachMaxMemory = flush.memory.each.maxmemory;
+            if (eachMaxMemory > EACH_HARD_MEMORY_LIMIT) {
+                LOG(warning, "flush.memory.each.maxmemory=%ld can not"
+                             " be set above the hard limit of %ld so we cap it",
+                             flush.memory.maxmemory,
+                             EACH_HARD_MEMORY_LIMIT);
+                eachMaxMemory = EACH_HARD_MEMORY_LIMIT;
+            }
+            strategy = std::make_shared<MemoryFlush>(
+                    MemoryFlush::Config(totalMaxMemory,
+                                        flush.memory.maxtlssize,
+                                        flush.memory.diskbloatfactor,
+                                        eachMaxMemory,
+                                        flush.memory.each.diskbloatfactor,
+                                        flush.memory.maxage.serial,
+                                        static_cast<long>
+                                        (flush.memory.maxage.time) *
+                                        fastos::TimeStamp::NANO));
+        }
+        break;
+    case ProtonConfig::Flush::SIMPLE:
+    default:
+        strategy.reset(new SimpleFlush());
+        break;
+    }
+    vespalib::mkdir(protonConfig.basedir + "/documents", true);
+    vespalib::chdir(protonConfig.basedir);
+    _tls->start();
+    _flushEngine.reset(new FlushEngine(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()),
+                                       strategy, flush.maxconcurrent, flush.idleinterval*1000, true));
+    _fs4Server.reset(new TransportServer(*_matchEngine, *_summaryEngine, *this, protonConfig.ptport, TransportServer::DEBUG_ALL));
+    _fs4Server->setTCPNoDelay(true);
+    _metricsEngine->addExternalMetrics(_fs4Server->getMetrics());
+
+    char tmp[1024];
+    LOG(debug, "Start proton server with root at %s and cwd at %s",
+        protonConfig.basedir.c_str(), getcwd(tmp, sizeof(tmp)));
+
+    _persistenceEngine.reset(new PersistenceEngine(*this,
+                                                   _diskMemUsageSampler->writeFilter(),
+                                                   protonConfig.visit.defaultserializedsize,
+                                                   protonConfig.visit.ignoremaxbytes));
+
+
+    vespalib::string fileConfigId;
+    _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024));
+    _summaryExecutor.reset(new vespalib::ThreadStackExecutor(protonConfig.summary.log.numthreads, 128*1024));
+    InitializeThreads initializeThreads;
+    if (protonConfig.initialize.threads > 0) {
+        initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>
+                            (protonConfig.initialize.threads, 128 * 1024);
+        _initDocumentDbsInSequence = (protonConfig.initialize.threads == 1);
+    }
+    applyConfig(configSnapshot, initializeThreads);
+    initializeThreads.reset();
+
+    if (_persistenceEngine.get() != NULL) {
+        _persistenceProxy.reset(new ProviderStub(protonConfig.
+                                                 persistenceprovider.port,
+                                                 protonConfig.
+                                                 persistenceprovider.threads,
+                                                 *configSnapshot->
+                                                 getDocumentTypeRepoSP(),
+                                                 *this));
+    }
+
+    RPCHooks::Params rpcParams(*this, protonConfig.rpcport, _configUri.getConfigId());
+    rpcParams.slobrok_config = _configUri.createWithNewId(protonConfig.slobrokconfigid);
+    _rpcHooks.reset(new RPCHooks(rpcParams));
+
+    waitForInitDone();
+
+    _metricsEngine->start(_configUri);
+    _stateServer.reset(new vespalib::StateServer(protonConfig.httpport, _healthAdapter, _metricsEngine->metrics_producer(), *this));
+    _customComponentBindToken = _stateServer->repo().bind(CUSTOM_COMPONENT_API_PATH, _genericStateHandler);
+    _customComponentRootToken = _stateServer->repo().add_root_resource(CUSTOM_COMPONENT_API_PATH);
+
+    _executor.sync();
+    waitForOnlineState();
+    _isReplayDone = true;
+    bool startOk = _fs4Server->start();
+    int port = _fs4Server->getListenPort();
+    _matchEngine->setOnline();
+    _matchEngine->setInService();
+    LOG(debug,
+        "Started fs4 interface (startOk=%s, port=%d)",
+        startOk ? "true" : "false",
+        port);
+    _flushEngine->start();
+    _allowReconfig = true;
+    _isInitializing = false;
+    _executor.execute(
+            vespalib::makeTask(
+                    vespalib::makeClosure(
+                            this,
+                            &proton::Proton::performReconfig)));
+    _initComplete = true;
+}
+
+bool
+Proton::performDataDirectoryUpgrade(const vespalib::string &baseDir)
+{
+    // TODO: Remove this functionality when going to Vespa 6.
+    vespalib::string scanDir = baseDir.substr(0, baseDir.rfind('/'));
+    LOG(debug, "About to perform data directory upgrade: scanDir='%s', destDir='%s'",
+        scanDir.c_str(), baseDir.c_str());
+    DataDirectoryUpgrader upgrader(scanDir, baseDir);
+    DataDirectoryUpgrader::ScanResult scanResult = upgrader.scan();
+    DataDirectoryUpgrader::UpgradeResult upgradeResult = upgrader.upgrade(scanResult);
+    if (upgradeResult.getStatus() == DataDirectoryUpgrader::ERROR) {
+        LOG(error, "Data directory upgrade failed: '%s'. Please consult Vespa release notes on how to manually fix this issue. "
+            "The search node will not start until this issue has been fixed", upgradeResult.getDesc().c_str());
+        return false;
+    } else if (upgradeResult.getStatus() == DataDirectoryUpgrader::IGNORE) {
+        LOG(debug, "Data directory upgrade ignored: %s", upgradeResult.getDesc().c_str());
+    } else if (upgradeResult.getStatus() == DataDirectoryUpgrader::COMPLETE) {
+        LOG(info, "Data directory upgrade completed: %s", upgradeResult.getDesc().c_str());
+    }
+    return true;
+}
+
+void
+Proton::loadLibrary(const vespalib::string &libName)
+{
+    searchcorespi::IIndexManagerFactory::SP factory(_libraries.create(libName));
+    if (factory.get() != NULL) {
+        LOG(info, "Successfully created index manager factory from library '%s'", libName.c_str());
+        _indexManagerFactoryRegistry.add(libName, factory);
+    } else {
+        LOG(error, "Failed creating index manager factory from library '%s'", libName.c_str());
+    }
+}
+
+searchcorespi::IIndexManagerFactory::SP
+Proton::getIndexManagerFactory(const vespalib::stringref & name) const
+{
+    return _indexManagerFactoryRegistry.get(name);
+}
+
+BootstrapConfig::SP
+Proton::getActiveConfigSnapshot() const
+{
+    BootstrapConfig::SP result;
+    {
+        LockGuard guard(_configLock);
+        result = _activeConfigSnapshot;
+    }
+    return result;
+}
+
+storage::spi::PersistenceProvider::UP
+Proton::create() const
+{
+    //TODO : Might be an idea to grab a lock here as this is not
+    //controlled by you.  Must lock with add/remove documentdb or
+    //reconfig or whatever.
+    if (_persistenceEngine.get() == NULL)
+        return storage::spi::PersistenceProvider::UP();
+    return storage::spi::PersistenceProvider::
+        UP(new PersistenceProviderProxy(*_persistenceEngine));
+}
+
+void
+Proton::reconfigure(const BootstrapConfig::SP & config)
+{
+    _pendingConfigSnapshot.set(config);
+    {
+        LockGuard guard(_configLock);
+        if (_activeConfigSnapshot.get() == NULL)
+            return;
+        if (!_allowReconfig)
+            return;
+        _executor.execute(
+                vespalib::makeTask(
+                        vespalib::makeClosure(
+                                this,
+                                &proton::Proton::performReconfig)));
+    }
+}
+
+void
+Proton::performReconfig()
+{
+    // Called by executor thread
+    BootstrapConfig::SP configSnapshot = _pendingConfigSnapshot.get();
+    bool generationChanged = false;
+    bool snapChanged = false;
+    {
+        LockGuard guard(_configLock);
+        if (_activeConfigSnapshotGeneration != configSnapshot->getGeneration())
+            generationChanged = true;
+        if (_activeConfigSnapshot.get() != configSnapshot.get()) {
+            snapChanged = true;
+        } else if (generationChanged) {
+            _activeConfigSnapshotGeneration = configSnapshot->getGeneration();
+        }
+    }
+    if (snapChanged) {
+        applyConfig(configSnapshot, InitializeThreads());
+    }
+    _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("proton.documentdbs",
+                                       _activeConfigSnapshotGeneration));
+    if (_initialProtonConfig) {
+        if (configSnapshot->getProtonConfig() == *_initialProtonConfig) {
+            _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("proton",
+                                       configSnapshot->getGeneration(),
+                                       "config same as on startup"));
+        } else {
+            LOG(debug, "cannot apply proton.cfg generation %ld, differs from initial config",
+                configSnapshot->getGeneration());
+        }
+    }
+}
+
+void
+Proton::applyConfig(const BootstrapConfig::SP & configSnapshot,
+                    InitializeThreads initializeThreads)
+{
+    // Called by executor thread during reconfig.
+    const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
+    setFS4Compression(protonConfig);
+
+    _queryLimiter.configure(protonConfig.search.memory.limiter.maxthreads,
+                            protonConfig.search.memory.limiter.mincoverage,
+                            protonConfig.search.memory.limiter.minhits);
+    typedef std::set<DocTypeName> DocTypeSet;
+    DocTypeSet oldDocTypes;
+    {
+        RWLockReader guard(_lock);
+        for (const auto &kv : _documentDBMap) {
+            const DocTypeName &docTypeName = kv.first;
+            oldDocTypes.insert(docTypeName);
+        }
+    }
+    DocTypeSet newDocTypes;
+    const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP();
+    // XXX: This assumes no feeding during reconfig.  Otherwise queued messages
+    // might incorrectly use freed document type repo.
+    if (_persistenceProxy.get() != NULL) {
+        _persistenceProxy->setRepo(*repo);
+    }
+    for (const auto &ddbConfig : protonConfig.documentdb) {
+        DocTypeName docTypeName(ddbConfig.inputdoctypename);
+        newDocTypes.insert(docTypeName);
+        DocTypeSet::const_iterator found(oldDocTypes.find(docTypeName));
+        if (found == oldDocTypes.end()) {
+            addDocumentDB(docTypeName, ddbConfig.configid, configSnapshot,
+                          initializeThreads);
+        }
+    }
+    for (const auto &docType : oldDocTypes) {
+        DocTypeSet::const_iterator found(newDocTypes.find(docType));
+        if (found != newDocTypes.end())
+            continue;
+        // remove old document type
+        DocTypeName docTypeName(docType.getName());
+        removeDocumentDB(docTypeName);
+    }
+    {
+        LockGuard guard(_configLock);
+        _activeConfigSnapshot = configSnapshot;
+        _activeConfigSnapshotGeneration = configSnapshot->getGeneration();
+    }
+    _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("proton.documentdbs",
+                                       configSnapshot->getGeneration()));
+    _diskMemUsageSampler->
+        setConfig(diskMemUsageSamplerConfig(protonConfig));
+}
+
+void
+Proton::addDocumentDB(const DocTypeName & docTypeName,
+                      const vespalib::string & configId,
+                      const BootstrapConfig::SP & configSnapshot,
+                      InitializeThreads initializeThreads)
+{
+    try {
+        const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP();
+        const document::DocumentType *docType = repo->getDocumentType(docTypeName.getName());
+        if (docType != NULL) {
+            LOG(info,
+                "Add document database: "
+                "doctypename(%s), configid(%s)",
+                docTypeName.toString().c_str(),
+                configId.c_str());
+            addDocumentDB(*docType, configSnapshot, initializeThreads);
+        } else {
+            LOG(warning,
+                "Did not find document type '%s' in the document manager. "
+                "Skipping creating document database for this type",
+                docTypeName.toString().c_str());
+        }
+    } catch (const document::DocumentTypeNotFoundException & e) {
+        LOG(warning,
+            "Did not find document type '%s' in the document manager. "
+            "Skipping creating document database for this type",
+            docTypeName.toString().c_str());
+    }
+}
+
+bool Proton::addExtraConfigs(DocumentDBConfigManager & dbCfgMan)
+{
+    (void) dbCfgMan;
+    return false;
+}
+
+Proton::~Proton()
+{
+    assert(_initStarted);
+    if ( ! _initComplete ) {
+        LOG(warning, "Initialization of proton was halted. Shutdown sequence has been initiated.");
+    }
+    _protonConfigurer.close();
+    {
+        LockGuard guard(_configLock);
+        _allowReconfig = false;
+    }
+    _executor.sync();
+    _customComponentRootToken.reset();
+    _customComponentBindToken.reset();
+    _stateServer.reset();
+    if (_metricsEngine.get() != NULL) {
+        _metricsEngine->removeMetricsHook(_metricsHook);
+        _metricsEngine->stop();
+    }
+    if (_matchEngine.get() != NULL) {
+        _matchEngine->close();
+    }
+    if (_summaryEngine.get() != NULL) {
+        _summaryEngine->close();
+    }
+    if (_rpcHooks.get() != NULL) {
+        _rpcHooks->close();
+    }
+    _executor.shutdown();
+    _executor.sync();
+    _rpcHooks.reset();
+    if (_flushEngine.get() != NULL) {
+        _flushEngine->close();
+    }
+    if (_warmupExecutor.get() != NULL) {
+        _warmupExecutor->sync();
+    }
+    if (_summaryExecutor.get() != NULL) {
+        _summaryExecutor->sync();
+    }
+    LOG(debug, "Shutting down fs4 interface");
+    if (_metricsEngine.get() != NULL) {
+        _metricsEngine->removeExternalMetrics(_fs4Server->getMetrics());
+    }
+    if (_fs4Server.get() != NULL) {
+        _fs4Server->shutDown();
+    }
+    _persistenceProxy.reset();
+    while (!_documentDBMap.empty()) {
+        const DocTypeName docTypeName(_documentDBMap.begin()->first);
+        removeDocumentDB(docTypeName);
+    }
+    _documentDBMap.clear();
+    _persistenceEngine.reset();
+    _tls.reset();
+    _warmupExecutor.reset();
+    _summaryExecutor.reset();
+    _clock.stop();
+    LOG(debug, "Explicit destructor done");
+}
+
+size_t Proton::getNumDocs() const
+{
+    size_t numDocs(0);
+    RWLockReader guard(_lock);
+    for (const auto &kv : _documentDBMap) {
+        numDocs += kv.second->getNumDocs();
+    }
+    return numDocs;
+}
+
+size_t Proton::getNumActiveDocs() const
+{
+    size_t numDocs(0);
+    RWLockReader guard(_lock);
+    for (const auto &kv : _documentDBMap) {
+        numDocs += kv.second->getNumActiveDocs();
+    }
+    return numDocs;
+}
+
+
+vespalib::string
+Proton::getBadConfigs(void) const
+{
+    std::ostringstream res;
+    bool first = true;
+    RWLockReader guard(_lock);
+    for (const auto &kv : _documentDBMap) {
+        if (kv.second->getRejectedConfig()) {
+            if (!first) {
+                res << ", ";
+            }
+            first = false;
+            res << kv.first.toString();
+        }
+    }
+    return res.str();
+}
+
+StatusReport::List
+Proton::getStatusReports() const
+{
+    StatusReport::List reports;
+    RWLockReader guard(_lock);
+    reports.push_back(StatusReport::SP(_matchEngine->
+                                       reportStatus().release()));
+    for (const auto &kv : _documentDBMap) {
+        reports.push_back(StatusReport::SP(kv.second->
+                                  reportStatus().release()));
+    }
+    return reports;
+}
+
+
+DocumentDB::SP
+Proton::getDocumentDB(const document::DocumentType &docType)
+{
+    RWLockReader guard(_lock);
+    DocTypeName docTypeName(docType.getName());
+    DocumentDBMap::iterator it = _documentDBMap.find(docTypeName);
+    if (it != _documentDBMap.end()) {
+        return it->second;
+    }
+    return DocumentDB::SP();
+}
+
+DocumentDB::SP
+Proton::addDocumentDB(const document::DocumentType &docType,
+                      const BootstrapConfig::SP &configSnapshot,
+                      InitializeThreads initializeThreads)
+{
+    const ProtonConfig &config(*configSnapshot->getProtonConfigSP());
+
+    RWLockWriter guard(_lock);
+    DocTypeName docTypeName(docType.getName());
+    DocumentDBMap::iterator it = _documentDBMap.find(docTypeName);
+    if (it != _documentDBMap.end()) {
+        return it->second;
+    }
+
+    DocumentDBConfig::SP dbConfig =
+        _protonConfigurer.getDocumentDBConfig(docTypeName);
+    vespalib::string db_dir = config.basedir + "/documents/" + docTypeName.toString();
+    vespalib::mkdir(db_dir, false); // Assume parent is created.
+    ConfigStore::UP config_store(
+            new FileConfigManager(db_dir + "/config",
+                                  dbConfig->getConfigId(),
+                                  docTypeName.getName()));
+    config_store->setProtonConfig(configSnapshot->getProtonConfigSP());
+    if (!initializeThreads) {
+        // If configured value for initialize threads was 0, or we
+        // are performing a reconfig after startup has completed, then use
+        // 1 thread per document type.
+        initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>
+                            (1, 128 * 1024);
+    }
+    DocumentDB::SP ret(new DocumentDB(config.basedir + "/documents",
+                                      dbConfig,
+                                      config.tlsspec,
+                                      _queryLimiter,
+                                      _clock,
+                                      docTypeName,
+                                      config,
+                                      *this,
+                                      *_warmupExecutor,
+                                      *_summaryExecutor,
+                                      _tls->getTransLogServer().get(),
+                                      *_metricsEngine,
+                                      _fileHeaderContext,
+                                      std::move(config_store),
+                                      initializeThreads));
+    _protonConfigurer.registerDocumentDB(docTypeName, ret.get());
+    try {
+        ret->start();
+    } catch (vespalib::Exception &e) {
+        LOG(warning,
+            "Failed to start database for document type '%s'; %s",
+            docTypeName.toString().c_str(),
+            e.what());
+        return DocumentDB::SP();
+    }
+    // Wait for replay done on document dbs added due to reconfigs, since engines are already up and running.
+    // Also wait for document db reaching online state if initializing in sequence.
+    if (!_isInitializing || _initDocumentDbsInSequence) {
+        ret->waitForOnlineState();
+    }
+    _metricsEngine->addDocumentDBMetrics(ret->getMetricsCollection());
+    _metricsEngine->addMetricsHook(ret->getMetricsUpdateHook());
+    _documentDBMap[docTypeName] = ret;
+    if (_persistenceEngine.get() != NULL) {
+        // Not allowed to get to service layer to call pause().
+        RWLockWriter persistenceWGuard(_persistenceEngine->getWLock());
+        PersistenceHandlerProxy::SP
+            persistenceHandler(new PersistenceHandlerProxy(ret));
+        if (!_isInitializing) {
+            _persistenceEngine->
+                propagateSavedClusterState(*persistenceHandler);
+            _persistenceEngine->populateInitialBucketDB(*persistenceHandler);
+        }
+        // TODO: Fix race with new cluster state setting.
+        _persistenceEngine->putHandler(docTypeName, persistenceHandler);
+    }
+    SearchHandlerProxy::SP searchHandler(new SearchHandlerProxy(ret));
+    _summaryEngine->putSearchHandler(docTypeName, searchHandler);
+    _matchEngine->putSearchHandler(docTypeName, searchHandler);
+    FlushHandlerProxy::SP flushHandler(new FlushHandlerProxy(ret));
+    _flushEngine->putFlushHandler(docTypeName, flushHandler);
+    return ret;
+}
+
+
+void
+Proton::removeDocumentDB(const DocTypeName &docTypeName)
+{
+    DocumentDB::SP old;
+    _protonConfigurer.unregisterDocumentDB(docTypeName);
+    {
+        RWLockWriter guard(_lock);
+        DocumentDBMap::iterator it = _documentDBMap.find(docTypeName);
+        if (it == _documentDBMap.end())
+            return;
+        old = it->second;
+        _documentDBMap.erase(it);
+    }
+
+    // Remove all entries into document db
+    if (_persistenceEngine) {
+        {
+            // Not allowed to get to service layer to call pause().
+            RWLockWriter persistenceWguard(_persistenceEngine->getWLock());
+            IPersistenceHandler::SP oldHandler;
+            oldHandler = _persistenceEngine->removeHandler(docTypeName);
+            if (_initComplete && oldHandler) {
+                // TODO: Fix race with bucket db modifying ops.
+                _persistenceEngine->grabExtraModifiedBuckets(*oldHandler);
+            }
+        }
+        _persistenceEngine->destroyIterators();
+    }
+    _matchEngine->removeSearchHandler(docTypeName);
+    _summaryEngine->removeSearchHandler(docTypeName);
+    _flushEngine->removeFlushHandler(docTypeName);
+    _metricsEngine->removeMetricsHook(old->getMetricsUpdateHook());
+    _metricsEngine->removeDocumentDBMetrics(old->getMetricsCollection());
+    // Caller should have removed & drained relevant timer tasks
+    old->close();
+}
+
+
+Proton::MonitorReply::UP
+Proton::ping(MonitorRequest::UP request, MonitorClient & client)
+{
+    (void) client;
+    MonitorReply::UP reply(new MonitorReply());
+    MonitorReply &ret = *reply;
+
+    BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot();
+    const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
+    ret.partid = protonConfig.partition;
+    if (_matchEngine->isOnline())
+        ret.timestamp = 42; // change to flush caches on tld/qrs
+    else
+        ret.timestamp = 0;
+    ret.activeDocs = getNumActiveDocs();
+    ret.activeDocsRequested = request->reportActiveDocs;
+    return reply;
+}
+
+bool
+Proton::triggerFlush()
+{
+    if ((_flushEngine.get() == NULL) || ! _flushEngine->HasThread()) {
+        return false;
+    }
+    _flushEngine->triggerFlush();
+    return true;
+}
+
+namespace {
+
+PrepareRestartFlushStrategy::Config
+createPrepareRestartConfig(const ProtonConfig &protonConfig)
+{
+    return PrepareRestartFlushStrategy::Config(protonConfig.flush.preparerestart.replaycost,
+            protonConfig.flush.preparerestart.writecost);
+}
+
+}
+
+bool
+Proton::prepareRestart()
+{
+    if ((_flushEngine.get() == NULL) || ! _flushEngine->HasThread()) {
+        return false;
+    }
+    BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot();
+    IFlushStrategy::SP strategy =
+            std::make_shared<PrepareRestartFlushStrategy>(
+                    createPrepareRestartConfig(configSnapshot->getProtonConfig()));
+    _flushEngine->setStrategy(strategy);
+    return false;
+}
+
+void
+Proton::wipeHistory()
+{
+    DocumentDBMap dbs;
+    {
+        RWLockReader guard(_lock);
+        dbs = _documentDBMap;
+    }
+    for (const auto &kv : dbs) {
+        kv.second->wipeHistory();
+    }
+}
+
+
+void
+Proton::listDocTypes(std::vector<vespalib::string> &documentTypes)
+{
+    DocumentDBMap dbs;
+    {
+        RWLockReader guard(_lock);
+        dbs = _documentDBMap;
+    }
+    for (const auto &kv : dbs) {
+        vespalib::string documentType;
+        const DocTypeName &docTypeName =
+            kv.second->getDocTypeName();
+        documentTypes.push_back(docTypeName.getName());
+    }
+}
+
+
+void
+Proton::listSchema(const vespalib::string &documentType,
+                   std::vector<vespalib::string> &fieldNames,
+                   std::vector<vespalib::string> &fieldDataTypes,
+                   std::vector<vespalib::string> &fieldCollectionTypes,
+                   std::vector<vespalib::string> &fieldLocations)
+{
+    DocumentDB::SP ddb;
+    DocTypeName docTypeName(documentType);
+    {
+        RWLockReader guard(_lock);
+        DocumentDBMap::const_iterator it = _documentDBMap.find(docTypeName);
+        if (it != _documentDBMap.end())
+            ddb = it->second;
+    }
+    if (ddb.get() == NULL)
+        return;
+    ddb->listSchema(fieldNames, fieldDataTypes, fieldCollectionTypes,
+                    fieldLocations);
+}
+
+
+void
+Proton::updateMetrics(const metrics::MetricLockGuard &)
+{
+    {
+        ContentProtonMetrics &metrics = _metricsEngine->root();
+        metrics.transactionLog.update(_tls->getTransLogServer()->getDomainStats());
+        const DiskMemUsageFilter &usageFilter = _diskMemUsageSampler->writeFilter();
+        metrics.resourceUsage.disk.set(usageFilter.getDiskUsedRatio());
+        metrics.resourceUsage.memory.set(usageFilter.getMemoryUsedRatio());
+        metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0));
+    }
+    {
+        LegacyProtonMetrics &metrics = _metricsEngine->legacyRoot();
+        metrics.executor.update(_executor.getStats());
+        metrics.flushExecutor.update(_flushEngine->getExecutorStats());
+        metrics.matchExecutor.update(_matchEngine->getExecutorStats());
+        metrics.summaryExecutor.update(_summaryEngine->getExecutorStats());
+    }
+}
+
+namespace {
+const std::string config_id_tag = "CONFIG ID";
+}  // namespace
+
+void
+Proton::waitForInitDone()
+{
+    RWLockReader guard(_lock);
+    for (const auto &kv : _documentDBMap) {
+        kv.second->waitForInitDone();
+    }
+}
+
+void
+Proton::waitForOnlineState()
+{
+    RWLockReader guard(_lock);
+    for (const auto &kv : _documentDBMap) {
+        kv.second->waitForOnlineState();
+    }
+}
+
+void
+Proton::getComponentConfig(Consumer &consumer)
+{
+    _componentConfig.getComponentConfig(consumer);
+    std::vector<DocumentDB::SP> dbs;
+    {
+        RWLockReader guard(_lock);
+        for (const auto &kv : _documentDBMap) {
+            dbs.push_back(kv.second);
+        }
+    }
+    for (const auto &docDb : dbs) {
+        vespalib::string name("proton.documentdb.");
+        name.append(docDb->getDocTypeName().getName());
+        int64_t gen = docDb->getActiveGeneration();
+        if (docDb->getRejectedConfig()) {
+            consumer.add(Config(name, gen, "has rejected config"));
+        } else {
+            consumer.add(Config(name, gen));
+        }
+    }
+}
+
+int64_t
+Proton::getConfigGeneration(void)
+{
+    int64_t g = 0;
+    std::vector<DocumentDB::SP> dbs;
+    {
+        LockGuard guard(_configLock);
+        g = _activeConfigSnapshot->getGeneration();
+    }
+    {
+        RWLockReader guard(_lock);
+        for (const auto &kv : _documentDBMap) {
+            dbs.push_back(kv.second);
+        }
+    }
+    for (const auto &docDb : dbs) {
+        int64_t ddbActiveGen = docDb->getActiveGeneration();
+        g = std::min(g, ddbActiveGen);
+    }
+    return g;
+}
+
+
+void
+Proton::setClusterState(const storage::spi::ClusterState &calc)
+{
+    // forward info sent by cluster controller to persistence engine
+    // about whether node is supposed to be up or not.  Match engine
+    // needs to know this in order to stop serving queries.
+    bool nodeUp(calc.nodeUp());
+    _matchEngine->setNodeUp(nodeUp);
+}
+
+namespace {
+
+const vespalib::string MATCH_ENGINE = "matchengine";
+const vespalib::string DOCUMENT_DB = "documentdb";
+const vespalib::string FLUSH_ENGINE = "flushengine";
+const vespalib::string TLS_NAME = "tls";
+const vespalib::string RESOURCE_USAGE = "resourceusage";
+
+struct StateExplorerProxy : vespalib::StateExplorer {
+    const StateExplorer &explorer;
+    explicit StateExplorerProxy(const StateExplorer &explorer_in) : explorer(explorer_in) {}
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override { explorer.get_state(inserter, full); }
+    virtual std::vector<vespalib::string> get_children_names() const override { return explorer.get_children_names(); }
+    virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override { return explorer.get_child(name); }
+};
+
+struct DocumentDBMapExplorer : vespalib::StateExplorer {
+    typedef std::map<DocTypeName, DocumentDB::SP> DocumentDBMap;
+    const DocumentDBMap &documentDBMap;
+    vespalib::RWLock &lock;
+    DocumentDBMapExplorer(const DocumentDBMap &documentDBMap_in, vespalib::RWLock &lock_in)
+        : documentDBMap(documentDBMap_in), lock(lock_in) {}
+    virtual void get_state(const vespalib::slime::Inserter &, bool) const override {}
+    virtual std::vector<vespalib::string> get_children_names() const override {
+        RWLockReader guard(lock);
+        std::vector<vespalib::string> names;
+        for (const auto &item: documentDBMap) {
+            names.push_back(item.first.getName());
+        }
+        return names;
+    }
+    virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override {
+        typedef std::unique_ptr<StateExplorer> Explorer_UP;
+        RWLockReader guard(lock);
+        auto result = documentDBMap.find(DocTypeName(vespalib::string(name)));
+        if (result == documentDBMap.end()) {
+            return Explorer_UP(nullptr);
+        }
+        return Explorer_UP(new DocumentDBExplorer(result->second));
+    }
+};
+
+} // namespace proton::<unnamed>
+
+void
+Proton::get_state(const vespalib::slime::Inserter &, bool) const
+{
+}
+
+std::vector<vespalib::string>
+Proton::get_children_names() const
+{
+    std::vector<vespalib::string> names({DOCUMENT_DB, MATCH_ENGINE, FLUSH_ENGINE, TLS_NAME, RESOURCE_USAGE});
+    return names;
+}
+
+std::unique_ptr<vespalib::StateExplorer>
+Proton::get_child(vespalib::stringref name) const
+{
+    typedef std::unique_ptr<StateExplorer> Explorer_UP;
+    if (name == MATCH_ENGINE && _matchEngine) {
+        return std::make_unique<StateExplorerProxy>(*_matchEngine);
+    } else if (name == DOCUMENT_DB) {
+        return std::make_unique<DocumentDBMapExplorer>(_documentDBMap, _lock);
+    } else if (name == FLUSH_ENGINE && _flushEngine) {
+        return std::make_unique<FlushEngineExplorer>(*_flushEngine);
+    } else if (name == TLS_NAME && _tls) {
+        return std::make_unique<search::transactionlog::TransLogServerExplorer>(_tls->getTransLogServer());
+    } else if (name == RESOURCE_USAGE && _diskMemUsageSampler) {
+        return std::make_unique<ResourceUsageExplorer>(_diskMemUsageSampler->writeFilter());
+    }
+    return Explorer_UP(nullptr);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
new file mode 100644
index 00000000000..023a6173984
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -0,0 +1,266 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "health_adapter.h"
+#include "idocumentdbowner.h"
+#include <vespa/persistence/proxy/providerstub.h>
+#include <vespa/searchcore/proton/flushengine/flushengine.h>
+#include <vespa/searchcore/proton/matchengine/matchengine.h>
+#include <vespa/searchcore/proton/metrics/metrics_engine.h>
+#include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
+#include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h>
+#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h>
+#include <vespa/searchcore/proton/server/bootstrapconfigmanager.h>
+#include <vespa/searchcore/proton/server/documentdb.h>
+#include <vespa/searchcore/proton/server/protonconfigurer.h>
+#include <vespa/searchcore/proton/server/rpc_hooks.h>
+#include <vespa/searchcore/proton/summaryengine/summaryengine.h>
+#include <vespa/searchcore/proton/summaryengine/docsum_by_slime.h>
+#include <vespa/searchcorespi/plugin/factoryloader.h>
+#include <vespa/searchcorespi/plugin/factoryregistry.h>
+#include <vespa/searchlib/common/fileheadercontext.h>
+#include <vespa/searchlib/engine/monitorapi.h>
+#include <vespa/searchlib/engine/transportserver.h>
+#include <vespa/searchlib/transactionlog/translogserverapp.h>
+#include <vespa/vespalib/net/component_config_producer.h>
+#include <vespa/vespalib/net/generic_state_handler.h>
+#include <vespa/vespalib/net/json_get_handler.h>
+#include <vespa/vespalib/net/simple_component_config_producer.h>
+#include <vespa/vespalib/net/state_explorer.h>
+#include <vespa/vespalib/net/state_server.h>
+#include <vespa/vespalib/util/rwlock.h>
+#include <vespa/vespalib/util/varholder.h>
+
+namespace proton
+{
+
+class DiskMemUsageSampler;
+
+class Proton : public IBootstrapOwner,
+               public search::engine::MonitorServer,
+               public IDocumentDBOwner,
+               public StatusProducer,
+               public storage::spi::ProviderStub::PersistenceProviderFactory,
+               public IPersistenceEngineOwner,
+               public vespalib::ComponentConfigProducer,
+               public vespalib::StateExplorer
+{
+private:
+    typedef search::transactionlog::TransLogServerApp     TLS;
+    typedef search::engine::TransportServer               TransportServer;
+    typedef search::engine::MonitorRequest                MonitorRequest;
+    typedef search::engine::MonitorReply                  MonitorReply;
+    typedef search::engine::MonitorClient                 MonitorClient;
+    typedef search::docsummary::JuniperProperties         JuniperProperties;
+    typedef storage::spi::ProviderStub                    ProviderStub;
+    typedef std::map<DocTypeName, DocumentDB::SP>         DocumentDBMap;
+    typedef BootstrapConfig::ProtonConfigSP               ProtonConfigSP;
+    typedef std::shared_ptr<FastOS_DynamicLibrary>      DynamicLibrarySP;
+    typedef std::map<vespalib::string, DynamicLibrarySP>  LibraryMap;
+    using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
+
+    struct MetricsUpdateHook : metrics::MetricManager::UpdateHook
+    {
+        Proton &self;
+        MetricsUpdateHook(Proton &s)
+            : metrics::MetricManager::UpdateHook("proton-hook"),
+              self(s) {}
+        void updateMetrics(const MetricLockGuard &guard) override { self.updateMetrics(guard); }
+    };
+    friend struct MetricsUpdateHook;
+
+    class ProtonFileHeaderContext : public search::common::FileHeaderContext
+    {
+        const Proton &_proton;
+        vespalib::string _hostName;
+        vespalib::string _creator;
+        vespalib::string _cluster;
+        pid_t _pid;
+
+    public:
+        ProtonFileHeaderContext(const Proton &proton_,
+                                const vespalib::string &creator);
+
+        virtual void
+        addTags(vespalib::GenericHeader &header,
+                const vespalib::string &name) const;
+
+        void
+        setClusterName(const vespalib::string &clusterName,
+                       const vespalib::string &baseDir);
+    };
+
+    config::IConfigContext::SP      _configContext;
+    ProtonConfigurer                _protonConfigurer;
+    const config::ConfigUri         _configUri;
+    vespalib::string                _dbFile;
+    mutable vespalib::RWLock        _lock;
+    MetricsUpdateHook               _metricsHook;
+    MetricsEngine::UP               _metricsEngine;
+    ProtonFileHeaderContext         _fileHeaderContext;
+    TLS::UP                         _tls;
+    std::unique_ptr<DiskMemUsageSampler> _diskMemUsageSampler;
+    PersistenceEngine::UP           _persistenceEngine;
+    ProviderStub::UP                _persistenceProxy;
+    DocumentDBMap                   _documentDBMap;
+    MatchEngine::UP                 _matchEngine;
+    SummaryEngine::UP               _summaryEngine;
+    DocsumBySlime::UP               _docsumBySlime;
+    FlushEngine::UP                 _flushEngine;
+    RPCHooks::UP                    _rpcHooks;
+    HealthAdapter                   _healthAdapter;
+    vespalib::SimpleComponentConfigProducer _componentConfig;
+    vespalib::GenericStateHandler   _genericStateHandler;
+    vespalib::JsonHandlerRepo::Token::UP _customComponentBindToken;
+    vespalib::JsonHandlerRepo::Token::UP _customComponentRootToken;
+    vespalib::StateServer::UP       _stateServer;
+    TransportServer::UP             _fs4Server;
+    vespalib::ThreadStackExecutor   _executor;
+    std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor;
+    std::unique_ptr<vespalib::ThreadStackExecutorBase> _summaryExecutor;
+    bool                            _allowReconfig;
+    ProtonConfig::UP                _initialProtonConfig;
+    BootstrapConfig::SP             _activeConfigSnapshot;
+    int64_t                         _activeConfigSnapshotGeneration;
+    vespalib::VarHolder<BootstrapConfig::SP> _pendingConfigSnapshot;
+    vespalib::Lock                  _configLock;	
+    matching::QueryLimiter          _queryLimiter;
+    vespalib::Clock                 _clock;
+    FastOS_ThreadPool               _threadPool;
+    searchcorespi::FactoryLoader    _libraries;
+    searchcorespi::FactoryRegistry  _indexManagerFactoryRegistry;
+    vespalib::Monitor               _configGenMonitor;
+    int64_t                         _configGen;
+    uint32_t                        _distributionKey;
+    bool                            _isInitializing;
+    bool                            _isReplayDone;
+    bool                            _abortInit;
+    bool                            _initStarted;
+    bool                            _initComplete;
+    bool                            _initDocumentDbsInSequence;
+
+    bool performDataDirectoryUpgrade(const vespalib::string &baseDir);
+    void loadLibrary(const vespalib::string &libName);
+    // Override from ProtonConfigManager
+    virtual void reconfigure(const BootstrapConfig::SP & config);
+
+    // Called by executor task to handle serialized reconfig.
+    void performReconfig();
+
+    void applyConfig(const BootstrapConfig::SP & configSnapshot,
+                     InitializeThreads initializeThreads);
+    void addDocumentDB(const DocTypeName & docTypeName,
+                       const vespalib::string & configid, 
+                       const BootstrapConfig::SP & configSnapshot,
+                       InitializeThreads initializeThreads);
+
+    virtual MonitorReply::UP ping(MonitorRequest::UP request, MonitorClient &client);
+
+    /**
+     * Called by the metrics update hook (typically in the context of
+     * the metric manager). Do not call this function in multiple
+     * threads at once.
+     **/
+    void updateMetrics(const metrics::MetricLockGuard &guard);
+
+    void waitForInitDone();
+    void waitForOnlineState();
+    virtual storage::spi::PersistenceProvider::UP create() const override;
+    virtual bool addExtraConfigs(DocumentDBConfigManager & dbCfgMan) override;
+    searchcorespi::IIndexManagerFactory::SP
+    getIndexManagerFactory(const vespalib::stringref & name) const override;
+    uint32_t getDistributionKey() const override { return _distributionKey; }
+    BootstrapConfig::SP getActiveConfigSnapshot() const;
+
+public:
+    typedef std::unique_ptr<Proton> UP;
+    typedef std::shared_ptr<Proton> SP;
+
+    Proton(const config::ConfigUri & configUri,
+           const vespalib::string &progName,
+           uint64_t subscribeTimeout);
+    virtual ~Proton();
+
+    /**
+     * This method must be called after the constructor and before the destructor.
+     * If not I will force a 'core' upon you.
+     * All relevant initialization is conducted here.
+     *
+     * 1st phase init: start cheap clock thread and get initial config
+     */
+    BootstrapConfig::SP
+    init();
+
+    /*
+     * 2nd phase init: setup data structures.
+     */
+    void
+    init(const BootstrapConfig::SP & configSnapshot);
+
+
+    DocumentDB::SP
+    getDocumentDB(const document::DocumentType &docType);
+
+    DocumentDB::SP
+    addDocumentDB(const document::DocumentType &docType,
+                  const BootstrapConfig::SP &configSnapshot,
+                  InitializeThreads initializeThreads);
+
+    void removeDocumentDB(const DocTypeName &docTypeName);
+
+    metrics::MetricManager & getMetricManager() { return _metricsEngine->getManager(); }
+    FastOS_ThreadPool & getThreadPool() { return _threadPool; }
+
+    bool triggerFlush();
+    bool prepareRestart();
+    void wipeHistory();
+    void listDocTypes(std::vector<vespalib::string> &documentTypes);
+
+    void
+    listSchema(const vespalib::string &documentType,
+               std::vector<vespalib::string> &fieldNames,
+               std::vector<vespalib::string> &fieldDataTypes,
+               std::vector<vespalib::string> &fieldCollectionTypes,
+               std::vector<vespalib::string> &fieldLocations);
+
+
+    // implements ComponentConfigProducer interface
+    virtual void getComponentConfig(Consumer &consumer) override;
+
+    // implements IPersistenceEngineOwner interface
+    virtual void setClusterState(const storage::spi::ClusterState &calc);
+
+    /**
+     * Return the oldest active config generation used by proton.
+     */
+    int64_t getConfigGeneration(void);
+
+    size_t getNumDocs() const;
+    size_t getNumActiveDocs() const;
+    DocsumBySlime & getDocsumBySlime() { return *_docsumBySlime; }
+
+    vespalib::string getBadConfigs(void) const;
+
+    virtual StatusReport::List getStatusReports() const;
+
+    MatchEngine & getMatchEngine() { return *_matchEngine; }
+    FlushEngine & getFlushEngine() { return *_flushEngine; }
+    vespalib::ThreadStackExecutorBase & getExecutor() { return _executor; }
+
+    bool isReplayDone() const { return _isReplayDone; }
+
+    virtual bool isInitializing() const {
+        return _isInitializing;
+    }
+
+    bool hasAbortedInit() const { return _abortInit; }
+    storage::spi::PersistenceProvider & getPersistence() { return *_persistenceEngine; }
+
+    // Implements vespalib::StateExplorer
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+    virtual std::vector<vespalib::string> get_children_names() const override;
+    virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/protonconfigurer.cpp b/searchcore/src/vespa/searchcore/proton/server/protonconfigurer.cpp
new file mode 100644
index 00000000000..58848af55cc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/protonconfigurer.cpp
@@ -0,0 +1,208 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.protonconfigurer");
+#include <vespa/vespalib/util/exceptions.h>
+#include "protonconfigurer.h"
+
+using namespace vespa::config::search;
+using namespace vespa::config::search::core;
+using namespace config;
+
+namespace proton {
+
+ProtonConfigurer::ProtonConfigurer(const config::ConfigUri & configUri, IBootstrapOwner * owner, uint64_t subscribeTimeout)
+    : _bootstrapConfigManager(configUri.getConfigId()),
+      _retriever(_bootstrapConfigManager.createConfigKeySet(), configUri.getContext(), subscribeTimeout),
+      _bootstrapOwner(owner),
+      _lock(),
+      _dbManagerMap(),
+      _documentDBOwnerMap(),
+      _threadPool(128 * 1024, 1)
+{
+}
+
+ProtonConfigurer::~ProtonConfigurer()
+{
+    close();
+}
+
+void
+ProtonConfigurer::Run(FastOS_ThreadInterface * thread, void *arg)
+{
+    (void) arg;
+    (void) thread;
+    while (!_retriever.isClosed()) {
+        fetchConfigs();
+    }
+}
+
+const ConfigKeySet
+ProtonConfigurer::pruneManagerMap(const BootstrapConfig::SP & config)
+{
+    const ProtonConfig & protonConfig = config->getProtonConfig();
+    DBManagerMap newMap;
+    ConfigKeySet set;
+
+    vespalib::LockGuard guard(_lock);
+    for (size_t i = 0; i < protonConfig.documentdb.size(); i++) {
+        const ProtonConfig::Documentdb & ddb(protonConfig.documentdb[i]);
+        DocTypeName docTypeName(ddb.inputdoctypename);
+        LOG(debug, "Document type(%s), configid(%s)", ddb.inputdoctypename.c_str(), ddb.configid.c_str());
+        DocumentDBConfigManager::SP mgr;
+        if (_dbManagerMap.find(docTypeName) != _dbManagerMap.end()) {
+            mgr = _dbManagerMap[docTypeName];
+        } else {
+            mgr = DocumentDBConfigManager::SP(new DocumentDBConfigManager
+                    (ddb.configid, docTypeName.getName()));
+        }
+        set.add(mgr->createConfigKeySet());
+        newMap[docTypeName] = mgr;
+    }
+    std::swap(_dbManagerMap, newMap);
+    return set;
+}
+
+void
+ProtonConfigurer::reconfigureBootstrap(const ConfigSnapshot & snapshot)
+{
+    assert(_bootstrapOwner != NULL);
+    _bootstrapConfigManager.update(snapshot);
+    _bootstrapOwner->reconfigure(_bootstrapConfigManager.getConfig());
+}
+
+bool
+ProtonConfigurer::updateDocumentDBConfigs(const BootstrapConfig::SP & bootstrapConfig, const ConfigSnapshot & snapshot)
+{
+    vespalib::LockGuard guard(_lock);
+    bool modifiedConfigKeySet(false);
+    for (DBManagerMap::iterator it(_dbManagerMap.begin()), mt(_dbManagerMap.end());
+         it != mt;
+         it++) {
+        it->second->forwardConfig(bootstrapConfig);
+        it->second->update(snapshot);
+        modifiedConfigKeySet = modifiedConfigKeySet || _bootstrapOwner->addExtraConfigs(*it->second);
+    }
+    return modifiedConfigKeySet;
+}
+
+void
+ProtonConfigurer::reconfigureDocumentDBs()
+{
+    vespalib::LockGuard guard(_lock);
+    for (DocumentDBOwnerMap::iterator it(_documentDBOwnerMap.begin()), mt(_documentDBOwnerMap.end());
+         it != mt;
+         it++) {
+
+        if (_dbManagerMap.find(it->first) != _dbManagerMap.end()) {
+            DocumentDBConfig::SP dbConfig(_dbManagerMap[it->first]->getConfig());
+            IDocumentDBConfigOwner * owner = it->second;
+            // In case the new config does not contain this document type
+            LOG(debug, "Reconfiguring documentdb with config with generation %" PRId64, dbConfig->getGeneration());
+            owner->reconfigure(dbConfig);
+        }
+    }
+}
+
+void
+ProtonConfigurer::fetchConfigs()
+{
+    LOG(debug, "Waiting for new config generation");
+    bool configured = false;
+    while (!configured) {
+        ConfigSnapshot bootstrapSnapshot = _retriever.getBootstrapConfigs(5000);
+        if (_retriever.isClosed())
+            return;
+        LOG(debug, "Fetching snapshot");
+        if (!bootstrapSnapshot.empty()) {
+            _bootstrapConfigManager.update(bootstrapSnapshot);
+            BootstrapConfig::SP config = _bootstrapConfigManager.getConfig();
+            for (bool needsMoreConfig(true); needsMoreConfig && !_retriever.bootstrapRequired(); ) {
+                const ConfigKeySet configKeySet(pruneManagerMap(config));
+                // If key set is empty, we have no document databases to configure.
+                // This is currently not a fatal error, so it will just try to fetch
+                // the bootstrap config again.
+                if (!configKeySet.empty()) {
+                    ConfigSnapshot snapshot;
+                    do {
+                        snapshot = _retriever.getConfigs(configKeySet);
+                        if (_retriever.isClosed()) {
+                            return;
+                        }
+                    } while(snapshot.empty() && ! _retriever.bootstrapRequired());
+                    if (!snapshot.empty()) {
+                        LOG(debug, "Set is not empty, reconfiguring with generation %" PRId64, _retriever.getGeneration());
+                        // Update document dbs first, so that we are prepared for
+                        // getConfigs.
+                        needsMoreConfig = updateDocumentDBConfigs(config, snapshot);
+
+                        // Perform callbacks
+                        reconfigureBootstrap(bootstrapSnapshot);
+                        reconfigureDocumentDBs();
+                        configured = true;
+                    }
+                } else {
+                    LOG(warning, "No document databases in config, trying to re-fetch bootstrap config");
+                    break;
+                }
+            }
+        }
+    }
+}
+
+int64_t
+ProtonConfigurer::getGeneration() const
+{
+    return _retriever.getGeneration();
+}
+
+void
+ProtonConfigurer::start()
+{
+    fetchConfigs();
+    if (_threadPool.NewThread(this, NULL) == NULL) {
+        throw vespalib::IllegalStateException(
+                "Failed starting thread for proton configurer");
+    }
+}
+
+void
+ProtonConfigurer::close()
+{
+    if (!_retriever.isClosed()) {
+        _retriever.close();
+        _threadPool.Close();
+    }
+}
+
+void
+ProtonConfigurer::registerDocumentDB(const DocTypeName & docTypeName, IDocumentDBConfigOwner * owner)
+{
+    vespalib::LockGuard guard(_lock);
+    assert(_documentDBOwnerMap.find(docTypeName) == _documentDBOwnerMap.end());
+    LOG(debug, "Registering new document db with checker");
+    _documentDBOwnerMap[docTypeName] = owner;
+}
+
+void
+ProtonConfigurer::unregisterDocumentDB(const DocTypeName & docTypeName)
+{
+    vespalib::LockGuard guard(_lock);
+    LOG(debug, "Removing document db from checker");
+    assert(_documentDBOwnerMap.find(docTypeName) != _documentDBOwnerMap.end());
+    _documentDBOwnerMap.erase(docTypeName);
+}
+
+DocumentDBConfig::SP
+ProtonConfigurer::getDocumentDBConfig(const DocTypeName & docTypeName) const
+{
+    vespalib::LockGuard guard(_lock);
+    DBManagerMap::const_iterator it(_dbManagerMap.find(docTypeName));
+    if (it == _dbManagerMap.end())
+        return DocumentDBConfig::SP();
+
+    return it->second->getConfig();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/protonconfigurer.h b/searchcore/src/vespa/searchcore/proton/server/protonconfigurer.h
new file mode 100644
index 00000000000..1d08ad1c88d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/protonconfigurer.h
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fastos/thread.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/config/config.h>
+#include "bootstrapconfigmanager.h"
+#include "documentdbconfigmanager.h"
+
+namespace proton {
+
+class IBootstrapOwner
+{
+public:
+    virtual ~IBootstrapOwner() { }
+    virtual void reconfigure(const BootstrapConfig::SP & config) = 0;
+    virtual bool addExtraConfigs(DocumentDBConfigManager & dbCfgMan) = 0;
+};
+
+class IDocumentDBConfigOwner
+{
+public:
+    virtual ~IDocumentDBConfigOwner() { }
+    virtual void reconfigure(const DocumentDBConfig::SP & config) = 0;
+};
+
+/**
+ * A ProtonConfigurer monitors all config in proton and document dbs for change
+ * and starts proton reconfiguration if config has been reloaded.
+ */
+class ProtonConfigurer : public FastOS_Runnable
+{
+public:
+    ProtonConfigurer(const config::ConfigUri & configUri,
+                     IBootstrapOwner * owner,
+                     uint64_t subscribeTimeout);
+    ~ProtonConfigurer();
+    /**
+     * Register a new document db that should receive config updates.
+     */
+    void registerDocumentDB(const DocTypeName & docTypeName, IDocumentDBConfigOwner * owner);
+
+    /**
+     * Remove document db from registry, ensuring that no callbacks will come
+     * after this method has returned.
+     */
+    void unregisterDocumentDB(const DocTypeName & docTypeName);
+
+    /**
+     * Get the current config generation.
+     */
+    int64_t getGeneration() const;
+
+    /**
+     * Start configurer, callbacks may come from now on.
+     */
+    void start();
+
+    /**
+     * Shutdown configurer, ensuring that no more callbacks arrive
+     */
+    void close();
+
+    DocumentDBConfig::SP getDocumentDBConfig(const DocTypeName & docTypeName) const;
+
+    void Run(FastOS_ThreadInterface * thread, void *arg);
+
+private:
+    typedef std::map<DocTypeName, IDocumentDBConfigOwner * > DocumentDBOwnerMap;
+    typedef std::map<DocTypeName, DocumentDBConfigManager::SP> DBManagerMap;
+
+    BootstrapConfigManager _bootstrapConfigManager;
+    config::ConfigRetriever _retriever;
+    IBootstrapOwner * _bootstrapOwner;
+
+    vespalib::Lock _lock; // Protects maps
+    DBManagerMap _dbManagerMap;
+    DocumentDBOwnerMap _documentDBOwnerMap;
+
+    FastOS_ThreadPool _threadPool;
+
+    void fetchConfigs();
+    void reconfigureBootstrap(const config::ConfigSnapshot & snapshot);
+    bool updateDocumentDBConfigs(const BootstrapConfig::SP & config, const config::ConfigSnapshot & snapshot);
+    void reconfigureDocumentDBs();
+    const config::ConfigKeySet pruneManagerMap(const BootstrapConfig::SP & config);
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.cpp b/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.cpp
new file mode 100644
index 00000000000..5282d861cb6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.cpp
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "prune_session_cache_job.h"
+#include <vespa/fastos/timestamp.h>
+
+using fastos::ClockSystem;
+using fastos::TimeStamp;
+
+namespace proton {
+
+using matching::ISessionCachePruner;
+
+PruneSessionCacheJob::PruneSessionCacheJob(ISessionCachePruner &pruner,
+                                           double jobInterval)
+    : IMaintenanceJob("prune_session_cache", jobInterval, jobInterval),
+      _pruner(pruner)
+{
+}
+
+bool
+PruneSessionCacheJob::run()
+{
+    TimeStamp now(ClockSystem::now());
+    _pruner.pruneTimedOutSessions(now);
+    return true;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.h b/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.h
new file mode 100644
index 00000000000..c53f40a3e26
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_maintenance_job.h"
+#include <vespa/searchcore/proton/matching/isessioncachepruner.h>
+
+namespace proton {
+
+/**
+ * Job that regularly prunes a session cache.
+ */
+class PruneSessionCacheJob : public IMaintenanceJob
+{
+private:
+    matching::ISessionCachePruner &_pruner;
+
+public:
+    PruneSessionCacheJob(matching::ISessionCachePruner &pruner,
+                         double jobInterval);
+
+    // Implements IMaintenanceJob
+    virtual bool run();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp
new file mode 100644
index 00000000000..5510a7a67bc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.pruneremoveddocumentsjob");
+#include "pruneremoveddocumentsjob.h"
+#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h>
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+#include "ipruneremoveddocumentshandler.h"
+#include "ifrozenbuckethandler.h"
+
+using document::BucketId;
+using storage::spi::Timestamp;
+
+namespace proton
+{
+
+PruneRemovedDocumentsJob::
+PruneRemovedDocumentsJob(const Config &config,
+                         const IDocumentMetaStore &metaStore,
+                         uint32_t subDbId,
+                         const vespalib::string &docTypeName,
+                         IPruneRemovedDocumentsHandler &handler,
+                         IFrozenBucketHandler &frozenHandler)
+    : IMaintenanceJob("prune_removed_documents." + docTypeName,
+                      config.getInterval(), config.getInterval()),
+      _metaStore(metaStore),
+      _subDbId(subDbId),
+      _cfgAgeLimit(config.getAge()),
+      _docTypeName(docTypeName),
+      _handler(handler),
+      _frozenHandler(frozenHandler),
+      _pruneLids(),
+      _nextLid(1u)
+{
+}
+
+
+void
+PruneRemovedDocumentsJob::flush(DocId lowLid, DocId nextLowLid,
+                                const Timestamp ageLimit)
+{
+    if (_pruneLids.empty())
+        return;
+    DocId docIdLimit = _metaStore.getCommittedDocIdLimit();
+    PruneRemovedDocumentsOperation pruneOp(docIdLimit, _subDbId);
+    LidVectorContext::LP lvCtx(pruneOp.getLidsToRemove());
+    for (std::vector<DocId>::const_iterator it = _pruneLids.begin(),
+                                           ite = _pruneLids.end();
+         it != ite; ++it) {
+        lvCtx->addLid(*it);
+    }
+    _pruneLids.clear();
+    LOG(debug,
+        "PruneRemovedDocumentsJob::flush called,"
+        " doctype(%s)"
+        " %u lids to prune,"
+        " range [%u..%u) limit %u, timestamp %" PRIu64,
+        _docTypeName.c_str(),
+        static_cast<uint32_t>(pruneOp.getLidsToRemove()->getNumLids()),
+        lowLid, nextLowLid, docIdLimit,
+        static_cast<uint64_t>(ageLimit));
+    _handler.performPruneRemovedDocuments(pruneOp);
+}
+
+
+bool
+PruneRemovedDocumentsJob::run(void)
+{
+    uint64_t tshz = 1000000;
+    fastos::TimeStamp now = fastos::ClockSystem::now();
+    const Timestamp ageLimit(static_cast<Timestamp::Type>
+                             ((now.sec() - _cfgAgeLimit) * tshz));
+    DocId lid(_nextLid);
+    const DocId olid(lid);
+    const DocId docIdLimit(_metaStore.getCommittedDocIdLimit());
+    for (uint32_t pass = 0; pass < 10 && lid < docIdLimit; ++pass) {
+        const DocId lidLimit = std::min(lid + 10000u, docIdLimit);
+        for (; lid < lidLimit; ++lid) {
+            if (!_metaStore.validLid(lid))
+                continue;
+            const RawDocumentMetaData &metaData = _metaStore.getRawMetaData(lid);
+            if (metaData.getTimestamp() >= ageLimit)
+                continue;
+            BucketId bucket(metaData.getBucketId());
+            IFrozenBucketHandler::ExclusiveBucketGuard::UP bucketGuard = _frozenHandler.acquireExclusiveBucket(bucket);
+            if ( ! bucketGuard ) {
+                setBlocked(true);
+                _nextLid = lid;
+                flush(olid, lid, ageLimit);
+                return true;
+            }
+            _pruneLids.push_back(lid);
+        }
+        if (_pruneLids.size() >= 500)
+            break;
+    }
+    _nextLid = lid;
+    flush(olid, lid, ageLimit);
+    if (lid >= docIdLimit) {
+        _nextLid = 1u;
+        return true;
+    }
+    return false;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h
new file mode 100644
index 00000000000..1d13bb35420
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "document_db_maintenance_config.h"
+#include "i_maintenance_job.h"
+#include <persistence/spi/types.h>
+// #include <vespa/searchlib/common/idocumentmetastore.h>
+
+namespace proton
+{
+
+class IDocumentMetaStore;
+class IPruneRemovedDocumentsHandler;
+class IFrozenBucketHandler;
+
+/**
+ * Job that regularly checks whether old removed documents should be
+ * forgotten.
+ */
+class PruneRemovedDocumentsJob : public IMaintenanceJob
+{
+private:
+    const IDocumentMetaStore      &_metaStore;	// external ownership
+    uint32_t                       _subDbId;
+    double                         _cfgAgeLimit;
+    const vespalib::string        &_docTypeName;
+    IPruneRemovedDocumentsHandler &_handler;
+    IFrozenBucketHandler          &_frozenHandler;
+
+    typedef uint32_t DocId;
+    std::vector<DocId>             _pruneLids;
+    DocId                          _nextLid;
+
+    void
+    flush(DocId lowLid, DocId nextLowLid, const storage::spi::Timestamp ageLimit);
+public:
+    using Config = DocumentDBPruneRemovedDocumentsConfig;
+
+    PruneRemovedDocumentsJob(const Config &config,
+                             const IDocumentMetaStore &metaStore,
+                             uint32_t subDbId,
+                             const vespalib::string &docTypeName,
+                             IPruneRemovedDocumentsHandler &handler,
+                             IFrozenBucketHandler &frozenHandler);
+
+    // Implements IMaintenanceJob
+    virtual bool run();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp
new file mode 100644
index 00000000000..b157b652f24
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "putdonecontext.h"
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/common/docid_limit.h>
+
+namespace proton
+{
+
+
+PutDoneContext::PutDoneContext(std::unique_ptr<FeedToken> token,
+                               const FeedOperation::Type opType,
+                               PerDocTypeFeedMetrics &metrics)
+    : OperationDoneContext(std::move(token), opType, metrics),
+      _lid(0),
+      _docIdLimit(nullptr)
+{
+}
+
+
+PutDoneContext::~PutDoneContext()
+{
+    if (_docIdLimit != nullptr) {
+        _docIdLimit->bumpUpLimit(_lid + 1);
+    }
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h
new file mode 100644
index 00000000000..53ebd0dda70
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "operationdonecontext.h"
+
+namespace proton
+{
+
+
+class DocIdLimit;
+
+/**
+ * Context class for document put operations that acks operation when
+ * instance is destroyed. Typically a shared pointer to an instance is
+ * passed around to multiple worker threads that performs portions of
+ * a larger task before dropping the shared pointer, triggering the
+ * ack when all worker threads have completed.
+ */
+class PutDoneContext : public OperationDoneContext
+{
+    uint32_t _lid;
+    DocIdLimit *_docIdLimit;
+
+public:
+    PutDoneContext(std::unique_ptr<FeedToken> token,
+                   const FeedOperation::Type opType,
+                   PerDocTypeFeedMetrics &metrics);
+
+    virtual ~PutDoneContext();
+
+    void registerPutLid(uint32_t lid, DocIdLimit *docIdLimit)
+    {
+        _lid = lid;
+        _docIdLimit = docIdLimit;
+    }
+};
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
new file mode 100644
index 00000000000..88bfe2c3380
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.reconfig_params");
+#include "reconfig_params.h"
+
+namespace proton {
+
+ReconfigParams::
+ReconfigParams(const DocumentDBConfig::ComparisonResult &res)
+    : _res(res)
+{
+}
+
+bool
+ReconfigParams::shouldSchemaChange() const
+{
+    return _res._schemaChanged;
+}
+
+bool
+ReconfigParams::shouldMatchersChange() const
+{
+    return _res.rankProfilesChanged || shouldSchemaChange();
+}
+
+bool
+ReconfigParams::shouldIndexManagerChange() const
+{
+    return _res.indexschemaChanged;
+}
+
+bool
+ReconfigParams::shouldAttributeManagerChange() const
+{
+    return _res.attributesChanged;
+}
+
+bool
+ReconfigParams::shouldSummaryManagerChange() const
+{
+    return _res.summaryChanged || _res.summarymapChanged || _res.juniperrcChanged;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.h b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.h
new file mode 100644
index 00000000000..0a9f73cfd26
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.h
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentdbconfig.h"
+
+namespace proton {
+
+class ReconfigParams
+{
+private:
+    const DocumentDBConfig::ComparisonResult _res;
+
+public:
+    ReconfigParams(const DocumentDBConfig::ComparisonResult &res);
+    bool shouldSchemaChange() const;
+    bool shouldMatchersChange() const;
+    bool shouldIndexManagerChange() const;
+    bool shouldAttributeManagerChange() const;
+    bool shouldSummaryManagerChange() const;
+    bool shouldSubDbsChange() const {
+        return shouldMatchersChange()
+                || shouldAttributeManagerChange()
+                || shouldSummaryManagerChange();
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp
new file mode 100644
index 00000000000..5baa903b0d9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "removedonecontext.h"
+#include "removedonetask.h"
+#include <vespa/searchcore/proton/common/feedtoken.h>
+
+namespace proton
+{
+
+
+RemoveDoneContext::RemoveDoneContext(std::unique_ptr<FeedToken> token,
+                                     const FeedOperation::Type opType,
+                                     PerDocTypeFeedMetrics &metrics,
+                                     vespalib::Executor &executor,
+                                     IDocumentMetaStore &documentMetaStore,
+                                     uint32_t lid)
+    : OperationDoneContext(std::move(token), opType, metrics),
+      _executor(executor),
+      _task()
+{
+    if (lid != 0) {
+        _task = std::make_unique<RemoveDoneTask>(documentMetaStore, lid);
+    }
+}
+
+
+RemoveDoneContext::~RemoveDoneContext()
+{
+    ack();
+    if (_task) {
+        vespalib::Executor::Task::UP res = _executor.execute(std::move(_task));
+        assert(!res);
+    }
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h
new file mode 100644
index 00000000000..84b1f3ccc3a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "operationdonecontext.h"
+#include <vespa/vespalib/util/executor.h>
+
+namespace proton
+{
+
+class IDocumentMetaStore;
+
+
+/**
+ * Context class for document removes that acks remove andschedules a
+ * task when instance is destroyed. Typically a shared pointer to an
+ * instance is passed around to multiple worker threads that performs
+ * portions of a larger task before dropping the shared pointer,
+ * triggering the ack and task scheduling when all worker threads have
+ * completed.
+ */
+class RemoveDoneContext : public OperationDoneContext
+{
+    vespalib::Executor &_executor;
+    std::unique_ptr<vespalib::Executor::Task> _task;
+
+public:
+    RemoveDoneContext(std::unique_ptr<FeedToken> token,
+                      const FeedOperation::Type opType,
+                      PerDocTypeFeedMetrics &metrics,
+                      vespalib::Executor &executor,
+                      IDocumentMetaStore &documentMetaStore,
+                      uint32_t lid);
+
+    virtual ~RemoveDoneContext();
+};
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonetask.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonetask.cpp
new file mode 100644
index 00000000000..423af443165
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/removedonetask.cpp
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "removedonetask.h"
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+
+namespace proton
+{
+
+
+RemoveDoneTask::RemoveDoneTask(IDocumentMetaStore &documentMetaStore,
+                               uint32_t lid)
+    : vespalib::Executor::Task(),
+      _documentMetaStore(documentMetaStore),
+      _lid(lid)
+{
+}
+
+
+RemoveDoneTask::~RemoveDoneTask()
+{
+}
+
+
+void
+RemoveDoneTask::run()
+{
+    if (_lid != 0u) {
+        _documentMetaStore.removeComplete(_lid);
+    }
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonetask.h b/searchcore/src/vespa/searchcore/proton/server/removedonetask.h
new file mode 100644
index 00000000000..7590c1e5fb9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/removedonetask.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/executor.h>
+
+namespace proton
+{
+
+class IDocumentMetaStore;
+
+/**
+ * Class for task to be executed when a document remove completed and
+ * memory index and attributes have been updated.
+ *
+ * The task handles one thing:
+ *
+ * 1.  Passing on lid that can be reused do document meta store.
+ *     It have to go through a hold cycle in order for searches that
+ *     might have posting lists referencing the lid in context of
+ *     its old identity.
+ *
+ */
+class RemoveDoneTask : public vespalib::Executor::Task
+{
+    IDocumentMetaStore &_documentMetaStore;
+    // lid to reuse, can be 0 if reuse was handled by lid reuse delayer
+    uint32_t _lid;
+
+
+public:
+    RemoveDoneTask(IDocumentMetaStore &documentMetaStore,
+                   uint32_t lid);
+
+    virtual ~RemoveDoneTask();
+
+    virtual void run() override;
+};
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp
new file mode 100644
index 00000000000..394292f86e6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp
@@ -0,0 +1,122 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.replaypacketdispatcher");
+#include "replaypacketdispatcher.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using vespalib::make_string;
+using vespalib::IllegalStateException;
+
+namespace proton {
+
+template <typename OperationType>
+void
+ReplayPacketDispatcher::replay(OperationType &op, vespalib::nbostream &is, const Packet::Entry &entry)
+{
+    op.deserialize(is, _handler.getDeserializeRepo());
+    op.setSerialNum(entry.serial());
+    store(op);
+    _handler.replay(op);
+}
+
+
+ReplayPacketDispatcher::ReplayPacketDispatcher(IReplayPacketHandler &handler)
+    : _handler(handler)
+{
+}
+
+
+void
+ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry)
+{
+    vespalib::nbostream is(entry.data().c_str(),
+                           entry.data().size(),
+                           false);
+    switch (entry.type()) {
+    case FeedOperation::PUT: {
+        PutOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::REMOVE: {
+        RemoveOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::UPDATE: {
+        UpdateOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::NOOP: {
+        NoopOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::NEW_CONFIG: {
+        NewConfigOperation op(entry.serial(), _handler.getNewConfigStreamHandler());
+        op.deserialize(is, _handler.getDeserializeRepo());
+        _handler.replay(op);
+        break;
+    } case FeedOperation::WIPE_HISTORY: {
+        WipeHistoryOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::DELETE_BUCKET: {
+        DeleteBucketOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::SPLIT_BUCKET: {
+        SplitBucketOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::JOIN_BUCKETS: {
+        JoinBucketsOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::PRUNE_REMOVED_DOCUMENTS: {
+        PruneRemovedDocumentsOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::SPOOLER_REPLAY_START: {
+        SpoolerReplayStartOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::SPOOLER_REPLAY_COMPLETE: {
+        SpoolerReplayCompleteOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::MOVE: {
+        MoveOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::CREATE_BUCKET: {
+        CreateBucketOperation op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::COMPACT_LID_SPACE: {
+        CompactLidSpaceOperation op;
+        replay(op, is, entry);
+        break;
+    } default:
+        throw IllegalStateException
+            (make_string("Got packet entry with unknown type id '%u' from TLS",
+                         entry.type()));
+    }
+    if (is.size() > 0) {
+        throw document::DeserializeException
+            (make_string("Too much data in packet entry (type id '%u', %ld bytes)",
+                         entry.type(), is.size()));
+    }
+}
+
+
+ReplayPacketDispatcher::~ReplayPacketDispatcher()
+{
+}
+
+
+void
+ReplayPacketDispatcher::store(const FeedOperation &)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h
new file mode 100644
index 00000000000..b9cc9c70f7e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "ireplaypackethandler.h"
+#include <vespa/searchlib/transactionlog/common.h>
+
+namespace proton {
+
+/**
+ * Utility class that deserializes packet entries into feed operations
+ * during replay from the transaction log and dispatches the feed operations
+ * to a given handler class.
+ */
+class ReplayPacketDispatcher
+{
+private:
+    typedef search::transactionlog::Packet Packet;
+    IReplayPacketHandler &_handler;
+
+    template <typename OperationType>
+    void replay(OperationType &op, vespalib::nbostream &is, const Packet::Entry &entry);
+
+protected:
+    virtual void
+    store(const FeedOperation &op);
+
+public:
+    ReplayPacketDispatcher(IReplayPacketHandler &handler);
+
+    virtual
+    ~ReplayPacketDispatcher();
+
+    void replayEntry(const Packet::Entry &entry);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
new file mode 100644
index 00000000000..81e607a53a3
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "resource_usage_explorer.h"
+#include "disk_mem_usage_filter.h"
+#include <vespa/vespalib/data/slime/cursor.h>
+
+using namespace vespalib::slime;
+
+namespace proton {
+
+void
+convertDiskStatsToSlime(const DiskMemUsageFilter::space_info &stats, Cursor &object)
+{
+    object.setLong("capacity", stats.capacity);
+    object.setLong("free", stats.free);
+    object.setLong("available", stats.available);
+}
+
+void
+convertMemoryStatsToSlime(const vespalib::ProcessMemoryStats &stats, Cursor &object)
+{
+    object.setLong("mappedVirt", stats.getMappedVirt());
+    object.setLong("mappedRss", stats.getMappedRss());
+    object.setLong("anonymousVirt", stats.getAnonymousVirt());
+    object.setLong("anonymousRss", stats.getAnonymousRss());
+}
+
+ResourceUsageExplorer::ResourceUsageExplorer(const DiskMemUsageFilter &usageFilter)
+    : _usageFilter(usageFilter)
+{
+}
+
+void
+ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool full) const
+{
+    Cursor &object = inserter.insertObject();
+    if (full) {
+        DiskMemUsageFilter::Config config = _usageFilter.getConfig();
+        Cursor &disk = object.setObject("disk");
+        disk.setDouble("usedRatio", _usageFilter.getDiskUsedRatio());
+        disk.setDouble("usedLimit", config._diskLimit);
+        convertDiskStatsToSlime(_usageFilter.getDiskStats(), disk.setObject("stats"));
+
+        Cursor &memory = object.setObject("memory");
+        memory.setDouble("usedRatio", _usageFilter.getMemoryUsedRatio());
+        memory.setDouble("usedLimit", config._memoryLimit);
+        memory.setLong("physicalMemory", _usageFilter.getPhysicalMemory());
+        convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats"));
+    } else {
+        object.setDouble("disk", _usageFilter.getDiskUsedRatio());
+        object.setDouble("memory", _usageFilter.getMemoryUsedRatio());
+    }
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.h b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.h
new file mode 100644
index 00000000000..682ca276733
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/net/state_explorer.h>
+
+namespace proton {
+
+class DiskMemUsageFilter;
+
+/**
+ * Class used to explore the resource usage of proton.
+ */
+class ResourceUsageExplorer : public vespalib::StateExplorer
+{
+private:
+    const DiskMemUsageFilter &_usageFilter;
+
+public:
+    ResourceUsageExplorer(const DiskMemUsageFilter &usageFilter);
+
+    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp
new file mode 100644
index 00000000000..5433b4aac20
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp
@@ -0,0 +1,645 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.rtchooks");
+
+#include "rpc_hooks.h"
+#include "proton.h"
+#include <vespa/messagebus/emptyreply.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/searchcore/proton/common/statusreport.h>
+
+using namespace vespalib;
+using document::CompressionConfig;
+
+namespace {
+
+struct Pair {
+    string key;
+    string value;
+    Pair(const string &k, const string &v)
+        : key(k),
+          value(v)
+    {
+    }
+};
+
+} // namespace anon
+
+namespace proton {
+
+void
+RPCHooksBase::checkState(StateArg::UP arg)
+{
+    fastos::TimeStamp now(fastos::ClockSystem::now());
+    if (now < arg->_dueTime) {
+        MonitorGuard guard(_stateMonitor);
+        if ( guard.wait(std::min(1000L, (arg->_dueTime - now)/fastos::TimeStamp::MS)) ) {
+            LOG(debug, "state has changed");
+            reportState(*arg->_session, arg->_req);
+            arg->_req->Return();
+        } else {
+            LOG(debug, "Will reschedule");
+            FRT_RPCRequest * req = arg->_req;
+            Session & session = *arg->_session;
+            Executor::Task::UP failedTask = _executor.execute(makeTask(
+                    makeClosure(this, &RPCHooksBase::checkState, std::move(arg))));
+            if (failedTask.get() != NULL) {
+                reportState(session, req);
+                req->Return();
+            }
+        }
+    } else {
+        reportState(*arg->_session, arg->_req);
+        arg->_req->Return();
+    }
+}
+
+void
+RPCHooksBase::reportState(Session & session, FRT_RPCRequest * req)
+{
+    std::vector<Pair> res;
+    int64_t numDocs(_proton.getNumDocs());
+    std::string badConfigs = _proton.getBadConfigs();
+    bool numDocsChanged = session.getNumDocs() != numDocs;
+    bool badConfigsChanged = session.getBadConfigs() != badConfigs;
+    bool changed = numDocsChanged || badConfigsChanged;
+
+    if (_proton.getMatchEngine().isOnline()) {
+        res.push_back(Pair("online", "true"));
+        res.push_back(Pair("onlineState", "online"));
+    } else {
+        res.push_back(Pair("online", "false"));
+        res.push_back(Pair("onlineState", "onlineSoon"));
+    }
+    if (session.getGen() < 0) {
+        if (badConfigsChanged)
+            res.push_back(Pair("badConfigs", badConfigs));
+        res.push_back(Pair("onlineDocs", make_string("%lu", numDocs)));
+        session.setGen(0);
+    } else if (changed) {
+        if (badConfigsChanged)
+            res.push_back(Pair("badConfigs", badConfigs));
+        res.push_back(Pair("onlineDocs", make_string("%lu", numDocs)));
+        session.setGen(session.getGen() + 1);
+    }
+    if (numDocsChanged)
+        session.setNumDocs(numDocs);
+    if (badConfigsChanged)
+        session.setBadConfigs(badConfigs);
+
+    FRT_Values &ret = *req->GetReturn();
+    FRT_StringValue *k = ret.AddStringArray(res.size());
+    FRT_StringValue *v = ret.AddStringArray(res.size());
+    for (uint32_t i = 0; i < res.size(); ++i) {
+        ret.SetString(&k[i], res[i].key.c_str());
+        ret.SetString(&v[i], res[i].value.c_str());
+    }
+    LOG(debug, "gen=%ld", session.getGen());
+    for (uint32_t i = 0; i < res.size(); ++i) {
+        LOG(debug,
+            "key=%s, value=%s",
+            res[i].key.c_str(), res[i].value.c_str());
+    }
+    ret.AddInt32(session.getGen());
+}
+
+RPCHooksBase::Session::Session()
+    : _createTime(fastos::ClockSystem::now()),
+      _numDocs(0u),
+      _badConfigs(),
+      _gen(-1),
+      _down(false)
+{
+}
+
+void
+RPCHooksBase::initRPC()
+{
+    _orb.SetSessionInitHook(FRT_METHOD(RPCHooksBase::initSession), this);
+    _orb.SetSessionFiniHook(FRT_METHOD(RPCHooksBase::finiSession), this);
+    _orb.SetSessionDownHook(FRT_METHOD(RPCHooksBase::downSession), this);
+    _orb.SetMethodMismatchHook(FRT_METHOD(RPCHooksBase::mismatch), this);
+
+    FRT_ReflectionBuilder rb(&_orb);
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.enableSearching", "", "", true,
+                    FRT_METHOD(RPCHooksBase::rpc_enableSearching), this);
+    rb.MethodDesc("Tell the node to enable searching");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.disableSearching", "", "", true,
+                    FRT_METHOD(RPCHooksBase::rpc_disableSearching), this);
+    rb.MethodDesc("Tell the node to disable searching");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("pandora.rtc.getState", "ii", "SSi", true,
+                    FRT_METHOD(RPCHooksBase::rpc_GetState), this);
+    rb.MethodDesc("Get the current state of node");
+    rb.ParamDesc("gencnt", "old state generation held by the client");
+    rb.ParamDesc("timeout", "How many milliseconds to wait for state update");
+    rb.ReturnDesc("keys", "Array of state keys");
+    rb.ReturnDesc("values", "Array of state values");
+    rb.ReturnDesc("newgen", "New state generation count");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.getStatus", "s", "SSSS", true,
+                    FRT_METHOD(RPCHooksBase::rpc_GetProtonStatus), this);
+    rb.MethodDesc("Get the current state of proton or a proton component");
+    rb.ParamDesc("component", "Which component to check the status for");
+    rb.ReturnDesc("components", "Array of component names");
+    rb.ReturnDesc("states", "Array of states ");
+    rb.ReturnDesc("internalStates", "Array of internal states ");
+    rb.ReturnDesc("message", "Array of status messages");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("pandora.rtc.getIncrementalState", "i", "SSi", true,
+                    FRT_METHOD(RPCHooksBase::rpc_getIncrementalState), this);
+    rb.MethodDesc("Get node state changes since last invocation");
+    rb.ParamDesc("timeout", "How many milliseconds to wait for state update");
+    rb.ReturnDesc("keys", "Array of state keys");
+    rb.ReturnDesc("values", "Array of state values");
+    rb.ReturnDesc("dummy", "Dummy value to enable code reuse");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("pandora.rtc.shutdown", "", "", true,
+                    FRT_METHOD(RPCHooksBase::rpc_Shutdown), this);
+    rb.MethodDesc("Shut down the rtc application");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("pandora.rtc.die", "", "", true,
+                    FRT_METHOD(RPCHooksBase::rpc_die), this);
+    rb.MethodDesc("Exit the rtc application without cleanup");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.triggerFlush", "", "b", true,
+                    FRT_METHOD(RPCHooksBase::rpc_triggerFlush), this);
+    rb.MethodDesc("Tell the node to trigger flush ASAP");
+    rb.ReturnDesc("success", "Whether or not a flush was triggered.");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.prepareRestart", "", "b", true,
+                    FRT_METHOD(RPCHooksBase::rpc_prepareRestart), this);
+    rb.MethodDesc("Tell the node to prepare for a restart by flushing components "
+            "such that TLS replay time + time spent flushing components is as low as possible");
+    rb.ReturnDesc("success", "Whether or not prepare for restart was triggered.");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.wipeHistory", "", "", true,
+                    FRT_METHOD(RPCHooksBase::rpc_wipeHistory), this);
+    rb.MethodDesc("Tell the node to wipe history");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.listDocTypes", "", "S", true,
+                    FRT_METHOD(RPCHooksBase::rpc_listDocTypes), this);
+    rb.MethodDesc("Get the current list of document types");
+    rb.ReturnDesc("documentTypes", "Current list of document types");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.listSchema", "s", "SSSS", true,
+                    FRT_METHOD(RPCHooksBase::rpc_listSchema), this);
+    rb.MethodDesc("Get the current schema for given document type");
+    rb.ParamDesc("documentType", "Document type name");
+    rb.ReturnDesc("fieldNames", "Schema field names");
+    rb.ReturnDesc("fieldDataTypes", "Schema field data types");
+    rb.ReturnDesc("fieldCollTypes", "Schema field collection types");
+    rb.ReturnDesc("fieldLocation", "Schema field locations");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.getConfigTime", "", "l", true,
+                    FRT_METHOD(RPCHooksBase::rpc_getConfigGeneration), this);
+    rb.MethodDesc("Get the oldest active config generation");
+    rb.ReturnDesc("configTime", "Oldest active config generation");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.getConfigGeneration", "", "l", true,
+                    FRT_METHOD(RPCHooksBase::rpc_getConfigGeneration), this);
+    rb.MethodDesc("Get the oldest active config generation");
+    rb.ReturnDesc("configTime", "Oldest active config generation");
+    //-------------------------------------------------------------------------
+    rb.DefineMethod("proton.getDocsums", "bix", "bix", true, FRT_METHOD(RPCHooksBase::rpc_getDocSums), this);
+    rb.MethodDesc("Get list of document summaries");
+    rb.ParamDesc("encoding", "0=raw, 6=lz4");
+    rb.ParamDesc("uncompressedBlobSize", "Uncompressed blob size");
+    rb.ParamDesc("getDocsumX", "The request blob in slime");
+    rb.ReturnDesc("encoding",  "0=raw, 6=lz4");
+    rb.ReturnDesc("uncompressedBlobSize", "Uncompressed blob size");
+    rb.ReturnDesc("docsums", "Blob with slime encoded summaries.");
+    
+}
+
+RPCHooksBase::RPCHooksBase(Params &params)
+    : _proton(params.proton),
+      _docsumByRPC(new DocsumByRPC(_proton.getDocsumBySlime())),
+      _orb(),
+      _regAPI(_orb, params.slobrok_config),
+      _executor(48, 128 * 1024),
+      _ooscli(params, _orb)
+{
+}
+
+void
+RPCHooksBase::open(Params & params)
+{
+    initRPC();
+    _regAPI.registerName((params.identity + "/realtimecontroller").c_str());
+    _orb.Listen(params.rtcPort);
+    _orb.Start();
+    LOG(debug, "started monitoring interface");
+}
+
+RPCHooksBase::~RPCHooksBase()
+{
+}
+
+void
+RPCHooksBase::close()
+{
+    LOG(info, "shutting down monitoring interface");
+    _orb.ShutDown(true);
+    _executor.shutdown();
+    {
+        MonitorGuard guard(_stateMonitor);
+        guard.broadcast();
+    }
+    _executor.sync();
+}
+
+void
+RPCHooksBase::letProtonDo(Closure::UP closure)
+{
+    Executor::Task::UP task = makeTask(std::move(closure));
+    _proton.getExecutor().execute(std::move(task));
+}
+
+void
+RPCHooksBase::triggerFlush(FRT_RPCRequest *req)
+{
+    if (_proton.triggerFlush()) {
+        req->GetReturn()->AddInt8(1);
+        LOG(info, "RPCHooksBase::Flush finished sucessfully");
+    } else {
+        req->GetReturn()->AddInt8(0);
+        LOG(warning, "RPCHooksBase::Flush failed");
+    }
+    req->Return();
+}
+
+void
+RPCHooksBase::prepareRestart(FRT_RPCRequest *req)
+{
+    if (_proton.prepareRestart()) {
+        req->GetReturn()->AddInt8(1);
+        LOG(info, "RPCHooksBase::prepareRestart finished sucessfully");
+    } else {
+        req->GetReturn()->AddInt8(0);
+        LOG(warning, "RPCHooksBase::prepareRestart failed");
+    }
+    req->Return();
+}
+
+void
+RPCHooksBase::enableSearching(FRT_RPCRequest *req)
+{
+    _proton.getMatchEngine().setOnline();
+    LOG(info, "Searching enabled");
+    req->Return();
+}
+
+void
+RPCHooksBase::rpc_enableSearching(FRT_RPCRequest *req)
+{
+    LOG(info, "RPCHooksBase::rpc_enableSearching");
+    req->Detach();
+    letProtonDo(makeClosure(this, &RPCHooksBase::enableSearching, req));
+}
+
+void
+RPCHooksBase::disableSearching(FRT_RPCRequest *req)
+{
+    _proton.getMatchEngine().setOffline();
+    LOG(info, "Search disabled");
+    req->Return();
+}
+
+void
+RPCHooksBase::rpc_disableSearching(FRT_RPCRequest *req)
+{
+    LOG(info, "RPCHooksBase::rpc_disableSearching");
+    req->Detach();
+    letProtonDo(makeClosure(this, &RPCHooksBase::disableSearching, req));
+}
+
+void
+RPCHooksBase::rpc_GetState(FRT_RPCRequest *req)
+{
+    FRT_Values &arg = *req->GetParams();
+    uint32_t gen = arg[0]._intval32;
+    uint32_t timeoutMS = arg[1]._intval32;
+    const Session::SP & sharedSession = getSession(req);
+    LOG(debug, "RPCHooksBase::rpc_GetState(gen=%d, timeoutMS=%d) , but using gen=%ld instead", gen, timeoutMS, sharedSession->getGen());
+
+    int64_t numDocs(_proton.getNumDocs());
+    if (sharedSession->getGen() < 0 || sharedSession->getNumDocs() != numDocs) {  // NB Should use something else to define generation.
+        reportState(*sharedSession, req);
+    } else {
+        fastos::TimeStamp dueTime(fastos::ClockSystem::now() + timeoutMS * fastos::TimeStamp::MS);
+        StateArg::UP stateArg(new StateArg(sharedSession, req, dueTime));
+        if (_executor.execute(makeTask(makeClosure(this, &RPCHooksBase::checkState, std::move(stateArg)))).get() != NULL) {
+            reportState(*sharedSession, req);
+            req->Return();
+        } else {
+            req->Detach();
+        }
+    }
+}
+
+void
+RPCHooksBase::rpc_GetProtonStatus(FRT_RPCRequest *req)
+{
+    LOG(debug, "RPCHooksBase::rpc_GetProtonStatus started");
+    req->Detach();
+    _executor.execute(makeTask(makeClosure(this, &RPCHooksBase::getProtonStatus, req)));
+}
+
+void
+RPCHooksBase::getProtonStatus(FRT_RPCRequest *req)
+{
+    StatusReport::List reports(_proton.getStatusReports());
+    FRT_Values &ret = *req->GetReturn();
+    FRT_StringValue *r = ret.AddStringArray(reports.size());
+    FRT_StringValue *k = ret.AddStringArray(reports.size());
+    FRT_StringValue *internalStates = ret.AddStringArray(reports.size());
+    FRT_StringValue *v = ret.AddStringArray(reports.size());
+    for (uint32_t i = 0; i < reports.size(); ++i) {
+        ret.SetString(&r[i], reports[i]->getComponent().c_str());
+        switch (reports[i]->getState()) {
+        case StatusReport::UPOK:
+            ret.SetString(&k[i], "OK");
+            break;
+        case StatusReport::PARTIAL:
+            ret.SetString(&k[i], "WARNING");
+            break;
+        case StatusReport::DOWN:
+            ret.SetString(&k[i], "CRITICAL");
+            break;
+        default:
+            ret.SetString(&k[i], "UNKNOWN");
+            break;
+        }
+        ret.SetString(&internalStates[i],
+                      reports[i]->getInternalStatesStr().c_str());
+        ret.SetString(&v[i], reports[i]->getMessage().c_str());
+        LOG(debug,
+            "component(%s), status(%s), internalState(%s), message(%s)",
+            reports[i]->getComponent().c_str(),
+            k[i]._str,
+            internalStates[i]._str,
+            reports[i]->getMessage().c_str());
+    }
+    req->Return();
+}
+
+void
+RPCHooksBase::rpc_getIncrementalState(FRT_RPCRequest *req)
+{
+    FRT_Values &arg = *req->GetParams();
+    uint32_t timeoutMS = arg[0]._intval32;
+    const Session::SP & sharedSession = getSession(req);
+    LOG(debug, "RPCHooksBase::rpc_getIncrementalState(timeoutMS=%d)", timeoutMS);
+
+    int64_t numDocs(_proton.getNumDocs());
+    if (sharedSession->getGen() < 0 || sharedSession->getNumDocs() != numDocs) {  // NB Should use something else to define generation.
+        reportState(*sharedSession, req);
+    } else {
+        fastos::TimeStamp dueTime(fastos::ClockSystem::now() + timeoutMS * fastos::TimeStamp::MS);
+        StateArg::UP stateArg(new StateArg(sharedSession, req, dueTime));
+        if (_executor.execute(makeTask(makeClosure(this, &RPCHooksBase::checkState, std::move(stateArg)))).get() != NULL) {
+            reportState(*sharedSession, req);
+            req->Return();
+        } else {
+            req->Detach();
+        }
+    }
+}
+
+void
+RPCHooksBase::rpc_Shutdown(FRT_RPCRequest *req)
+{
+    (void) req;
+    LOG(debug, "RPCHooksBase::rpc_Shutdown");
+}
+
+void
+RPCHooksBase::rpc_die(FRT_RPCRequest *req)
+{
+    (void) req;
+    LOG(debug, "RPCHooksBase::rpc_die");
+    _exit(0);
+}
+
+void
+RPCHooksBase::rpc_triggerFlush(FRT_RPCRequest *req)
+{
+    LOG(info, "RPCHooksBase::rpc_triggerFlush started");
+    req->Detach();
+    letProtonDo(makeClosure(this, &RPCHooksBase::triggerFlush, req));
+}
+
+void
+RPCHooksBase::rpc_prepareRestart(FRT_RPCRequest *req)
+{
+    LOG(info, "RPCHooksBase::rpc_prepareRestart started");
+    req->Detach();
+    letProtonDo(makeClosure(this, &RPCHooksBase::prepareRestart, req));
+}
+
+void
+RPCHooksBase::wipeHistory(FRT_RPCRequest *req)
+{
+    _proton.wipeHistory();
+    LOG(info, "RPCHooksBase::wipeHistory finished sucessfully");
+    req->Return();
+}
+
+void
+RPCHooksBase::rpc_wipeHistory(FRT_RPCRequest *req)
+{
+    LOG(info, "RPCHooksBase::rpc_wipeHistory started");
+    req->Detach();
+    letProtonDo(makeClosure(this, &RPCHooksBase::wipeHistory, req));
+}
+
+void
+RPCHooksBase::listDocTypes(FRT_RPCRequest *req)
+{
+    std::vector<string> documentTypes;
+
+    _proton.listDocTypes(documentTypes);
+
+    FRT_Values &ret = *req->GetReturn();
+    FRT_StringValue *dt = ret.AddStringArray(documentTypes.size());
+    for (uint32_t i = 0; i < documentTypes.size(); ++i)
+        ret.SetString(&dt[i], documentTypes[i].c_str());
+
+    LOG(info,
+        "RPCHooksBase::listDocTypes finished successfully");
+    req->Return();
+}
+
+
+void
+RPCHooksBase::rpc_listDocTypes(FRT_RPCRequest *req)
+{
+    LOG(info,
+        "RPCHooksBase::rpc_listDocTypes started");
+    req->Detach();
+    letProtonDo(makeClosure(this, &RPCHooksBase::listDocTypes, req));
+}
+
+
+void
+RPCHooksBase::listSchema(FRT_RPCRequest *req)
+{
+    std::vector<string> fieldNames;
+    std::vector<string> fieldDataTypes;
+    std::vector<string> fieldCollectionTypes;
+    std::vector<string> fieldLocations;
+
+    FRT_Values &arg = *req->GetParams();
+    string documentType(arg[0]._string._str, arg[0]._string._len);
+
+    _proton.listSchema(documentType,
+                       fieldNames, fieldDataTypes, fieldCollectionTypes,
+                       fieldLocations);
+
+    FRT_Values &ret = *req->GetReturn();
+    FRT_StringValue *fn = ret.AddStringArray(fieldNames.size());
+    for (uint32_t i = 0; i < fieldNames.size(); ++i)
+        ret.SetString(&fn[i], fieldNames[i].c_str());
+
+    FRT_StringValue *fdt = ret.AddStringArray(fieldDataTypes.size());
+    for (uint32_t i = 0; i < fieldDataTypes.size(); ++i)
+        ret.SetString(&fdt[i], fieldDataTypes[i].c_str());
+
+    FRT_StringValue *fct = ret.AddStringArray(fieldCollectionTypes.size());
+    for (uint32_t i = 0; i < fieldCollectionTypes.size(); ++i)
+        ret.SetString(&fct[i], fieldCollectionTypes[i].c_str());
+
+    FRT_StringValue *fl = ret.AddStringArray(fieldLocations.size());
+    for (uint32_t i = 0; i < fieldLocations.size(); ++i)
+        ret.SetString(&fl[i], fieldLocations[i].c_str());
+
+    LOG(info,
+        "RPCHooksBase::listSchema finished successfully");
+    req->Return();
+}
+
+
+void
+RPCHooksBase::rpc_listSchema(FRT_RPCRequest *req)
+{
+    LOG(info,
+        "RPCHooksBase::rpc_listSchema started");
+    req->Detach();
+    letProtonDo(makeClosure(this, &RPCHooksBase::listSchema, req));
+}
+
+
+void
+RPCHooksBase::getConfigGeneration(FRT_RPCRequest *req)
+{
+    int64_t configGeneration = _proton.getConfigGeneration();
+    FRT_Values &ret = *req->GetReturn();
+    ret.AddInt64(configGeneration);
+
+    LOG(info,
+        "RPCHooksBase::getConfigGeneration finished successfully, "
+        "configGeneration=%" PRId64,
+        configGeneration);
+    req->Return();
+}
+
+void
+RPCHooksBase::rpc_getDocSums(FRT_RPCRequest *req)
+{
+    LOG(debug, "proton.getDocsums()");
+    req->Detach();
+    _executor.execute(makeTask(makeClosure(this, &RPCHooksBase::getDocsums, req)));
+}
+
+void
+RPCHooksBase::getDocsums(FRT_RPCRequest *req)
+{
+    _docsumByRPC->getDocsums(*req);
+    req->Return();
+}
+
+void
+RPCHooksBase::rpc_getConfigGeneration(FRT_RPCRequest *req)
+{
+    LOG(info,
+        "RPCHooksBase::rpc_getConfigGeneration started");
+    req->Detach();
+    letProtonDo(makeClosure(this, &RPCHooksBase::getConfigGeneration, req));
+}
+
+
+const RPCHooksBase::Session::SP &
+RPCHooksBase::getSession(FRT_RPCRequest *req)
+{
+    FNET_Connection *conn = req->GetConnection();
+    void *vctx = conn->GetContext()._value.VOIDP;
+    Session::SP *sessionspp = static_cast<Session::SP *>(vctx);
+    return *sessionspp;
+}
+
+
+void
+RPCHooksBase::initSession(FRT_RPCRequest *req)
+{
+    FNET_Connection *conn = req->GetConnection();
+    const void * voidp(conn->GetContext()._value.VOIDP);
+    req->GetConnection()->SetContext(new Session::SP(new Session()));
+    void *vctx = conn->GetContext()._value.VOIDP;
+    LOG(debug,
+        "RPCHooksBase::iniSession req=%p connection=%p session=%p oldSession=%p",
+        req,
+        conn,
+        vctx,
+        voidp);
+}
+
+void
+RPCHooksBase::finiSession(FRT_RPCRequest *req)
+{
+    FNET_Connection *conn = req->GetConnection();
+    void *vctx = conn->GetContext()._value.VOIDP;
+    conn->GetContextPT()->_value.VOIDP = NULL;
+    LOG(debug,
+        "RPCHooksBase::finiSession req=%p connection=%p session=%p",
+        req,
+        conn,
+        vctx);
+
+    Session::SP *sessionspp = static_cast<Session::SP *>(vctx);
+    delete sessionspp;
+}
+
+
+void
+RPCHooksBase::downSession(FRT_RPCRequest *req)
+{
+    LOG(debug,
+        "RPCHooksBase::downSession req=%p connection=%p session=%p",
+        req,
+        req->GetConnection(),
+        req->GetConnection()->GetContext()._value.VOIDP);
+    getSession(req)->setDown();
+}
+
+
+void
+RPCHooksBase::mismatch(FRT_RPCRequest *req)
+{
+    LOG(warning, "RPC Mismatch: %s, (%d): %s",
+        req->GetMethodName(),
+        req->GetErrorCode(), req->GetErrorMessage());
+}
+
+RPCHooks::RPCHooks(Params &params) :
+    RPCHooksBase(params)
+{
+    open(params);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h
new file mode 100644
index 00000000000..d0332a2b391
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h
@@ -0,0 +1,139 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fnet/frt/frt.h>
+#include <vespa/slobrok/sbregister.h>
+#include <vespa/vespalib/util/atomic.h>
+#include <vespa/vespalib/util/executor.h>
+#include <vespa/vespalib/util/closure.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/searchlib/common/packets.h>
+
+#include "ooscli.h"
+
+namespace proton {
+
+class Proton;
+class DocsumByRPC;
+
+class RPCHooksBase : public FRT_Invokable, public vespalib::noncopyable
+{
+private:
+    class Session {
+    private:
+        fastos::TimeStamp _createTime;
+        int64_t		  _numDocs;
+        vespalib::string  _badConfigs;	
+        int64_t           _gen;
+        bool		  _down;
+    public:
+        typedef std::shared_ptr<Session> SP;
+        Session();
+        int64_t                  getGen() const { return _gen; }
+        fastos::TimeStamp getCreateTime() const { return _createTime; }
+        Session & setGen(int64_t gen) { _gen = gen; return *this; }
+
+        int64_t getNumDocs(void) const { return _numDocs; }
+        void setNumDocs(int64_t numDocs) { _numDocs = numDocs; }
+        bool getDown(void) const { return _down; }
+        void setDown(void) { _down = true; }
+
+        const vespalib::string & getBadConfigs(void) const {
+            return _badConfigs;
+        }
+
+        void setBadConfigs(const vespalib::string &badConfigs) {
+            _badConfigs = badConfigs;
+        }
+
+    };
+    struct StateArg {
+        typedef std::unique_ptr<StateArg> UP;
+        StateArg(Session::SP session, FRT_RPCRequest * req, fastos::TimeStamp dueTime) :
+            _session(session),
+            _req(req),
+            _dueTime(dueTime)
+        { }
+        Session::SP _session;
+        FRT_RPCRequest * _req;
+        fastos::TimeStamp _dueTime;
+    };
+
+    Proton                      & _proton;
+    std::unique_ptr<DocsumByRPC>  _docsumByRPC;
+    FRT_Supervisor                _orb;
+    slobrok::api::RegisterAPI     _regAPI;
+    vespalib::Monitor             _stateMonitor;
+    vespalib::ThreadStackExecutor _executor;
+    OosCli                        _ooscli;
+
+    void initRPC();
+    void letProtonDo(vespalib::Closure::UP closure);
+
+    void triggerFlush(FRT_RPCRequest *req);
+    void prepareRestart(FRT_RPCRequest *req);
+    void wipeHistory(FRT_RPCRequest * req);
+    void enableSearching(FRT_RPCRequest * req);
+    void disableSearching(FRT_RPCRequest * req);
+    void checkState(StateArg::UP arg);
+    void reportState(Session & session, FRT_RPCRequest * req) __attribute__((noinline));
+    void getProtonStatus(FRT_RPCRequest * req);
+    void listDocTypes(FRT_RPCRequest *req);
+    void listSchema(FRT_RPCRequest *req);
+    void getConfigGeneration(FRT_RPCRequest *req);
+    void getDocsums(FRT_RPCRequest *req);
+
+    static const Session::SP & getSession(FRT_RPCRequest *req);
+public:
+    typedef std::unique_ptr<RPCHooksBase> UP;
+    struct Params : public OosCli::OosParams {
+        vespalib::string identity;
+        uint32_t rtcPort;
+
+        Params(Proton &parent, uint32_t port, const vespalib::string &ident)
+            : OosParams(parent),
+              identity(ident),
+              rtcPort(port)
+        {
+            my_oos_name = identity;
+            my_oos_name.append("/feed-destination");
+        }
+    };
+    RPCHooksBase(Params &params);
+    virtual ~RPCHooksBase();
+    void close();
+
+    void rpc_enableSearching(FRT_RPCRequest *req);
+    void rpc_disableSearching(FRT_RPCRequest *req);
+    void rpc_GetState(FRT_RPCRequest *req);
+    void rpc_GetProtonStatus(FRT_RPCRequest *req);
+    void rpc_getIncrementalState(FRT_RPCRequest *req);
+    void rpc_Shutdown(FRT_RPCRequest *req);
+    void rpc_die(FRT_RPCRequest *req);
+    void rpc_triggerFlush(FRT_RPCRequest *req);
+    void rpc_prepareRestart(FRT_RPCRequest *req);
+    void rpc_wipeHistory(FRT_RPCRequest *req);
+    void rpc_listDocTypes(FRT_RPCRequest *req);
+    void rpc_listSchema(FRT_RPCRequest *req);
+    void rpc_getConfigGeneration(FRT_RPCRequest *req);
+    void rpc_getDocSums(FRT_RPCRequest *req);
+
+    void initSession(FRT_RPCRequest *req);
+    void finiSession(FRT_RPCRequest *req);
+    void downSession(FRT_RPCRequest *req);
+    void mismatch(FRT_RPCRequest *req);
+protected:
+    void open(Params & params);
+};
+
+class RPCHooks : public RPCHooksBase
+{
+public:
+    RPCHooks(Params &params);
+};
+
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp
new file mode 100644
index 00000000000..5a485a3744d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "sample_attribute_usage_job.h"
+#include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
+#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h>
+#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h>
+#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h>
+
+namespace proton {
+
+SampleAttributeUsageJob::
+SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager,
+                        IAttributeManagerSP notReadyAttributeManager,
+                        AttributeUsageFilter &attributeUsageFilter,
+                        const vespalib::string &docTypeName,
+                        double interval)
+    : IMaintenanceJob("sample_attribute_usage." + docTypeName,
+                      0.0, interval),
+      _readyAttributeManager(readyAttributeManager),
+      _notReadyAttributeManager(notReadyAttributeManager),
+      _attributeUsageFilter(attributeUsageFilter)
+{
+}
+
+SampleAttributeUsageJob::~SampleAttributeUsageJob()
+{
+}
+
+bool
+SampleAttributeUsageJob::run()
+{
+    auto context = std::make_shared<AttributeUsageSamplerContext>
+                   (_attributeUsageFilter);
+    _readyAttributeManager->asyncForEachAttribute(
+            std::make_shared<AttributeUsageSamplerFunctor>(context, "ready"));
+    _notReadyAttributeManager->asyncForEachAttribute(
+            std::make_shared<AttributeUsageSamplerFunctor>(context,
+                                                           "notready"));
+    return true;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h
new file mode 100644
index 00000000000..838f002e9c5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_maintenance_job.h"
+
+namespace proton
+{
+
+class IAttributeManager;
+class AttributeUsageFilter;
+
+/**
+ * Class used to sample attribute resource usage and pass aggregated
+ * information to attribute usage filter to block feeding before
+ * proton crashes due to attribute structure size limitations.
+ */
+class SampleAttributeUsageJob : public IMaintenanceJob
+{
+    using IAttributeManagerSP = std::shared_ptr<IAttributeManager>;
+
+    IAttributeManagerSP   _readyAttributeManager;
+    IAttributeManagerSP   _notReadyAttributeManager;
+    AttributeUsageFilter &_attributeUsageFilter;
+public:
+    SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager,
+                            IAttributeManagerSP notReadyAttributeManager,
+                            AttributeUsageFilter &attributeUsageFilter,
+                            const vespalib::string &docTypeName,
+                            double interval);
+    ~SampleAttributeUsageJob();
+
+    virtual bool run() override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/schema_config_validator.cpp b/searchcore/src/vespa/searchcore/proton/server/schema_config_validator.cpp
new file mode 100644
index 00000000000..e0bb45e63c4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/schema_config_validator.cpp
@@ -0,0 +1,343 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.schema_config_validator");
+#include "schema_config_validator.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+using search::index::Schema;
+using vespalib::make_string;
+
+namespace proton {
+
+namespace {
+
+const vespalib::string INDEX_TYPE_NAME = "index";
+const vespalib::string ATTRIBUTE_TYPE_NAME = "attribute";
+const vespalib::string SUMMARY_TYPE_NAME = "summary";
+
+typedef ConfigValidator CV;
+
+struct SchemaSpec
+{
+    const Schema &_newSchema;
+    const Schema &_oldSchema;
+    const Schema &_oldHistory;
+    SchemaSpec(const Schema &newSchema,
+               const Schema &oldSchema,
+               const Schema &oldHistory)
+        : _newSchema(newSchema),
+          _oldSchema(oldSchema),
+          _oldHistory(oldHistory)
+    {
+    }
+};
+
+struct IndexChecker {
+    static vespalib::string TypeName;
+    static CV::ResultType AspectAdded;
+    static CV::ResultType AspectRemoved;
+
+    static bool
+    inSchema(const vespalib::string &name,
+             const Schema &schema,
+             const Schema &hSchema)
+    {
+        return (schema.isIndexField(name) || hSchema.isIndexField(name)) &&
+            (schema.isAttributeField(name) || schema.isSummaryField(name) ||
+             hSchema.isAttributeField(name) || hSchema.isSummaryField(name));
+    }
+    static bool
+    notInSchema(const vespalib::string &name,
+                const Schema &schema,
+                const Schema &hSchema)
+    {
+        return !schema.isIndexField(name) &&
+            (schema.isAttributeField(name) || schema.isSummaryField(name) ||
+             hSchema.isAttributeField(name) || hSchema.isSummaryField(name));
+    }
+};
+vespalib::string IndexChecker::TypeName = INDEX_TYPE_NAME;
+CV::ResultType IndexChecker::AspectAdded = CV::INDEX_ASPECT_ADDED;
+CV::ResultType IndexChecker::AspectRemoved = CV::INDEX_ASPECT_REMOVED;
+
+struct AttributeChecker {
+    static vespalib::string TypeName;
+    static CV::ResultType AspectAdded;
+    static CV::ResultType AspectRemoved;
+
+    static bool
+    inSchema(const vespalib::string &name,
+             const Schema &schema,
+             const Schema &hSchema)
+    {
+        return (schema.isAttributeField(name) ||
+                hSchema.isAttributeField(name)) &&
+            (schema.isSummaryField(name) ||
+             hSchema.isSummaryField(name));
+    }
+    static bool
+    notInSchema(const vespalib::string &name,
+                const Schema &schema,
+                const Schema &hSchema)
+    {
+        return !schema.isAttributeField(name) &&
+            (schema.isIndexField(name) || schema.isSummaryField(name) ||
+             hSchema.isIndexField(name) || hSchema.isSummaryField(name));
+    }
+};
+vespalib::string AttributeChecker::TypeName = ATTRIBUTE_TYPE_NAME;
+CV::ResultType AttributeChecker::AspectAdded = CV::ATTRIBUTE_ASPECT_ADDED;
+CV::ResultType AttributeChecker::AspectRemoved = CV::ATTRIBUTE_ASPECT_REMOVED;
+
+bool
+unchangedAspects(const vespalib::string &fieldName,
+                 const Schema &newSchema,
+                 const Schema &oldSchema,
+                 const Schema &oldHistory)
+{
+    if (oldSchema.isIndexField(fieldName) ||
+        oldSchema.isAttributeField(fieldName) ||
+        oldSchema.isSummaryField(fieldName))
+        return false; // field not removed
+    return (newSchema.isIndexField(fieldName) ==
+            oldHistory.isIndexField(fieldName) &&
+            newSchema.isAttributeField(fieldName) ==
+            oldHistory.isAttributeField(fieldName) &&
+            newSchema.isSummaryField(fieldName) ==
+            oldHistory.isSummaryField(fieldName));
+}
+
+template <typename Checker>
+CV::Result
+checkAspectAdded(const Schema::Field &field,
+                 const SchemaSpec &spec)
+{
+    // Special check for undo scenarios.
+    if (unchangedAspects(field.getName(), spec._newSchema, spec._oldSchema, spec._oldHistory)) {
+        return CV::Result();
+    }
+    if (Checker::notInSchema(field.getName(), spec._oldSchema, spec._oldHistory)) {
+        return CV::Result(Checker::AspectAdded,
+                make_string("Trying to add %s field `%s', but it has existed as a field before",
+                        Checker::TypeName.c_str(), field.getName().c_str()));
+    }
+    return CV::Result();
+}
+
+template <typename Checker>
+CV::Result
+checkAspectRemoved(const Schema::Field &field,
+                   const SchemaSpec &spec)
+{
+    // Special check for undo scenarios.
+    if (unchangedAspects(field.getName(), spec._newSchema, spec._oldSchema, spec._oldHistory)) {
+        return CV::Result();
+    }
+    if (Checker::inSchema(field.getName(), spec._oldSchema, spec._oldHistory)) {
+        return CV::Result(Checker::AspectRemoved,
+                make_string("Trying to remove %s field `%s', but it still exists as a field",
+                        Checker::TypeName.c_str(), field.getName().c_str()));
+    }
+    return CV::Result();
+}
+
+struct IndexTraits
+{
+    static vespalib::string TypeName;
+    static uint32_t getFieldId(const vespalib::string &name, const Schema &schema) {
+        return schema.getIndexFieldId(name);
+    }
+    static const Schema::Field &getField(uint32_t fieldId, const Schema &schema) {
+        return schema.getIndexField(fieldId);
+    }
+};
+vespalib::string IndexTraits::TypeName = INDEX_TYPE_NAME;
+
+struct AttributeTraits
+{
+    static vespalib::string TypeName;
+    static uint32_t getFieldId(const vespalib::string &name, const Schema &schema) {
+        return schema.getAttributeFieldId(name);
+    }
+    static const Schema::Field &getField(uint32_t fieldId, const Schema &schema) {
+        return schema.getAttributeField(fieldId);
+    }
+};
+vespalib::string AttributeTraits::TypeName = ATTRIBUTE_TYPE_NAME;
+
+struct SummaryTraits
+{
+    static vespalib::string TypeName;
+    static uint32_t getFieldId(const vespalib::string &name, const Schema &schema) {
+        return schema.getSummaryFieldId(name);
+    }
+    static const Schema::Field &getField(uint32_t fieldId, const Schema &schema) {
+        return schema.getSummaryField(fieldId);
+    }
+};
+vespalib::string SummaryTraits::TypeName = SUMMARY_TYPE_NAME;
+
+CV::Result
+checkDataTypeFunc(const Schema::Field &oldField,
+                  const Schema::Field &newField,
+                  const vespalib::string &fieldClass)
+{
+    if (oldField.getDataType() != newField.getDataType()) {
+        return CV::Result(CV::DATA_TYPE_CHANGED,
+                make_string("Trying to add %s field `%s' of data type %s, "
+                        "but it has been of of data type %s earlier",
+                        fieldClass.c_str(), newField.getName().c_str(),
+                        Schema::getTypeName(newField.getDataType()).c_str(),
+                        Schema::getTypeName(oldField.getDataType()).c_str()));
+    }
+    return CV::Result();
+}
+
+CV::Result
+checkCollectionTypeFunc(const Schema::Field &oldField,
+                        const Schema::Field &newField,
+                        const vespalib::string &fieldClass)
+{
+    if (oldField.getCollectionType() != newField.getCollectionType()) {
+        return CV::Result(CV::COLLECTION_TYPE_CHANGED,
+                make_string("Trying to add %s field `%s' of collection type %s, "
+                        "but it has been of of collection type %s earlier",
+                        fieldClass.c_str(), newField.getName().c_str(),
+                        Schema::getTypeName(newField.getCollectionType()).c_str(),
+                        Schema::getTypeName(oldField.getCollectionType()).c_str()));
+    }
+    return CV::Result();
+}
+
+template <typename T, typename CheckFunc>
+CV::Result
+checkType(const Schema::Field &field, const Schema &oldSchema, CheckFunc func)
+{
+    uint32_t oFieldId = T::getFieldId(field.getName(), oldSchema);
+    if (oFieldId != Schema::UNKNOWN_FIELD_ID) {
+        const Schema::Field &oField = T::getField(oFieldId, oldSchema);
+        CV::Result res = func(oField, field, T::TypeName);
+        if (!res.ok()) {
+            return res;
+        }
+    }
+    return CV::Result();
+}
+
+template <typename T, typename CheckFunc>
+CV::Result
+checkType(const Schema::Field &field, const SchemaSpec &spec, CheckFunc func)
+{
+    CV::Result res;
+    if (!(res = checkType<T>(field, spec._oldSchema, func)).ok()) return res;
+    if (!(res = checkType<T>(field, spec._oldHistory, func)).ok()) return res;
+    return CV::Result();
+}
+
+template <typename CheckFunc>
+CV::Result
+checkType(const SchemaSpec &spec, CheckFunc func)
+{
+    CV::Result res;
+    for (const auto &f : spec._newSchema.getIndexFields()) {
+        if (!(res = checkType<IndexTraits>(f, spec, func)).ok()) return res;
+    }
+    for (const auto &f : spec._newSchema.getAttributeFields()) {
+        if (!(res = checkType<AttributeTraits>(f, spec, func)).ok()) return res;
+    }
+    for (const auto &f : spec._newSchema.getSummaryFields()) {
+        if (!(res = checkType<SummaryTraits>(f, spec, func)).ok()) return res;
+    }
+    return CV::Result();
+}
+
+CV::Result
+checkDataType(const SchemaSpec &spec)
+{
+    return checkType(spec, checkDataTypeFunc);
+}
+
+CV::Result
+checkCollectionType(const SchemaSpec &spec)
+{
+    return checkType(spec, checkCollectionTypeFunc);
+}
+
+CV::Result
+checkIndexAspectAdded(const SchemaSpec &spec)
+{
+    CV::Result res;
+    for (const auto &f : spec._newSchema.getIndexFields()) {
+        if (!(res = checkAspectAdded<IndexChecker>(f, spec)).ok()) return res;
+    }
+    return CV::Result();
+}
+
+CV::Result
+checkIndexAspectRemoved(const SchemaSpec &spec)
+{
+    CV::Result res;
+    for (const auto &f : spec._newSchema.getAttributeFields()) {
+        if (!spec._newSchema.isIndexField(f.getName())) {
+            if (!(res = checkAspectRemoved<IndexChecker>(f, spec)).ok()) return res;
+        }
+    }
+    for (const auto &f : spec._newSchema.getSummaryFields()) {
+        if (!spec._newSchema.isIndexField(f.getName())) {
+            if (!(res = checkAspectRemoved<IndexChecker>(f, spec)).ok()) return res;
+        }
+    }
+    return CV::Result();
+}
+
+CV::Result
+checkAttributeAspectAdded(const SchemaSpec &spec)
+{
+    CV::Result res;
+    for (const auto &f : spec._newSchema.getAttributeFields()) {
+        if (!(res = checkAspectAdded<AttributeChecker>(f, spec)).ok()) return res;
+    }
+    return CV::Result();
+}
+
+CV::Result
+checkAttributeAspectRemoved(const SchemaSpec &spec)
+{
+    CV::Result res;
+    // Note: remove as attribute is allowed when still existing as index
+    // so no need to iterator all index fields.
+
+    for (const auto &f : spec._newSchema.getSummaryFields()) {
+        if (!spec._newSchema.isAttributeField(f.getName()) &&
+                !spec._newSchema.isIndexField(f.getName()) &&
+                !spec._oldSchema.isIndexField(f.getName()))
+        {
+            if (!(res = checkAspectRemoved<AttributeChecker>(f, spec)).ok()) return res;
+        }
+    }
+    return CV::Result();
+}
+
+}
+
+CV::Result
+SchemaConfigValidator::validate(const Schema &newSchema,
+                                const Schema &oldSchema,
+                                const Schema &oldHistory)
+{
+    LOG(debug, "validate(): newSchema='%s', oldSchema='%s', oldHistory='%s'",
+        newSchema.toString().c_str(), oldSchema.toString().c_str(), oldHistory.toString().c_str());
+    CV::Result res;
+    SchemaSpec spec(newSchema, oldSchema, oldHistory);
+    if (!(res = checkDataType(spec)).ok()) return res;
+    if (!(res = checkCollectionType(spec)).ok()) return res;
+    if (!(res = checkIndexAspectAdded(spec)).ok()) return res;
+    if (!(res = checkIndexAspectRemoved(spec)).ok()) return res;
+    if (!(res = checkAttributeAspectRemoved(spec)).ok()) return res;
+    if (!(res = checkAttributeAspectAdded(spec)).ok()) return res;
+    return CV::Result();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/schema_config_validator.h b/searchcore/src/vespa/searchcore/proton/server/schema_config_validator.h
new file mode 100644
index 00000000000..5b301fd93ba
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/schema_config_validator.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "configvalidator.h"
+
+namespace proton {
+
+/**
+ * Class used to validate new schema before starting using it.
+ **/
+struct SchemaConfigValidator
+{
+    static ConfigValidator::Result
+    validate(const search::index::Schema &newSchema,
+             const search::index::Schema &oldSchema,
+             const search::index::Schema &oldHistory);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
new file mode 100644
index 00000000000..fe272cffe44
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
@@ -0,0 +1,258 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.searchable_doc_subdb_configurer");
+#include "searchable_doc_subdb_configurer.h"
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/common/document_type_inspector.h>
+#include <vespa/searchcore/proton/matching/matcher.h>
+#include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h>
+
+using namespace vespa::config::search;
+using namespace config;
+using search::index::Schema;
+using searchcorespi::IndexSearchable;
+using document::DocumentTypeRepo;
+using vespa::config::search::RankProfilesConfig;
+
+namespace proton {
+
+using matching::Matcher;
+
+typedef AttributeReprocessingInitializer::Config ARIConfig;
+
+void
+SearchableDocSubDBConfigurer::reconfigureFeedView(const SearchView::SP &searchView)
+{
+    SearchableFeedView::SP curr = _feedView.get();
+    reconfigureFeedView(curr->getIndexWriter(),
+                        curr->getSummaryAdapter(),
+                        curr->getAttributeWriter(),
+                        curr->getSchema(),
+                        curr->getDocumentTypeRepo(),
+                        searchView);
+}
+
+void
+SearchableDocSubDBConfigurer::reconfigureFeedView(const IIndexWriter::SP &indexWriter,
+                                                  const ISummaryAdapter::SP &summaryAdapter,
+                                                  const IAttributeWriter::SP &attrWriter,
+                                                  const Schema::SP &schema,
+                                                  const DocumentTypeRepo::SP &repo,
+                                                  const SearchView::SP &searchView)
+{
+    SearchableFeedView::SP curr = _feedView.get();
+    _feedView.set(SearchableFeedView::SP(new SearchableFeedView(
+            StoreOnlyFeedView::Context(summaryAdapter,
+                    schema,
+                    searchView->getDocumentMetaStore(),
+                    repo,
+                    curr->getWriteService(),
+                    curr->getLidReuseDelayer(), curr->getCommitTimeTracker()),
+            curr->getPersistentParams(),
+            FastAccessFeedView::Context(attrWriter, curr->getDocIdLimit()),
+            SearchableFeedView::Context(indexWriter))));
+}
+
+void
+SearchableDocSubDBConfigurer::
+reconfigureMatchView(const IndexSearchable::SP &indexSearchable)
+{
+    SearchView::SP curr = _searchView.get();
+    reconfigureMatchView(curr->getMatchers(),
+                         indexSearchable,
+                         curr->getAttributeManager());
+}
+
+void
+SearchableDocSubDBConfigurer::
+reconfigureMatchView(const Matchers::SP &matchers,
+                     const IndexSearchable::SP &indexSearchable,
+                     const IAttributeManager::SP &attrMgr)
+{
+    SearchView::SP curr = _searchView.get();
+    MatchView::SP matchView(new MatchView(matchers,
+                                          indexSearchable,
+                                          attrMgr,
+                                          curr->getSessionManager(),
+                                          curr->getDocumentMetaStore(),
+                                          curr->getDocIdLimit()));
+    reconfigureSearchView(matchView);
+}
+
+void
+SearchableDocSubDBConfigurer::reconfigureSearchView(const MatchView::SP &matchView)
+{
+    SearchView::SP curr = _searchView.get();
+    _searchView.set(SearchView::SP(new SearchView(curr->getSummarySetup(), matchView)));
+}
+
+void
+SearchableDocSubDBConfigurer::reconfigureSearchView(const ISummaryManager::ISummarySetup::SP &summarySetup,
+                                                    const MatchView::SP &matchView)
+{
+    _searchView.set(SearchView::SP(new SearchView(summarySetup, matchView)));
+}
+
+SearchableDocSubDBConfigurer::
+SearchableDocSubDBConfigurer(const ISummaryManager::SP & summaryMgr,
+                             SearchViewHolder & searchView,
+                             FeedViewHolder & feedView,
+                             matching::QueryLimiter & queryLimiter,
+                             const vespalib::Clock & clock,
+                             const vespalib::string & subDbName,
+                             uint32_t distributionKey) :
+    _summaryMgr(summaryMgr),
+    _searchView(searchView),
+    _feedView(feedView),
+    _queryLimiter(queryLimiter),
+    _clock(clock),
+    _subDbName(subDbName),
+    _distributionKey(distributionKey)
+{
+}
+
+Matchers::UP
+SearchableDocSubDBConfigurer::createMatchers(const Schema::SP &schema,
+                                             const RankProfilesConfig &cfg)
+{
+    Matchers::UP newMatchers(new Matchers(_clock, _queryLimiter));
+    for (const auto &profile : cfg.rankprofile) {
+        vespalib::string name = profile.name;
+        search::fef::Properties properties;
+        for (const auto &property : profile.fef.property) {
+            properties.add(property.name,
+                           property.value);
+        }
+        LOG(debug, "Adding matcher for rankprofile '%s'", name.c_str());
+        // schema instance only used during call.
+        Matcher::SP profptr(new Matcher(*schema, properties, _clock, _queryLimiter, _distributionKey));
+        newMatchers->add(name, profptr);
+    }
+    return newMatchers;
+}
+
+void
+SearchableDocSubDBConfigurer::reconfigureIndexSearchable()
+{
+    SearchableFeedView::SP feedView(_feedView.get());
+    const IIndexWriter::SP &indexWriter = feedView->getIndexWriter();
+    const IIndexManager::SP &indexManager = indexWriter->getIndexManager();
+    reconfigureMatchView(indexManager->getSearchable());
+    const SearchView::SP searchView(_searchView.get());
+    reconfigureFeedView(searchView);
+}
+
+void
+SearchableDocSubDBConfigurer::
+reconfigure(const DocumentDBConfig &newConfig,
+            const DocumentDBConfig &oldConfig,
+            const ReconfigParams &params)
+{
+    assert(!params.shouldAttributeManagerChange());
+    AttributeCollectionSpec attrSpec(AttributeCollectionSpec::AttributeList(),
+                                    0, 0);
+    reconfigure(newConfig, oldConfig, attrSpec, params);
+}
+
+namespace {
+
+IReprocessingInitializer::UP
+createAttributeReprocessingInitializer(const DocumentDBConfig &newConfig,
+                                       const IAttributeManager::SP &newAttrMgr,
+                                       const DocumentDBConfig &oldConfig,
+                                       const IAttributeManager::SP &oldAttrMgr,
+                                       const vespalib::string &subDbName)
+{
+    const document::DocumentType *newDocType = newConfig.getDocumentType();
+    const document::DocumentType *oldDocType = oldConfig.getDocumentType();
+    assert(newDocType != nullptr);
+    assert(oldDocType != nullptr);
+    return IReprocessingInitializer::UP(new AttributeReprocessingInitializer(
+            ARIConfig(newAttrMgr, *newConfig.getSchemaSP(),
+                    IDocumentTypeInspector::SP(new DocumentTypeInspector(*newDocType))),
+                    ARIConfig(oldAttrMgr, *oldConfig.getSchemaSP(),
+                    IDocumentTypeInspector::SP(new DocumentTypeInspector(*oldDocType))),
+                    subDbName));
+}
+
+}
+
+IReprocessingInitializer::UP
+SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig,
+                                          const DocumentDBConfig &oldConfig,
+                                          const AttributeCollectionSpec &attrSpec,
+                                          const ReconfigParams &params)
+{
+    bool shouldMatchViewChange = false;
+    bool shouldSearchViewChange = false;
+    bool shouldFeedViewChange = params.shouldSchemaChange();
+
+    SearchView::SP searchView = _searchView.get();
+    Matchers::SP matchers = searchView->getMatchers();
+    if (params.shouldMatchersChange()) {
+        Matchers::SP newMatchers(
+                createMatchers(newConfig.getSchemaSP(),
+                               newConfig.getRankProfilesConfig()).
+                release());
+        matchers = newMatchers;
+        shouldMatchViewChange = true;
+    }
+    IReprocessingInitializer::UP initializer;
+    IAttributeManager::SP attrMgr = searchView->getAttributeManager();
+    IAttributeWriter::SP attrWriter = _feedView.get()->getAttributeWriter();
+    if (params.shouldAttributeManagerChange()) {
+        IAttributeManager::SP newAttrMgr = attrMgr->create(attrSpec);
+        IAttributeManager::SP oldAttrMgr = attrMgr;
+        attrMgr = newAttrMgr;
+        shouldMatchViewChange = true;
+
+        IAttributeWriter::SP newAttrWriter(new AttributeWriter(newAttrMgr));
+        attrWriter = newAttrWriter;
+        shouldFeedViewChange = true;
+        initializer.reset(createAttributeReprocessingInitializer(newConfig, newAttrMgr,
+                oldConfig, oldAttrMgr, _subDbName).release());
+    }
+
+    ISummaryManager::ISummarySetup::SP sumSetup =
+        _searchView.get()->getSummarySetup();
+    if (params.shouldSummaryManagerChange() ||
+        params.shouldAttributeManagerChange())
+    {
+        ISummaryManager::SP sumMgr(_summaryMgr);
+        ISummaryManager::ISummarySetup::SP newSumSetup =
+            sumMgr->createSummarySetup(newConfig.getSummaryConfig(),
+                                       newConfig.getSummarymapConfig(),
+                                       newConfig.getJuniperrcConfig(),
+                                       newConfig.getDocumentTypeRepoSP(),
+                                       attrMgr);
+        sumSetup = newSumSetup;
+        shouldSearchViewChange = true;
+    }
+
+    if (shouldMatchViewChange) {
+        IndexSearchable::SP indexSearchable = searchView->getIndexSearchable();
+        reconfigureMatchView(matchers, indexSearchable, attrMgr);
+        searchView = _searchView.get();
+        shouldFeedViewChange = true;
+    }
+
+    if (shouldSearchViewChange) {
+        reconfigureSearchView(sumSetup, searchView->getMatchView());
+    }
+
+    if (shouldFeedViewChange) {
+        SearchableFeedView::SP curr = _feedView.get();
+        reconfigureFeedView(curr->getIndexWriter(),
+                            curr->getSummaryAdapter(),
+                            attrWriter,
+                            newConfig.getSchemaSP(),
+                            newConfig.getDocumentTypeRepoSP(),
+                            searchView);
+    }
+    return initializer;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h
new file mode 100644
index 00000000000..dd177a8de01
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentdbconfigmanager.h"
+#include "isummaryadapter.h"
+#include "matchers.h"
+#include "matchview.h"
+#include "reconfig_params.h"
+#include "searchable_feed_view.h"
+#include "searchview.h"
+#include <vespa/searchcore/proton/attribute/i_attribute_writer.h>
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include <vespa/searchcore/proton/index/i_index_writer.h>
+#include <vespa/searchcore/proton/reprocessing/i_reprocessing_initializer.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchsummary/config/config-juniperrc.h>
+#include <vespa/searchcommon/common/schema.h>
+#include <vespa/config-attributes.h>
+#include <vespa/config-indexschema.h>
+#include <vespa/config-rank-profiles.h>
+#include <vespa/config-summary.h>
+#include <vespa/config-summarymap.h>
+#include <vespa/vespalib/util/varholder.h>
+
+namespace proton {
+
+/**
+ * Class used to reconfig the feed view and search view used in a searchable sub database.
+ */
+class SearchableDocSubDBConfigurer : public boost::noncopyable
+{
+private:
+    typedef vespalib::VarHolder<SearchView::SP> SearchViewHolder;
+    typedef vespalib::VarHolder<SearchableFeedView::SP> FeedViewHolder;
+    const ISummaryManager::SP & _summaryMgr;
+    SearchViewHolder          & _searchView;
+    FeedViewHolder            & _feedView;
+    matching::QueryLimiter    & _queryLimiter;
+    const vespalib::Clock     & _clock;
+    vespalib::string            _subDbName;
+    uint32_t                    _distributionKey;
+
+    void
+    reconfigureFeedView(const SearchView::SP &searchView);
+
+    void
+    reconfigureFeedView(const IIndexWriter::SP &indexWriter,
+                        const ISummaryAdapter::SP &summaryAdapter,
+                        const IAttributeWriter::SP &attrWriter,
+                        const search::index::Schema::SP &schema,
+                        const document::DocumentTypeRepo::SP &repo,
+                        const SearchView::SP &searchView);
+
+    void
+    reconfigureMatchView(const searchcorespi::IndexSearchable::SP &indexSearchable);
+
+    void
+    reconfigureMatchView(const Matchers::SP &matchers,
+                         const searchcorespi::IndexSearchable::SP &indexSearchable,
+                         const IAttributeManager::SP &attrMgr);
+
+    void
+    reconfigureSearchView(const MatchView::SP &matchView);
+
+    void
+    reconfigureSearchView(const ISummaryManager::ISummarySetup::SP &summarySetup,
+                           const MatchView::SP &matchView);
+
+public:
+    SearchableDocSubDBConfigurer(const ISummaryManager::SP & summaryMgr,
+                                 SearchViewHolder & searchView,
+                                 FeedViewHolder & feedView,
+                                 matching::QueryLimiter & queryLimiter,
+                                 const vespalib::Clock & clock,
+                                 const vespalib::string &subDbName,
+                                 uint32_t distributionKey);
+
+    Matchers::UP
+    createMatchers(const search::index::Schema::SP &schema,
+                   const vespa::config::search::RankProfilesConfig &cfg);
+
+    void
+    reconfigureIndexSearchable();
+
+    void
+    reconfigure(const DocumentDBConfig &newConfig,
+                const DocumentDBConfig &oldConfig,
+                const ReconfigParams &params);
+
+    IReprocessingInitializer::UP
+    reconfigure(const DocumentDBConfig &newConfig,
+                const DocumentDBConfig &oldConfig,
+                const AttributeCollectionSpec &attrSpec,
+                const ReconfigParams &params);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_document_retriever.cpp
new file mode 100644
index 00000000000..11f103ca7b4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_document_retriever.cpp
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "searchable_document_retriever.h"
+
+namespace proton {
+
+SearchableDocumentRetriever::SearchableDocumentRetriever(
+        const SearchableFeedView::SP &fw, const SearchView::SP &sv)
+    : DocumentRetriever(fw->getPersistentParams()._docTypeName,
+                        *fw->getDocumentTypeRepo(),
+                        *fw->getSchema(),
+                        *sv->getDocumentMetaStore(),
+                        *sv->getAttributeManager(),
+                        fw->getDocumentStore()),
+      feedView(fw)
+{
+}
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/searchable_document_retriever.h
new file mode 100644
index 00000000000..dd27e549af7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_document_retriever.h
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentretriever.h"
+#include "searchable_feed_view.h"
+#include "searchview.h"
+
+namespace proton {
+
+struct SearchableDocumentRetriever : DocumentRetriever {
+    SearchableFeedView::SP feedView;
+
+    // Assumes the FeedView also ensures that the MatchView stays alive.
+    SearchableDocumentRetriever(const SearchableFeedView::SP &fw, const SearchView::SP &sv);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
new file mode 100644
index 00000000000..75ab2181db6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
@@ -0,0 +1,305 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.searchable_feed_view");
+#include "searchable_feed_view.h"
+#include "ireplayconfig.h"
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/documentapi/messagebus/messages/documentreply.h>
+#include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
+#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+#include <vespa/searchcore/proton/matching/match_context.h>
+#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/searchlib/common/lambdatask.h>
+#include "forcecommitcontext.h"
+#include "operationdonecontext.h"
+#include "removedonecontext.h"
+
+using document::BucketId;
+using document::Document;
+using document::DocumentId;
+using document::DocumentTypeRepo;
+using document::DocumentUpdate;
+using documentapi::DocumentProtocol;
+using documentapi::DocumentReply;
+using documentapi::RemoveDocumentReply;
+using documentapi::UpdateDocumentReply;
+using proton::matching::ISearchContext;
+using proton::matching::MatchContext;
+using proton::matching::Matcher;
+using search::index::Schema;
+using storage::spi::BucketInfoResult;
+using storage::spi::Timestamp;
+using vespalib::IllegalStateException;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+using vespalib::make_string;
+using search::makeLambdaTask;
+
+namespace proton {
+
+
+namespace
+{
+
+bool shouldTrace(StoreOnlyFeedView::OnOperationDoneType onWriteDone,
+                 uint32_t traceLevel)
+{
+    return onWriteDone && onWriteDone->shouldTrace(traceLevel);
+}
+
+}
+
+SearchableFeedView::SearchableFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx,
+                                       const PersistentParams &params,
+                                       const FastAccessFeedView::Context &fastUpdateCtx,
+                                       Context ctx)
+    : Parent(storeOnlyCtx, params, fastUpdateCtx),
+      _indexWriter(ctx._indexWriter),
+      _hasIndexedFields(_schema->getNumIndexFields() > 0)
+{
+}
+
+
+void
+SearchableFeedView::performSync()
+{
+    // Called by index write thread, delays when sync() method on it completes.
+    assert(_writeService.index().isCurrentThread());
+    _writeService.indexFieldInverter().sync();
+    _writeService.indexFieldWriter().sync();
+}
+
+
+void
+SearchableFeedView::sync()
+{
+    assert(_writeService.master().isCurrentThread());
+    Parent::sync();
+    indexExecute(makeClosure(this, &proton::SearchableFeedView::performSync));
+    _writeService.index().sync();
+}
+
+
+void
+SearchableFeedView::indexExecute(vespalib::Closure::UP closure)
+{
+    _writeService.index().execute(makeTask(std::move(closure)));
+}
+
+
+void
+SearchableFeedView::putIndexedFields(SerialNum serialNum,
+                                     search::DocumentIdT lid,
+                                     const Document::SP &newDoc,
+                                     bool immediateCommit,
+                                     OnOperationDoneType onWriteDone)
+{
+    if (!hasIndexedFields()) {
+        return;
+    }
+    _writeService.index().execute(
+            makeLambdaTask([=]
+                           { performIndexPut(serialNum, lid, newDoc,
+                                   immediateCommit, onWriteDone); }));
+}
+
+
+void
+SearchableFeedView::performIndexPut(SerialNum serialNum,
+                                    search::DocumentIdT lid,
+                                    const Document::SP &doc,
+                                    bool immediateCommit,
+                                    OnOperationDoneType onWriteDone)
+{
+    assert(_writeService.index().isCurrentThread());
+    VLOG(getDebugLevel(lid, doc->getId()),
+        "database(%s): performIndexPut: serialNum(%" PRIu64 "), "
+        "docId(%s), lid(%d)",
+        _params._docTypeName.toString().c_str(),
+        serialNum,
+        doc->getId().toString().c_str(), lid);
+
+    _indexWriter->put(serialNum, *doc, lid);
+    if (immediateCommit) {
+        _indexWriter->commit(serialNum, onWriteDone);
+    }
+    if (shouldTrace(onWriteDone, 1)) {
+        FeedToken *token = onWriteDone->getToken();
+        token->trace(1, "Document indexed = . New Value : " +
+                doc->toString(token->shouldTrace(2)));
+    }
+}
+
+
+void
+SearchableFeedView::heartBeatIndexedFields(SerialNum serialNum)
+{
+    indexExecute(makeClosure(this,
+                             &proton::SearchableFeedView::performIndexHeartBeat,
+                             serialNum));
+}
+
+
+void
+SearchableFeedView::performIndexHeartBeat(SerialNum serialNum)
+{
+    _indexWriter->heartBeat(serialNum);
+}
+
+
+SearchableFeedView::UpdateScope
+SearchableFeedView::getUpdateScope(const DocumentUpdate &upd)
+{
+    UpdateScope updateScope;
+    const Schema &schema = *_schema;
+    for(size_t i(0), m(upd.getUpdates().size());
+        !(updateScope._indexedFields && updateScope._nonAttributeFields) &&
+            (i < m); i++) {
+        const document::FieldUpdate & fu(upd.getUpdates()[i]);
+        const vespalib::string &name = fu.getField().getName();
+        if (schema.isIndexField(name)) {
+            updateScope._indexedFields = true;
+        }
+        if (!fastPartialUpdateAttribute(name)) {
+            updateScope._nonAttributeFields = true;
+        }
+    }
+    return updateScope;
+}
+
+
+void
+SearchableFeedView::updateIndexedFields(SerialNum serialNum,
+                                        search::DocumentIdT lid,
+                                        const Document::SP &newDoc,
+                                        bool immediateCommit,
+                                        OnOperationDoneType onWriteDone)
+{
+    if (shouldTrace(onWriteDone, 1)) {
+        onWriteDone->getToken()->trace(1, "Then we can update the index.");
+    }
+    _writeService.index().execute(
+            makeLambdaTask([=]()
+                           { performIndexPut(serialNum, lid, newDoc,
+                                   immediateCommit, onWriteDone); }));
+}
+
+
+void
+SearchableFeedView::removeIndexedFields(SerialNum serialNum,
+                                        search::DocumentIdT lid,
+                                        bool immediateCommit,
+                                        OnRemoveDoneType onWriteDone)
+{
+    if (!hasIndexedFields()) {
+        return;
+    }
+    _writeService.index().execute(
+            makeLambdaTask([=]()
+                           { performIndexRemove(serialNum, lid,
+                                                immediateCommit,
+                                                onWriteDone); }));
+}
+
+
+void
+SearchableFeedView::performIndexRemove(SerialNum serialNum,
+                                       search::DocumentIdT lid,
+                                       bool immediateCommit,
+                                       OnRemoveDoneType onWriteDone)
+{
+    assert(_writeService.index().isCurrentThread());
+    VLOG(getDebugLevel(lid, NULL),
+        "database(%s): performIndexRemove: serialNum(%" PRIu64 "), "
+        "lid(%d)",
+         _params._docTypeName.toString().c_str(),
+         serialNum,
+         lid);
+
+    _indexWriter->remove(serialNum, lid);
+    if (immediateCommit) {
+        _indexWriter->commit(serialNum, onWriteDone);
+    }
+    FeedToken *token = onWriteDone ? onWriteDone->getToken() : nullptr;
+    if (token != nullptr && token->shouldTrace(1)) {
+        token->trace(1, make_string("Document with lid %d removed.", lid));
+    }
+}
+
+
+void
+SearchableFeedView::performIndexRemove(SerialNum serialNum,
+                                       const LidVector &lidsToRemove,
+                                       bool immediateCommit,
+                                       OnWriteDoneType onWriteDone)
+{
+    assert(_writeService.index().isCurrentThread());
+    for (const auto lid : lidsToRemove) {
+        VLOG(getDebugLevel(lid, NULL),
+             "database(%s): performIndexRemove: serialNum(%" PRIu64 "), "
+             "lid(%d)",
+             _params._docTypeName.toString().c_str(),
+             serialNum,
+             lid);
+
+        _indexWriter->remove(serialNum, lid);
+    }
+    if (immediateCommit) {
+        _indexWriter->commit(serialNum, onWriteDone);
+    }
+}
+
+
+void
+SearchableFeedView::removeIndexedFields(SerialNum serialNum,
+                                        const LidVector &lidsToRemove,
+                                        bool immediateCommit,
+                                        OnWriteDoneType onWriteDone)
+{
+    if (!hasIndexedFields())
+        return;
+
+    _writeService.index().execute(
+            makeLambdaTask([=]() { performIndexRemove(serialNum,
+                                                      lidsToRemove,
+                                                      immediateCommit,
+                                                      onWriteDone); }));
+}
+
+void
+SearchableFeedView::internalDeleteBucket(const DeleteBucketOperation &delOp)
+{
+    Parent::internalDeleteBucket(delOp);
+    _writeService.sync();
+}
+
+
+void
+SearchableFeedView::performIndexForceCommit(SerialNum serialNum,
+                                            OnForceCommitDoneType onCommitDone)
+{
+    assert(_writeService.index().isCurrentThread());
+    _indexWriter->commit(serialNum, onCommitDone);
+}
+
+
+void
+SearchableFeedView::forceCommit(SerialNum serialNum,
+                                OnForceCommitDoneType onCommitDone)
+{
+    Parent::forceCommit(serialNum, onCommitDone);
+    _writeService.index().execute(
+            makeLambdaTask([=]()
+                           { performIndexForceCommit(serialNum,
+                                                     onCommitDone); }));
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
new file mode 100644
index 00000000000..d795fe40eb4
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "fast_access_feed_view.h"
+#include "matchview.h"
+#include <vespa/searchcore/proton/attribute/i_attribute_writer.h>
+#include <vespa/searchcore/proton/index/i_index_writer.h>
+
+namespace proton
+{
+
+/**
+ * The feed view used by the searchable sub database.
+ *
+ * Handles inserting/updating/removing of documents to the underlying attributes,
+ * memory index and document store.
+ */
+class SearchableFeedView : public FastAccessFeedView
+{
+    typedef FastAccessFeedView Parent;
+public:
+    typedef std::unique_ptr<SearchableFeedView> UP;
+    typedef std::shared_ptr<SearchableFeedView> SP;
+
+    struct Context {
+        const IIndexWriter::SP &_indexWriter;
+
+        Context(const IIndexWriter::SP &indexWriter)
+            : _indexWriter(indexWriter)
+        { }
+    };
+
+private:
+    const IIndexWriter::SP    _indexWriter;
+    const bool                _hasIndexedFields;
+
+    bool hasIndexedFields() const { return _hasIndexedFields; }
+    void indexExecute(vespalib::Closure::UP closure);
+
+    void
+    performIndexPut(SerialNum serialNum,
+                    search::DocumentIdT lid,
+                    const document::Document::SP &doc,
+                    bool immediateCommit,
+                    OnOperationDoneType onWriteDone);
+
+    void
+    performIndexRemove(SerialNum serialNum,
+                       search::DocumentIdT lid,
+                       bool immediateCommit,
+                       OnRemoveDoneType onWriteDone);
+
+    void
+    performIndexRemove(SerialNum serialNum,
+                       const LidVector &lidsToRemove,
+                       bool immediateCommit,
+                       OnWriteDoneType onWriteDone);
+
+    void performIndexHeartBeat(SerialNum serialNum);
+
+
+    void internalDeleteBucket(const DeleteBucketOperation &delOp) override;
+    void performSync();
+    void heartBeatIndexedFields(SerialNum serialNum) override;
+
+    virtual void
+    putIndexedFields(SerialNum serialNum,
+                     search::DocumentIdT lid,
+                     const document::Document::SP &newDoc,
+                     bool immediateCommit,
+                     OnOperationDoneType onWriteDone) override;
+
+    UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override;
+
+    virtual void
+    updateIndexedFields(SerialNum serialNum,
+                        search::DocumentIdT lid,
+                        const document::Document::SP &newDoc,
+                        bool immediateCommit,
+                        OnOperationDoneType onWriteDone) override;
+
+    virtual void
+    removeIndexedFields(SerialNum serialNum,
+                        search::DocumentIdT lid,
+                        bool immediateCommit,
+                        OnRemoveDoneType onWriteDone) override;
+
+    virtual void
+    removeIndexedFields(SerialNum serialNum,
+                        const LidVector &lidsToRemove,
+                        bool immediateCommit,
+                        OnWriteDoneType onWriteDone) override;
+
+    void performIndexForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone);
+    void forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) override;
+
+public:
+    SearchableFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx,
+                       const PersistentParams &params,
+                       const FastAccessFeedView::Context &fastUpdateCtx,
+                       Context ctx);
+
+    virtual ~SearchableFeedView() {}
+    const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; }
+    void sync() override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
new file mode 100644
index 00000000000..e2db3bf0d70
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -0,0 +1,369 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.searchabledocsubdb");
+
+#include "searchable_document_retriever.h"
+#include "searchabledocsubdb.h"
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/flushengine/threadedflushtarget.h>
+#include <vespa/searchcore/proton/index/index_manager_initializer.h>
+#include <vespa/searchcore/proton/index/index_writer.h>
+#include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h>
+#include <vespa/searchcore/proton/metrics/metricswireservice.h>
+#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h>
+#include <vespa/searchlib/common/indexmetainfo.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/closuretask.h>
+
+using vespa::config::search::AttributesConfig;
+using vespa::config::search::RankProfilesConfig;
+using vespa::config::search::core::ProtonConfig;
+using proton::matching::MatchingStats;
+using proton::matching::SessionManager;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::GrowStrategy;
+using search::TuneFileDocumentDB;
+using search::index::Schema;
+using search::SerialNum;
+using searchcorespi::IndexManagerConfig;
+using searchcorespi::index::IndexMaintainerConfig;
+using searchcorespi::index::IndexMaintainerContext;
+using vespalib::IllegalStateException;
+using vespalib::ThreadStackExecutorBase;
+using namespace searchcorespi;
+
+
+namespace proton {
+
+SearchableDocSubDB::SearchableDocSubDB(const Config &cfg,
+                                       const Context &ctx)
+    : FastAccessDocSubDB(cfg._fastUpdCfg,
+                         ctx._fastUpdCtx),
+      IIndexManager::Reconfigurer(),
+      _indexMgr(),
+      _indexWriter(),
+      _rSearchView(),
+      _rFeedView(),
+      _configurer(_iSummaryMgr, _rSearchView, _rFeedView, ctx._queryLimiter, ctx._clock,
+                  getSubDbName(), ctx._fastUpdCtx._storeOnlyCtx._owner.getDistributionKey()),
+      _numSearcherThreads(cfg._numSearcherThreads),
+      _warmupExecutor(ctx._warmupExecutor)
+{
+}
+
+SearchableDocSubDB::~SearchableDocSubDB()
+{
+    // XXX: Disk index wrappers should not live longer than index manager
+    // which owns map of active disk indexes.
+    clearViews();
+}
+
+void
+SearchableDocSubDB::syncViews()
+{
+    _iSearchView.set(_rSearchView.get());
+    _iFeedView.set(_rFeedView.get());
+    _owner.syncFeedView();
+}
+
+SerialNum
+SearchableDocSubDB::getOldestFlushedSerial()
+{
+    SerialNum lowest(Parent::getOldestFlushedSerial());
+    lowest = std::min(lowest, _indexMgr->getFlushedSerialNum());
+    return lowest;
+}
+
+SerialNum
+SearchableDocSubDB::getNewestFlushedSerial()
+{
+    SerialNum highest(Parent::getNewestFlushedSerial());
+    highest = std::max(highest, _indexMgr->getFlushedSerialNum());
+    return highest;
+}
+
+initializer::InitializerTask::SP
+SearchableDocSubDB::
+createIndexManagerInitializer(const DocumentDBConfig &configSnapshot,
+                              const Schema::SP &unionSchema,
+                              const ProtonConfig::Index &indexCfg,
+                              std::shared_ptr<searchcorespi::IIndexManager::SP> indexManager) const
+{
+    Schema::SP schema(configSnapshot.getSchemaSP());
+    vespalib::string vespaIndexDir(_baseDir + "/index");
+    // Note: const_cast for reconfigurer role
+    return std::make_shared<IndexManagerInitializer>
+        (vespaIndexDir,
+         indexCfg.warmup.time,
+         indexCfg.maxflushed,
+         indexCfg.cache.size,
+         *schema,
+         *unionSchema,
+         const_cast<SearchableDocSubDB &>(*this),
+         _writeService,
+         _warmupExecutor,
+         configSnapshot.getTuneFileDocumentDBSP()->_index,
+         configSnapshot.getTuneFileDocumentDBSP()->_attr,
+         _fileHeaderContext,
+         indexManager);
+}
+
+void
+SearchableDocSubDB::setupIndexManager(searchcorespi::IIndexManager::SP
+                                      indexManager)
+{
+    _indexMgr = indexManager;
+    _indexWriter.reset(new IndexWriter(_indexMgr));
+}
+
+DocumentSubDbInitializer::UP
+SearchableDocSubDB::
+createInitializer(const DocumentDBConfig &configSnapshot,
+                  SerialNum configSerialNum,
+                  const Schema::SP &unionSchema,
+                  const ProtonConfig::Summary &protonSummaryCfg,
+                  const ProtonConfig::Index &indexCfg) const
+{
+    auto result = Parent::createInitializer(configSnapshot,
+                                            configSerialNum,
+                                            unionSchema,
+                                            protonSummaryCfg,
+                                            indexCfg);
+    auto indexTask = createIndexManagerInitializer(configSnapshot,
+                                                   unionSchema,
+                                                   indexCfg,
+                                                   result->writableResult().
+                                                   writableIndexManager());
+    result->addDependency(indexTask);
+    return result;
+}
+
+void
+SearchableDocSubDB::setup(const DocumentSubDbInitializerResult &initResult)
+{
+    Parent::setup(initResult);
+    setupIndexManager(initResult.indexManager());
+    _docIdLimit.set(_dms->getCommittedDocIdLimit());
+}
+
+void
+SearchableDocSubDB::
+reconfigureMatchingMetrics(const RankProfilesConfig &cfg)
+{
+    _metricsWireService.cleanRankProfiles(_metrics);
+    for (const auto &profile : cfg.rankprofile) {
+        search::fef::Properties properties;
+        for (const auto &property : profile.fef.property) {
+            properties.add(property.name,
+                           property.value);
+        }
+        size_t numDocIdPartitions = search::fef::indexproperties::matching::NumThreadsPerSearch::lookup(properties);
+        _metricsWireService.addRankProfile(_metrics,
+                                           profile.name,
+                                           numDocIdPartitions);
+    }
+}
+
+IReprocessingTask::List
+SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                                const DocumentDBConfig &oldConfigSnapshot,
+                                SerialNum serialNum,
+                                const ReconfigParams params)
+{
+    IReprocessingTask::List tasks;
+    updateLidReuseDelayer(&newConfigSnapshot);
+    if (params.shouldMatchersChange() && _addMetrics) {
+        reconfigureMatchingMetrics(newConfigSnapshot.getRankProfilesConfig());
+    }
+    if (params.shouldAttributeManagerChange()) {
+        proton::IAttributeManager::SP oldMgr = getAttributeManager();
+        AttributeCollectionSpec::UP attrSpec =
+            createAttributeSpec(newConfigSnapshot.getAttributesConfig(), serialNum);
+        IReprocessingInitializer::UP initializer =
+                _configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec, params);
+        if (initializer.get() != nullptr && initializer->hasReprocessors()) {
+            tasks.push_back(IReprocessingTask::SP(createReprocessingTask(*initializer,
+                    newConfigSnapshot.getDocumentTypeRepoSP()).release()));
+        }
+        proton::IAttributeManager::SP newMgr = getAttributeManager();
+        if (_addMetrics) {
+            reconfigureAttributeMetrics(*newMgr, *oldMgr);
+        }
+    } else {
+        _configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, params);
+    }
+    syncViews();
+    return tasks;
+}
+
+void
+SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot,
+                              const SessionManager::SP &sessionManager)
+{
+    assert(_writeService.master().isCurrentThread());
+
+    AttributeManager::SP attrMgr = getAndResetInitAttributeManager();
+    const Schema::SP &schema = configSnapshot.getSchemaSP();
+    const IIndexManager::SP &indexMgr = getIndexManager();
+    Matchers::SP matchers(_configurer.
+                          createMatchers(schema,
+                                  configSnapshot.getRankProfilesConfig()).
+                          release());
+    MatchView::SP matchView(new MatchView(matchers,
+                                          indexMgr->getSearchable(),
+                                          attrMgr,
+                                          sessionManager,
+                                          _metaStoreCtx,
+                                          _docIdLimit));
+    _rSearchView.set(SearchView::SP(
+                              new SearchView(
+                                      getSummaryManager()->createSummarySetup(
+                                              configSnapshot.getSummaryConfig(),
+                                              configSnapshot.getSummarymapConfig(),
+                                              configSnapshot.getJuniperrcConfig(),
+                                              configSnapshot.getDocumentTypeRepoSP(),
+                                              matchView->getAttributeManager()),
+                                      matchView)));
+
+    IAttributeWriter::SP attrWriter(new AttributeWriter(attrMgr));
+    {
+        vespalib::LockGuard guard(_configLock);
+        initFeedView(attrWriter, configSnapshot);
+    }
+    if (_addMetrics) {
+        reconfigureMatchingMetrics(configSnapshot.getRankProfilesConfig());
+    }
+}
+
+void
+SearchableDocSubDB::initFeedView(const IAttributeWriter::SP &attrWriter,
+                                 const DocumentDBConfig &configSnapshot)
+{
+    assert(_writeService.master().isCurrentThread());
+    SearchableFeedView::UP feedView(new SearchableFeedView(getStoreOnlyFeedViewContext(configSnapshot),
+            getFeedViewPersistentParams(),
+            FastAccessFeedView::Context(attrWriter, _docIdLimit),
+            SearchableFeedView::Context(getIndexWriter())));
+
+    // XXX: Not exception safe.
+    _rFeedView.set(SearchableFeedView::SP(feedView.release()));
+    syncViews();
+}
+
+/**
+ * Handle reconfigure caused by index manager changing state.
+ *
+ * Flush engine is disabled (for all document dbs) during initial replay and
+ * recovery feed modes, the flush engine has not started.  For a resurrected
+ * document type, flushing might occur during replay.
+ */
+bool
+SearchableDocSubDB::
+reconfigure(vespalib::Closure0<bool>::UP closure)
+{
+    assert(_writeService.master().isCurrentThread());
+
+    _writeService.sync();
+
+    // Everything should be quiet now.
+
+    SearchView::SP oldSearchView = _rSearchView.get();
+    IFeedView::SP oldFeedView = _iFeedView.get();
+
+    bool ret = true;
+
+    if (closure.get() != NULL)
+        ret = closure->call();	// Perform index manager reconfiguration now
+    reconfigureIndexSearchable();
+    return ret;
+}
+
+void
+SearchableDocSubDB::reconfigureIndexSearchable()
+{
+    vespalib::LockGuard guard(_configLock);
+    // Create new views as needed.
+    _configurer.reconfigureIndexSearchable();
+    // Activate new feed view at once
+    syncViews();
+}
+
+IFlushTarget::List
+SearchableDocSubDB::getFlushTargetsInternal()
+{
+    IFlushTarget::List ret(Parent::getFlushTargetsInternal());
+
+    IFlushTarget::List tmp = _indexMgr->getFlushTargets();
+    ret.insert(ret.end(), tmp.begin(), tmp.end());
+
+    return ret;
+}
+
+void
+SearchableDocSubDB::wipeHistory(SerialNum wipeSerial,
+                                const Schema &newHistorySchema,
+                                const Schema &wipeSchema)
+{
+    assert(_writeService.master().isCurrentThread());
+    SearchView::SP oldSearchView = _rSearchView.get();
+    IFeedView::SP oldFeedView = _iFeedView.get();
+    _indexMgr->wipeHistory(wipeSerial, newHistorySchema);
+    reconfigureIndexSearchable();
+    getAttributeManager()->wipeHistory(wipeSchema);
+}
+
+void
+SearchableDocSubDB::setIndexSchema(const Schema::SP &schema,
+                                   const Schema::SP &fusionSchema)
+{
+    assert(_writeService.master().isCurrentThread());
+
+    SearchView::SP oldSearchView = _rSearchView.get();
+    IFeedView::SP oldFeedView = _iFeedView.get();
+
+    _indexMgr->setSchema(*schema, *fusionSchema);
+    reconfigureIndexSearchable();
+}
+
+size_t
+SearchableDocSubDB::getNumActiveDocs() const
+{
+    return _metaStoreCtx->getReadGuard()->get().getNumActiveLids();
+}
+
+search::SearchableStats
+SearchableDocSubDB::getSearchableStats() const
+{
+    return _indexMgr->getSearchableStats();
+}
+
+IDocumentRetriever::UP
+SearchableDocSubDB::getDocumentRetriever()
+{
+    return IDocumentRetriever::UP(new SearchableDocumentRetriever(
+                    _rFeedView.get(), _rSearchView.get()));
+}
+
+MatchingStats
+SearchableDocSubDB::getMatcherStats(const vespalib::string &rankProfile) const
+{
+    return _rSearchView.get()->getMatcherStats(rankProfile);
+}
+
+void
+SearchableDocSubDB::updateLidReuseDelayer(const LidReuseDelayerConfig &config)
+{
+    Parent::updateLidReuseDelayer(config);
+    /*
+     * The lid reuse delayer should not have any pending lids stored at this
+     * time, since DocumentDB::applyConfig() calls forceCommit() on the
+     * feed view before applying the new config to the sub dbs.
+     */
+    _lidReuseDelayer->setHasIndexedFields(config.hasIndexedFields());
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
new file mode 100644
index 00000000000..71f84434e41
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -0,0 +1,205 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentdbconfig.h"
+#include "searchable_doc_subdb_configurer.h"
+#include "executorthreadingservice.h"
+#include "fast_access_doc_subdb.h"
+#include "feedhandler.h"
+#include "searchable_feed_view.h"
+#include "searchview.h"
+#include "summaryadapter.h"
+#include <memory>
+#include <vector>
+#include <vespa/searchcore/proton/attribute/attributemanager.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
+#include <vespa/searchcorespi/index/iindexmanager.h>
+#include <vespa/searchcore/proton/index/i_index_writer.h>
+#include <vespa/searchcore/proton/index/indexmanager.h>
+#include <vespa/searchcore/config/config-proton.h>
+#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
+#include <vespa/vespalib/util/varholder.h>
+
+
+namespace proton
+{
+
+class MetricsWireService;
+class DocumentDBMetrics;
+
+/**
+ * The searchable sub database supports searching and keeps all attribute fields in memory and
+ * inserts all index fields into the memory index in addition to storing documents in the
+ * underlying document store.
+ *
+ * This class is used directly by the "0.ready" sub database for handling all ready documents.
+ */
+class SearchableDocSubDB : public FastAccessDocSubDB,
+                           public searchcorespi::IIndexManager::Reconfigurer
+
+{
+public:
+    struct Config {
+        const FastAccessDocSubDB::Config _fastUpdCfg;
+        const size_t _numSearcherThreads;
+
+        Config(const FastAccessDocSubDB::Config &fastUpdCfg,
+               size_t numSearcherThreads)
+            : _fastUpdCfg(fastUpdCfg),
+              _numSearcherThreads(numSearcherThreads)
+        {
+        }
+    };
+
+    struct Context {
+        const FastAccessDocSubDB::Context _fastUpdCtx;
+        matching::QueryLimiter   &_queryLimiter;
+        const vespalib::Clock    &_clock;
+        vespalib::ThreadExecutor &_warmupExecutor;
+
+        Context(const FastAccessDocSubDB::Context &fastUpdCtx,
+                matching::QueryLimiter &queryLimiter,
+                const vespalib::Clock &clock,
+                vespalib::ThreadExecutor &warmupExecutor)
+            : _fastUpdCtx(fastUpdCtx),
+              _queryLimiter(queryLimiter),
+              _clock(clock),
+              _warmupExecutor(warmupExecutor)
+        {
+        }
+    };
+
+private:
+    typedef FastAccessDocSubDB Parent;
+
+    IIndexManager::SP                   _indexMgr;
+    IIndexWriter::SP                    _indexWriter;
+    vespalib::VarHolder<SearchView::SP> _rSearchView;
+    vespalib::VarHolder<SearchableFeedView::SP>   _rFeedView;
+    SearchableDocSubDBConfigurer                _configurer;
+    const size_t                        _numSearcherThreads;
+    vespalib::ThreadExecutor           &_warmupExecutor;
+
+    // Note: lifetime of indexManager must be handled by caller.
+    initializer::InitializerTask::SP
+    createIndexManagerInitializer(const DocumentDBConfig &configSnapshot,
+                                  const search::index::Schema::SP &unionSchema,
+                                  const vespa::config::search::core::ProtonConfig::Index &indexCfg,
+                                  std::shared_ptr<searchcorespi::IIndexManager::SP> indexManager) const;
+
+    void setupIndexManager(searchcorespi::IIndexManager::SP indexManager);
+
+    void
+    initFeedView(const IAttributeWriter::SP &attrWriter,
+                 const DocumentDBConfig &configSnapshot);
+
+    void
+    reconfigureMatchingMetrics(const vespa::config::search::RankProfilesConfig &config);
+
+    /**
+     * Implements IndexManagerReconfigurer API.
+     */
+    virtual bool
+    reconfigure(vespalib::Closure0<bool>::UP closure);
+
+    void
+    reconfigureIndexSearchable();
+
+    void
+    syncViews();
+
+protected:
+    virtual IFlushTarget::List
+    getFlushTargetsInternal();
+
+    using Parent::updateLidReuseDelayer;
+
+    virtual void
+    updateLidReuseDelayer(const LidReuseDelayerConfig &config) override;
+public:
+    SearchableDocSubDB(const Config &cfg,
+                       const Context &ctx);
+
+    virtual
+    ~SearchableDocSubDB();
+
+    virtual DocumentSubDbInitializer::UP
+    createInitializer(const DocumentDBConfig &configSnapshot,
+                      SerialNum configSerialNum,
+                      const search::index::Schema::SP &unionSchema,
+                      const vespa::config::search::core::
+                      ProtonConfig::Summary &protonSummaryCfg,
+                      const vespa::config::search::core::
+                      ProtonConfig::Index &indexCfg) const override;
+
+    virtual void setup(const DocumentSubDbInitializerResult &initResult)
+        override;
+
+    virtual void
+    initViews(const DocumentDBConfig &configSnapshot,
+              const matching::SessionManager::SP &sessionManager);
+
+    virtual IReprocessingTask::List
+    applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                const DocumentDBConfig &oldConfigSnapshot,
+                SerialNum serialNum,
+                const ReconfigParams params);
+
+    virtual void
+    clearViews()
+    {
+        _rFeedView.clear();
+        _rSearchView.clear();
+        Parent::clearViews();
+    }
+
+    virtual proton::IAttributeManager::SP
+    getAttributeManager() const
+    {
+        return _rSearchView.get()->getAttributeManager();
+    }
+
+    virtual const IIndexManager::SP &
+    getIndexManager() const
+    {
+        return _indexMgr;
+    }
+
+    virtual const IIndexWriter::SP &
+    getIndexWriter() const
+    {
+        return _indexWriter;
+    }
+
+    virtual SerialNum
+    getOldestFlushedSerial();
+
+    virtual SerialNum
+    getNewestFlushedSerial();
+
+    virtual void
+    wipeHistory(SerialNum wipeSerial,
+                const search::index::Schema &newHistorySchema,
+                const search::index::Schema &wipeSchema);
+
+    virtual void
+    setIndexSchema(const search::index::Schema::SP &schema,
+                   const search::index::Schema::SP &fusionSchema);
+
+    virtual size_t
+    getNumActiveDocs() const override;
+
+    virtual search::SearchableStats
+    getSearchableStats() const;
+
+    virtual IDocumentRetriever::UP
+    getDocumentRetriever();
+
+    virtual matching::MatchingStats
+    getMatcherStats(const vespalib::string &rankProfile) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp b/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp
new file mode 100644
index 00000000000..99770b0af75
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "searchcontext.h"
+
+using search::queryeval::Searchable;
+
+namespace proton {
+
+Searchable &
+SearchContext::getIndexes()
+{
+    return *_indexSearchable;
+}
+
+Searchable &
+SearchContext::getAttributes()
+{
+    return _attributeBlueprintFactory;
+}
+
+uint32_t SearchContext::getDocIdLimit()
+{
+    return _docIdLimit;
+}
+
+SearchContext::SearchContext(const Searchable::SP &indexSearchable, uint32_t docIdLimit)
+    : _indexSearchable(indexSearchable),
+      _attributeBlueprintFactory(),
+      _docIdLimit(docIdLimit)
+{
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchcontext.h b/searchcore/src/vespa/searchcore/proton/server/searchcontext.h
new file mode 100644
index 00000000000..058390d7c64
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchcontext.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/noncopyable.hpp>
+#include <vespa/searchlib/attribute/attribute_blueprint_factory.h>
+#include <vespa/searchcore/proton/matching/isearchcontext.h>
+
+namespace proton {
+
+/**
+ * Defines the context for a search within the document type owned by
+ * this database. SearchContext contains the context for a search for
+ * the documenttype.  First create, search and rank, then group/sort,
+ * collect hits.
+ */
+class SearchContext : public boost::noncopyable,
+                      public matching::ISearchContext
+{
+private:
+    /// Snapshot of the indexes used.
+    Searchable::SP  _indexSearchable;
+    search::AttributeBlueprintFactory _attributeBlueprintFactory;
+    uint32_t                          _docIdLimit;
+
+    Searchable &getIndexes() override;
+    Searchable &getAttributes() override;
+    uint32_t getDocIdLimit() override;
+
+public:
+    SearchContext(const Searchable::SP &indexSearchable, uint32_t docIdLimit);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp
new file mode 100644
index 00000000000..f996135b5d7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.searchhandlerproxy");
+#include "searchhandlerproxy.h"
+
+namespace proton {
+
+SearchHandlerProxy::SearchHandlerProxy(const DocumentDB::SP &documentDB)
+    : _documentDB(documentDB)
+{
+    _documentDB->retain();
+}
+
+
+SearchHandlerProxy::~SearchHandlerProxy(void)
+{
+    _documentDB->release();
+}
+
+
+search::engine::DocsumReply::UP
+SearchHandlerProxy::getDocsums(const search::engine::DocsumRequest & request)
+{
+    return _documentDB->getDocsums(request);
+}
+
+search::engine::SearchReply::UP
+SearchHandlerProxy::match(const ISearchHandler::SP &searchHandler,
+                          const search::engine::SearchRequest &req,
+                          vespalib::ThreadBundle &threadBundle) const
+{
+    return _documentDB->match(searchHandler, req, threadBundle);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h
new file mode 100644
index 00000000000..59bf6cdac32
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
+#include "documentdb.h"
+
+namespace proton {
+
+class SearchHandlerProxy : public boost::noncopyable,
+                           public ISearchHandler
+{
+private:
+    DocumentDB::SP _documentDB;
+public:
+    SearchHandlerProxy(const DocumentDB::SP &documentDB);
+
+    virtual
+    ~SearchHandlerProxy(void);
+
+    /**
+     * Implements ISearchHandler.
+     */
+    virtual search::engine::DocsumReply::UP
+    getDocsums(const search::engine::DocsumRequest & request);
+
+    /**
+     * @return Use the request and produce the matching result.
+     */
+    virtual search::engine::SearchReply::UP match(
+            const ISearchHandler::SP &searchHandler,
+            const search::engine::SearchRequest &req,
+            vespalib::ThreadBundle &threadBundle) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.cpp b/searchcore/src/vespa/searchcore/proton/server/searchview.cpp
new file mode 100644
index 00000000000..692be5fe183
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchview.cpp
@@ -0,0 +1,147 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.searchview");
+
+#include "searchcontext.h"
+#include "searchview.h"
+#include <vespa/searchcore/proton/docsummary/docsumcontext.h>
+#include <vespa/searchcore/proton/matching/match_context.h>
+
+using proton::matching::MatchContext;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::attribute::IAttributeContext;
+using search::docsummary::IDocsumStore;
+using search::engine::DocsumReply;
+using search::engine::DocsumRequest;
+
+namespace proton
+{
+
+using matching::ISearchContext;
+using matching::Matcher;
+
+namespace
+{
+
+/**
+ * Maps the gids in the request to lids using the given document meta store.
+ * A reader guard must be taken before calling this function.
+ **/
+void
+convertGidsToLids(const DocsumRequest & request,
+                  const search::IDocumentMetaStore &metaStore,
+                  uint32_t docIdLimit)
+{
+    document::GlobalId empty;
+    uint32_t lid = 0;
+    for (size_t i = 0; i < request.hits.size(); ++i) {
+        const DocsumRequest::Hit & h = request.hits[i];
+        if (metaStore.getLid(h.gid, lid) && lid < docIdLimit) {
+            h.docid = lid;
+        } else {
+            h.docid = search::endDocId;
+            LOG(debug,
+                "Document with global id '%s' is not in the document db,"
+                " will return empty docsum",
+                h.gid.toString().c_str());
+        }
+        LOG(spam,
+            "convertGidToLid(DocsumRequest): hit[%zu]: gid(%s) -> lid(%u)",
+            i, h.gid.toString().c_str(), h.docid);
+    }
+}
+
+/**
+ * Maps the lids in the reply to gids using the original request.
+ **/
+void
+convertLidsToGids(DocsumReply &reply, const DocsumRequest &request)
+{
+    LOG_ASSERT(reply.docsums.size() == request.hits.size());
+    for (size_t i = 0; i < reply.docsums.size(); ++i) {
+        const DocsumRequest::Hit & h = request.hits[i];
+        DocsumReply::Docsum & d = reply.docsums[i];
+        d.gid = h.gid;
+        LOG(spam,
+            "convertLidToGid(DocsumReply): docsum[%zu]: lid(%u) -> gid(%s)",
+            i, d.docid, d.gid.toString().c_str());
+    }
+}
+
+/**
+ * Create empty docsum reply
+ **/
+DocsumReply::UP
+createEmptyReply(const DocsumRequest & request)
+{
+    DocsumReply::UP reply(new DocsumReply());
+    for (size_t i = 0; i < request.hits.size(); ++i) {
+        reply->docsums.push_back(DocsumReply::Docsum());
+        reply->docsums.back().gid = request.hits[i].gid;
+    }
+    return reply;
+}
+
+}
+
+
+SearchView::SearchView(const ISummaryManager::ISummarySetup::SP &
+                       summarySetup,
+                       const MatchView::SP & matchView)
+    : boost::noncopyable(),
+      ISearchHandler(),
+      _summarySetup(summarySetup),
+      _matchView(matchView)
+{
+}
+
+
+DocsumReply::UP
+SearchView::getDocsums(const DocsumRequest & req)
+{
+    LOG(debug, "getDocsums(): resultClass(%s), numHits(%zu)",
+        req.resultClassName.c_str(), req.hits.size());
+    if (_summarySetup->getResultConfig().
+        LookupResultClassId(req.resultClassName.c_str()) ==
+        search::docsummary::ResultConfig::NoClassID()) {
+        LOG(warning,
+            "There is no summary class with name '%s' in the summary config. "
+            "Returning empty document summary for %zu hit(s)",
+            req.resultClassName.c_str(), req.hits.size());
+        return createEmptyReply(req);
+    }
+    { // convert from gids to lids
+        IDocumentMetaStoreContext::IReadGuard::UP readGuard =
+            _matchView->getDocumentMetaStore()->getReadGuard();
+        convertGidsToLids(req, readGuard->get(), _matchView->getDocIdLimit().get());
+    }
+    IDocsumStore::UP store(_summarySetup->createDocsumStore(req.resultClassName));
+    Matcher::SP matcher = _matchView->getMatcher(req.ranking);
+    MatchContext::UP mctx = _matchView->createContext();
+    DocsumContext::UP
+        ctx(new DocsumContext(req,
+                              _summarySetup->getDocsumWriter(),
+                              *store,
+                              matcher,
+                              mctx->getSearchContext(),
+                              mctx->getAttributeContext(),
+                              *_summarySetup->getAttributeManager(),
+                              *getSessionManager()));
+    DocsumReply::UP reply = ctx->getDocsums();
+    if ( ! req.useRootSlime()) {
+        convertLidsToGids(*reply, req);
+    }
+    return reply;
+}
+
+search::engine::SearchReply::UP
+SearchView::match(const ISearchHandler::SP &self,
+                  const search::engine::SearchRequest &req,
+                  vespalib::ThreadBundle &threadBundle) const {
+    return _matchView->match(self, req, threadBundle);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.h b/searchcore/src/vespa/searchcore/proton/server/searchview.h
new file mode 100644
index 00000000000..d5e09489231
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/searchview.h
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "matchview.h"
+#include <vespa/searchcore/proton/docsummary/isummarymanager.h>
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
+
+namespace proton {
+
+class SearchView : public boost::noncopyable,
+                   public ISearchHandler
+{
+private:
+    ISummaryManager::ISummarySetup::SP _summarySetup;
+    MatchView::SP                      _matchView;
+
+public:
+    typedef std::shared_ptr<SearchView> SP;
+
+    SearchView(const ISummaryManager::ISummarySetup::SP &summarySetup,
+               const MatchView::SP &matchView);
+
+    const ISummaryManager::ISummarySetup::SP & getSummarySetup() const
+    { return _summarySetup; }
+
+    const MatchView::SP &getMatchView() const { return _matchView; }
+
+    const Matchers::SP &getMatchers() const
+    { return _matchView->getMatchers(); }
+
+    const searchcorespi::IndexSearchable::SP &
+    getIndexSearchable() const { return _matchView->getIndexSearchable(); }
+
+    const IAttributeManager::SP &getAttributeManager() const
+    { return _matchView->getAttributeManager(); }
+
+    const matching::SessionManager::SP &
+    getSessionManager() const { return _matchView->getSessionManager(); }
+
+    const IDocumentMetaStoreContext::SP &
+    getDocumentMetaStore() const { return _matchView->getDocumentMetaStore(); }
+
+    DocIdLimit &getDocIdLimit() const { return _matchView->getDocIdLimit(); }
+
+    matching::MatchingStats
+    getMatcherStats(const vespalib::string &rankProfile) const
+    { return _matchView->getMatcherStats(rankProfile); }
+
+
+    /**
+     * Implements ISearchHandler
+     */
+    virtual search::engine::DocsumReply::UP
+    getDocsums(const search::engine::DocsumRequest & req);
+
+    virtual search::engine::SearchReply::UP match(
+            const ISearchHandler::SP &self,
+            const search::engine::SearchRequest &req,
+            vespalib::ThreadBundle &threadBundle) const;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp b/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp
new file mode 100644
index 00000000000..12f85effb2c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.simpleflush");
+
+#include "simpleflush.h"
+#include <algorithm>
+
+namespace proton {
+
+SimpleFlush::SimpleFlush()
+{
+    // empty
+}
+
+FlushContext::List
+SimpleFlush::getFlushTargets(const FlushContext::List &targetList,
+                             const flushengine::TlsStatsMap &tlsStatsMap) const
+{
+    (void) tlsStatsMap;
+    FlushContext::List fv(targetList);
+    std::sort(fv.begin(), fv.end(), CompareTarget());
+    return fv;
+}
+
+bool
+SimpleFlush::CompareTarget::compare(const IFlushTarget & lhs, const IFlushTarget & rhs) const
+{
+    IFlushTarget::MemoryGain lhsMgain(lhs.getApproxMemoryGain());
+    IFlushTarget::SerialNum lhsSerial(lhs.getFlushedSerialNum());
+
+    IFlushTarget::MemoryGain rhsMgain(rhs.getApproxMemoryGain());
+    IFlushTarget::SerialNum rhsSerial(rhs.getFlushedSerialNum());
+
+
+    bool ret = lhsSerial < rhsSerial;
+    LOG(spam, "SimpleFlush::compare("
+        "[name = '%s', before = %" PRIu64 ", after = %" PRIu64 ", serial = %" PRIu64 "], "
+        "[name = '%s', before = %" PRIu64 ", after = %" PRIu64 ", serial = %" PRIu64 "]) => %d",
+        lhs.getName().c_str(), lhsMgain.getBefore(), lhsMgain.getAfter(), lhsSerial,
+        rhs.getName().c_str(), rhsMgain.getBefore(), rhsMgain.getAfter(), rhsSerial,
+        ret ? 1 : 0);
+    return ret;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/simpleflush.h b/searchcore/src/vespa/searchcore/proton/server/simpleflush.h
new file mode 100644
index 00000000000..dcec3e1f283
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/simpleflush.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/utility.hpp>
+#include <vespa/searchcore/proton/flushengine/iflushstrategy.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton {
+
+class SimpleFlush : public boost::noncopyable,
+                    public IFlushStrategy
+{
+private:
+    class CompareTarget {
+    public:
+        bool operator () (const FlushContext::SP &lhs, const FlushContext::SP &rhs) const {
+            return compare(*lhs->getTarget(), *rhs->getTarget());
+        }
+    private:
+        bool compare(const IFlushTarget & lhs, const IFlushTarget & rhs) const;
+    };
+public:
+    SimpleFlush();
+
+    // Implements IFlushStrategy
+    virtual FlushContext::List getFlushTargets(const FlushContext::List &targetList,
+                                               const flushengine::TlsStatsMap &tlsStatsMap) const;
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
new file mode 100644
index 00000000000..db64b57698e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -0,0 +1,481 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.storeonlydocsubdb");
+
+#include "emptysearchview.h"
+#include "docstorevalidator.h"
+#include "document_subdb_initializer_result.h"
+#include "minimal_document_retriever.h"
+#include "storeonlydocsubdb.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/bucketdb/ibucketdbhandlerinitializer.h>
+#include <vespa/searchcore/proton/docsummary/summarymanagerinitializer.h>
+#include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h>
+#include <vespa/searchcore/proton/flushengine/threadedflushtarget.h>
+#include <vespa/searchcore/proton/index/index_writer.h>
+#include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h>
+#include <vespa/searchcore/proton/metrics/metricswireservice.h>
+#include <vespa/searchcorespi/index/iindexmanager.h>
+#include <vespa/searchlib/attribute/configconverter.h>
+#include <vespa/searchlib/common/lambdatask.h>
+#include <vespa/searchlib/docstore/document_store_visitor_progress.h>
+#include <vespa/searchlib/util/fileheadertk.h>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/closuretask.h>
+
+using vespa::config::search::AttributesConfig;
+using vespa::config::search::RankProfilesConfig;
+using vespa::config::search::core::ProtonConfig;
+using search::GrowStrategy;
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::IndexMetaInfo;
+using search::makeLambdaTask;
+using search::TuneFileDocumentDB;
+using search::index::Schema;
+using search::SerialNum;
+using vespalib::IllegalStateException;
+using vespalib::ThreadStackExecutorBase;
+using proton::matching::MatchingStats;
+using proton::matching::SessionManager;
+using vespalib::GenericHeader;
+using search::common::FileHeaderContext;
+using vespalib::makeTask;
+using vespalib::makeClosure;
+using proton::documentmetastore::LidReuseDelayer;
+using fastos::TimeStamp;
+using proton::initializer::InitializerTask;
+
+namespace proton
+{
+
+namespace
+{
+
+IIndexManager::SP nullIndexManager;
+IIndexWriter::SP nullIndexWriter;
+
+}
+
+StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg,
+                                     const Context &ctx)
+    : DocSubDB(ctx._owner, ctx._tlSyncer),
+      _docTypeName(cfg._docTypeName),
+      _subName(cfg._subName),
+      _baseDir(cfg._baseDir),
+      _bucketDB(ctx._bucketDB),
+      _bucketDBHandlerInitializer(ctx._bucketDBHandlerInitializer),
+      _metaStoreCtx(),
+      _attributeGrow(cfg._attributeGrow),
+      _attributeGrowNumDocs(cfg._attributeGrowNumDocs),
+      _flushedDocumentMetaStoreSerialNum(0u),
+      _flushedDocumentStoreSerialNum(0u),
+      _dms(),
+      _iSummaryMgr(),
+      _rSummaryMgr(),
+      _summaryAdapter(),
+      _writeService(ctx._writeService),
+      _summaryExecutor(ctx._summaryExecutor),
+      _metrics(ctx._metrics),
+      _iSearchView(),
+      _iFeedView(),
+      _configLock(ctx._configLock),
+      _getSerialNum(ctx._getSerialNum),
+      _tlsSyncer(ctx._writeService.master(), ctx._getSerialNum, ctx._tlSyncer),
+      _dmsFlushTarget(),
+      _subDbId(cfg._subDbId),
+      _subDbType(cfg._subDbType),
+      _fileHeaderContext(*this, ctx._fileHeaderContext, _docTypeName, _baseDir),
+      _lidReuseDelayer(),
+      _commitTimeTracker(TimeStamp::Seconds(3600.0))
+{
+    vespalib::mkdir(_baseDir, false); // Assume parent is created.
+}
+
+StoreOnlyDocSubDB::~StoreOnlyDocSubDB()
+{
+    // XXX: Disk index wrappers should not live longer than index manager
+    // which owns map of active disk indexes.
+    clearViews();
+    // Metastore must live longer than summarystore.
+    _iSummaryMgr.reset();
+    _rSummaryMgr.reset();
+}
+
+size_t
+StoreOnlyDocSubDB::getNumDocs() const
+{
+    if (_metaStoreCtx.get() != NULL) {
+        return _metaStoreCtx->get().getNumUsedLids();
+    } else {
+        return 0u;
+    }
+}
+
+size_t
+StoreOnlyDocSubDB::getNumActiveDocs() const
+{
+    return 0;
+}
+
+bool
+StoreOnlyDocSubDB::hasDocument(const document::DocumentId &id)
+{
+    search::DocumentIdT lid;
+    IDocumentMetaStoreContext::IReadGuard::UP guard = _metaStoreCtx->getReadGuard();
+    return guard->get().getLid(id.getGlobalId(), lid);
+}
+
+void
+StoreOnlyDocSubDB::onReplayDone()
+{
+    _metaStoreCtx->get().constructFreeList();
+}
+
+
+void
+StoreOnlyDocSubDB::onReprocessDone(SerialNum serialNum)
+{
+    (void) serialNum;
+    _commitTimeTracker.setReplayDone();
+}
+
+
+SerialNum
+StoreOnlyDocSubDB::getOldestFlushedSerial()
+{
+    SerialNum lowest(_iSummaryMgr->getBackingStore().lastSyncToken());
+    lowest = std::min(lowest, _dmsFlushTarget->getFlushedSerialNum());
+    return lowest;
+}
+
+
+SerialNum
+StoreOnlyDocSubDB::getNewestFlushedSerial()
+{
+    SerialNum highest(_iSummaryMgr->getBackingStore().lastSyncToken());
+    highest = std::max(highest, _dmsFlushTarget->getFlushedSerialNum());
+    return highest;
+}
+
+
+initializer::InitializerTask::SP
+StoreOnlyDocSubDB::
+createSummaryManagerInitializer(const ProtonConfig::Summary protonSummaryCfg,
+                                const search::TuneFileSummary &tuneFile,
+                                search::IBucketizer::SP bucketizer,
+                                std::shared_ptr<SummaryManager::SP> result) const
+{
+    GrowStrategy grow = _attributeGrow;
+    vespalib::string baseDir(_baseDir + "/summary");
+    return std::make_shared<SummaryManagerInitializer>
+        (grow, baseDir, getSubDbName(), _docTypeName, _summaryExecutor,
+         protonSummaryCfg,
+         tuneFile,
+         _fileHeaderContext, _tlSyncer, bucketizer, result);
+}
+
+void
+StoreOnlyDocSubDB::setupSummaryManager(SummaryManager::SP summaryManager)
+{
+    _rSummaryMgr = summaryManager;
+    _iSummaryMgr = _rSummaryMgr; // Upcast allowed with std::shared_ptr
+    _flushedDocumentStoreSerialNum =
+        _iSummaryMgr->getBackingStore().lastSyncToken();
+    _summaryAdapter.reset(new SummaryAdapter(_rSummaryMgr));
+}
+
+
+InitializerTask::SP
+StoreOnlyDocSubDB::
+createDocumentMetaStoreInitializer(const search::TuneFileAttributes &tuneFile,
+                                   std::shared_ptr<DocumentMetaStoreInitializerResult::SP> result) const
+{
+    GrowStrategy grow = _attributeGrow;
+    // Amortize memory spike cost over N docs
+    grow.setDocsGrowDelta(grow.getDocsGrowDelta() + _attributeGrowNumDocs);
+    vespalib::string baseDir(_baseDir + "/documentmetastore");
+    vespalib::string name = DocumentMetaStore::getFixedName();
+    vespalib::string attrFileName = baseDir + "/" + name; // XXX: Wrong
+    DocumentMetaStore::IGidCompare::SP
+        gidCompare(std::make_shared<DocumentMetaStore::DefaultGidCompare>());
+    // make preliminary result visible early, allowing dependent
+    // initializers to get hold of document meta store instance in
+    // their constructors.
+    *result = std::make_shared<DocumentMetaStoreInitializerResult>
+              (std::make_shared<DocumentMetaStore>(_bucketDB, attrFileName,
+                                                   grow,
+                                                   gidCompare, _subDbType),
+               tuneFile);
+    return std::make_shared<documentmetastore::DocumentMetaStoreInitializer>
+        (baseDir,
+         getSubDbName(),
+         _docTypeName.toString(),
+         (*result)->documentMetaStore());
+}
+
+
+void
+StoreOnlyDocSubDB::setupDocumentMetaStore(DocumentMetaStoreInitializerResult::SP dmsResult)
+{
+    vespalib::string baseDir(_baseDir + "/documentmetastore");
+    vespalib::string name = DocumentMetaStore::getFixedName();
+    DocumentMetaStore::SP dms(dmsResult->documentMetaStore());
+    if (dms->isLoaded()) {
+        _flushedDocumentMetaStoreSerialNum =
+            dms->getStatus().getLastSyncToken();
+    }
+    _bucketDBHandlerInitializer.
+        addDocumentMetaStore(dms.get(),
+                             _flushedDocumentMetaStoreSerialNum);
+    _metaStoreCtx.reset(new DocumentMetaStoreContext(dms));
+    LOG(debug,
+        "Added document meta store '%s'"
+        " with flushed serial num %" PRIu64,
+        name.c_str(), _flushedDocumentMetaStoreSerialNum);
+    _dms = dms;
+    _dmsFlushTarget.reset(new DocumentMetaStoreFlushTarget(dms,
+                                                           _tlsSyncer,
+                                  baseDir,
+                                  dmsResult->tuneFile(),
+                                  _fileHeaderContext));
+}
+
+DocumentSubDbInitializer::UP
+StoreOnlyDocSubDB::createInitializer(const DocumentDBConfig &configSnapshot,
+                                     SerialNum configSerialNum,
+                                     const Schema::SP &unionSchema,
+                                     const ProtonConfig::Summary &
+                                     protonSummaryCfg,
+                                     const ProtonConfig::Index &indexCfg) const
+{
+    (void) unionSchema;
+    (void) configSerialNum;
+    (void) indexCfg;
+    auto result = std::make_unique<DocumentSubDbInitializer>
+                  (const_cast<StoreOnlyDocSubDB &>(*this),
+                   _writeService.master());
+    auto dmsInitTask =
+    createDocumentMetaStoreInitializer(configSnapshot.
+                                       getTuneFileDocumentDBSP()->_attr,
+                                       result->writableResult().
+                                       writableDocumentMetaStore());
+    result->addDocumentMetaStoreInitTask(dmsInitTask);
+    auto summaryTask =
+        createSummaryManagerInitializer(protonSummaryCfg,
+                                        configSnapshot.
+                                        getTuneFileDocumentDBSP()->_summary,
+                                        result->result().documentMetaStore()->
+                                        documentMetaStore(),
+                                        result->writableResult().
+                                        writableSummaryManager());
+    result->addDependency(summaryTask);
+    summaryTask->addDependency(dmsInitTask);
+
+    LidReuseDelayerConfig lidReuseDelayerConfig(configSnapshot);
+    result->writableResult().setLidReuseDelayerConfig(lidReuseDelayerConfig);
+    return result;
+}
+
+void
+StoreOnlyDocSubDB::setup(const DocumentSubDbInitializerResult &initResult)
+{
+    setupDocumentMetaStore(initResult.documentMetaStore());
+    setupSummaryManager(initResult.summaryManager());
+    _lidReuseDelayer.reset(new LidReuseDelayer(_writeService, *_dms));
+    updateLidReuseDelayer(initResult.lidReuseDelayerConfig());
+}
+
+IFlushTarget::List
+StoreOnlyDocSubDB::getFlushTargets()
+{
+    IFlushTarget::List ret;
+    for (const auto &target : getFlushTargetsInternal()) {
+        ret.push_back(IFlushTarget::SP
+                (new ThreadedFlushTarget(_writeService.master(),
+                                         _getSerialNum,
+                                         target, _subName)));
+    }
+    return ret;
+}
+
+IFlushTarget::List
+StoreOnlyDocSubDB::getFlushTargetsInternal()
+{
+    IFlushTarget::List ret(_rSummaryMgr->getFlushTargets());
+    ret.push_back(_dmsFlushTarget);
+    return ret;
+}
+
+StoreOnlyFeedView::Context
+StoreOnlyDocSubDB::getStoreOnlyFeedViewContext(const DocumentDBConfig &configSnapshot)
+{
+    return StoreOnlyFeedView::Context(getSummaryAdapter(),
+            configSnapshot.getSchemaSP(),
+            _metaStoreCtx,
+            configSnapshot.getDocumentTypeRepoSP(),
+            _writeService,
+            *_lidReuseDelayer, _commitTimeTracker);
+}
+
+StoreOnlyFeedView::PersistentParams
+StoreOnlyDocSubDB::getFeedViewPersistentParams()
+{
+    SerialNum flushedDMSSN(_flushedDocumentMetaStoreSerialNum);
+    SerialNum flushedDSSN(_flushedDocumentStoreSerialNum);
+    return StoreOnlyFeedView::PersistentParams(flushedDMSSN,
+            flushedDSSN,
+            _docTypeName,
+            _metrics.feed,
+            _subDbId,
+            _subDbType);
+}
+
+void
+StoreOnlyDocSubDB::initViews(const DocumentDBConfig &configSnapshot,
+                             const SessionManager::SP &sessionManager)
+{
+    assert(_writeService.master().isCurrentThread());
+    _iSearchView.set(ISearchHandler::SP(new EmptySearchView));
+    {
+        vespalib::LockGuard guard(_configLock);
+        initFeedView(configSnapshot);
+    }
+    (void) sessionManager;
+}
+
+void
+StoreOnlyDocSubDB::initFeedView(const DocumentDBConfig &configSnapshot)
+{
+    assert(_writeService.master().isCurrentThread());
+    StoreOnlyFeedView::UP feedView(new StoreOnlyFeedView(
+            getStoreOnlyFeedViewContext(configSnapshot),
+            getFeedViewPersistentParams()));
+
+    // XXX: Not exception safe.
+    _iFeedView.set(StoreOnlyFeedView::SP(feedView.release()));
+}
+
+
+void
+StoreOnlyDocSubDB::updateLidReuseDelayer(const DocumentDBConfig *
+                                         newConfigSnapshot)
+{
+    LidReuseDelayerConfig lidReuseDelayerConfig(*newConfigSnapshot);
+    updateLidReuseDelayer(lidReuseDelayerConfig);
+}
+
+void
+StoreOnlyDocSubDB::updateLidReuseDelayer(const LidReuseDelayerConfig &config)
+{
+    bool immediateCommit = config.visibilityDelay() == 0;
+    /*
+     * The lid reuse delayer should not have any pending lids stored at this
+     * time, since DocumentDB::applyConfig() calls forceCommit() on the
+     * feed view before applying the new config to the sub dbs.
+     */
+    _lidReuseDelayer->setImmediateCommit(immediateCommit);
+    _commitTimeTracker.setVisibilityDelay(config.visibilityDelay());
+}
+
+
+IReprocessingTask::List
+StoreOnlyDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                               const DocumentDBConfig &oldConfigSnapshot,
+                               SerialNum serialNum,
+                               const ReconfigParams params)
+{
+    (void) oldConfigSnapshot;
+    assert(_writeService.master().isCurrentThread());
+    (void) serialNum;
+    (void) params;
+    initFeedView(newConfigSnapshot);
+    updateLidReuseDelayer(&newConfigSnapshot);
+    _owner.syncFeedView();
+    return IReprocessingTask::List();
+}
+
+proton::IAttributeManager::SP
+StoreOnlyDocSubDB::getAttributeManager() const
+{
+    return proton::IAttributeManager::SP();
+}
+
+const IIndexManager::SP &
+StoreOnlyDocSubDB::getIndexManager() const
+{
+    return nullIndexManager;
+}
+
+const IIndexWriter::SP &
+StoreOnlyDocSubDB::getIndexWriter() const
+{
+    return nullIndexWriter;
+}
+
+void
+StoreOnlyDocSubDB::wipeHistory(SerialNum, const Schema &, const Schema &)
+{
+}
+
+void
+StoreOnlyDocSubDB::setIndexSchema(const Schema::SP &schema,
+                                const Schema::SP &fusionSchema)
+{
+    assert(_writeService.master().isCurrentThread());
+    (void) schema;
+    (void) fusionSchema;
+}
+
+search::SearchableStats
+StoreOnlyDocSubDB::getSearchableStats() const
+{
+    return search::SearchableStats();
+}
+
+IDocumentRetriever::UP
+StoreOnlyDocSubDB::getDocumentRetriever()
+{
+    return IDocumentRetriever::UP(new MinimalDocumentRetriever(
+                    _docTypeName,
+                    _iFeedView.get()->getDocumentTypeRepo(),
+                    *_metaStoreCtx,
+                    _iSummaryMgr->getBackingStore(),
+                    _subDbType != SubDbType::REMOVED));
+}
+
+MatchingStats
+StoreOnlyDocSubDB::getMatcherStats(const vespalib::string &rankProfile) const
+{
+    (void) rankProfile;
+    return MatchingStats();
+}
+
+void
+StoreOnlyDocSubDB::close()
+{
+    assert(_writeService.master().isCurrentThread());
+    search::IDocumentStore & store(_rSummaryMgr->getBackingStore());
+    SerialNum syncToken = store.initFlush(store.lastSyncToken());
+    _tlSyncer.sync(syncToken);
+    store.flush(syncToken);
+}
+
+void
+StoreOnlySubDBFileHeaderContext::
+addTags(vespalib::GenericHeader &header,
+        const vespalib::string &name) const
+{
+    _parentFileHeaderContext.addTags(header, name);
+    typedef GenericHeader::Tag Tag;
+    header.putTag(Tag("documentType", _docTypeName.toString()));
+    header.putTag(Tag("subDB", _subDB));
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
new file mode 100644
index 00000000000..bee288b7cc0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -0,0 +1,385 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "documentdbconfig.h"
+#include "idocumentsubdb.h"
+#include "ifeedview.h"
+#include "summaryadapter.h"
+#include "tlssyncer.h"
+#include <memory>
+#include <vector>
+#include <vespa/searchcore/config/config-proton.h>
+#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/common/subdbtype.h>
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h>
+#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
+#include <vespa/searchcore/proton/matchengine/imatchhandler.h>
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
+#include <vespa/searchcore/proton/common/commit_time_tracker.h>
+#include <vespa/searchlib/common/fileheadercontext.h>
+#include <vespa/vespalib/util/varholder.h>
+
+
+namespace proton
+{
+
+class MetricsWireService;
+class LegacyDocumentDBMetrics;
+class FeedHandler;
+
+namespace bucketdb
+{
+
+class IBucketDBHandlerInitializer;
+
+}
+
+namespace documentmetastore { class LidReuseDelayerConfig; }
+
+/**
+ * Base class for a document sub database.
+ */
+class DocSubDB : public IDocumentSubDB
+{
+protected:
+    IOwner			  &_owner;
+    search::transactionlog::SyncProxy &_tlSyncer;
+
+public:
+    DocSubDB(IOwner &owner, search::transactionlog::SyncProxy &tlSyncer)
+        : IDocumentSubDB(),
+          _owner(owner),
+          _tlSyncer(tlSyncer)
+    { }
+
+    virtual ~DocSubDB() { }
+    void close() override { }
+};
+
+
+class StoreOnlyDocSubDB;
+
+/**
+ * File header context used by the store-only sub database.
+ *
+ * This header context contains information that will be part of the header of all data files
+ * written by a store-only sub database.
+ */
+class StoreOnlySubDBFileHeaderContext : public search::common::FileHeaderContext
+{
+    StoreOnlyDocSubDB &_owner;
+    const search::common::FileHeaderContext &_parentFileHeaderContext;
+    const DocTypeName &_docTypeName;
+    vespalib::string _subDB;
+
+public:
+    StoreOnlySubDBFileHeaderContext(StoreOnlyDocSubDB &owner,
+                                     const search::common::FileHeaderContext &
+                                     parentFileHeaderContext,
+                                     const DocTypeName &docTypeName,
+                                     const vespalib::string &baseDir)
+        : search::common::FileHeaderContext(),
+          _owner(owner),
+          _parentFileHeaderContext(parentFileHeaderContext),
+          _docTypeName(docTypeName),
+          _subDB()
+    {
+        size_t pos = baseDir.rfind('/');
+        if (pos != vespalib::string::npos)
+            _subDB = baseDir.substr(pos + 1);
+        else
+            _subDB = baseDir;
+    }
+
+    virtual void
+    addTags(vespalib::GenericHeader &header,
+            const vespalib::string &name) const;
+};
+
+/**
+ * The store-only sub database handles only storing and retrieving of documents.
+ *
+ * lid<->gid mapping is handled via DocumentMetaStore and storing of documents via DocumentStore.
+ * This class is used as base class for other sub databases and directly by the "1.removed"
+ * sub database for for storing removed documents.
+ */
+class StoreOnlyDocSubDB : public DocSubDB
+{
+public:
+    struct Config {
+        const DocTypeName _docTypeName;
+        const vespalib::string _subName;
+        const vespalib::string _baseDir;
+        const search::GrowStrategy _attributeGrow;
+        const size_t _attributeGrowNumDocs;
+        const uint32_t _subDbId;
+        const SubDbType _subDbType;
+
+        Config(const DocTypeName &docTypeName,
+               const vespalib::string &subName,
+               const vespalib::string &baseDir,
+               const search::GrowStrategy &attributeGrow,
+               size_t attributeGrowNumDocs,
+               uint32_t subDbId,
+               SubDbType subDbType)
+            : _docTypeName(docTypeName),
+              _subName(subName),
+              _baseDir(baseDir + "/" + subName),
+              _attributeGrow(attributeGrow),
+              _attributeGrowNumDocs(attributeGrowNumDocs),
+              _subDbId(subDbId),
+              _subDbType(subDbType)
+        {
+        }
+    };
+
+    struct Context {
+        IDocumentSubDB::IOwner &_owner;
+        search::transactionlog::SyncProxy &_tlSyncer;
+        const IGetSerialNum &_getSerialNum;
+        const search::common::FileHeaderContext &_fileHeaderContext;
+        searchcorespi::index::IThreadingService &_writeService;
+        vespalib::ThreadStackExecutorBase &_summaryExecutor;
+        std::shared_ptr<BucketDBOwner> _bucketDB;
+        bucketdb::IBucketDBHandlerInitializer &_bucketDBHandlerInitializer;
+        LegacyDocumentDBMetrics &_metrics;
+        vespalib::Lock &_configLock;
+
+        Context(IDocumentSubDB::IOwner &owner,
+                search::transactionlog::SyncProxy &tlSyncer,
+                const IGetSerialNum &getSerialNum,
+                const search::common::FileHeaderContext &fileHeaderContext,
+                searchcorespi::index::IThreadingService &writeService,
+                vespalib::ThreadStackExecutorBase &summaryExecutor,
+                std::shared_ptr<BucketDBOwner> bucketDB,
+                bucketdb::IBucketDBHandlerInitializer &
+                bucketDBHandlerInitializer,
+                LegacyDocumentDBMetrics &metrics,
+                vespalib::Lock &configLock)
+            : _owner(owner),
+              _tlSyncer(tlSyncer),
+              _getSerialNum(getSerialNum),
+              _fileHeaderContext(fileHeaderContext),
+              _writeService(writeService),
+              _summaryExecutor(summaryExecutor),
+              _bucketDB(bucketDB),
+              _bucketDBHandlerInitializer(bucketDBHandlerInitializer),
+              _metrics(metrics),
+              _configLock(configLock)
+        {
+        }
+    };
+
+
+protected:
+    const DocTypeName             _docTypeName;
+    const vespalib::string        _subName;
+    const vespalib::string        _baseDir;
+    BucketDBOwner::SP             _bucketDB;
+    bucketdb::IBucketDBHandlerInitializer &_bucketDBHandlerInitializer;
+    IDocumentMetaStoreContext::SP _metaStoreCtx;
+    const search::GrowStrategy    _attributeGrow;
+    const size_t                  _attributeGrowNumDocs;
+    // The following two serial numbers reflect state at program startup
+    // and are used by replay logic.
+    SerialNum                     _flushedDocumentMetaStoreSerialNum;
+    SerialNum                     _flushedDocumentStoreSerialNum;
+    DocumentMetaStore::SP         _dms;
+    ISummaryManager::SP           _iSummaryMgr;	// Interface class
+private:
+    SummaryManager::SP            _rSummaryMgr; // Our specific subclass
+    ISummaryAdapter::SP           _summaryAdapter;
+protected:
+    searchcorespi::index::IThreadingService &_writeService;
+    vespalib::ThreadStackExecutorBase       &_summaryExecutor;
+    LegacyDocumentDBMetrics                 &_metrics;
+    vespalib::VarHolder<ISearchHandler::SP> _iSearchView;
+    vespalib::VarHolder<IFeedView::SP>      _iFeedView;
+    vespalib::Lock                         &_configLock;
+private:
+    const IGetSerialNum             &_getSerialNum;
+    TlsSyncer                        _tlsSyncer;
+    DocumentMetaStoreFlushTarget::SP _dmsFlushTarget;
+
+    virtual IFlushTarget::List
+    getFlushTargets();
+protected:
+    const uint32_t		  _subDbId;
+    const SubDbType               _subDbType;
+    StoreOnlySubDBFileHeaderContext _fileHeaderContext;
+    std::unique_ptr<documentmetastore::ILidReuseDelayer> _lidReuseDelayer;
+    CommitTimeTracker               _commitTimeTracker;
+
+    initializer::InitializerTask::SP
+    createSummaryManagerInitializer(const vespa::config::search::core::
+                                    ProtonConfig::Summary protonSummaryCfg,
+                                    const search::TuneFileSummary &tuneFile,
+                                    search::IBucketizer::SP bucketizer,
+                                    std::shared_ptr<SummaryManager::SP> result)
+        const;
+
+    void
+    setupSummaryManager(SummaryManager::SP summaryManager);
+
+    initializer::InitializerTask::SP
+    createDocumentMetaStoreInitializer(const search::TuneFileAttributes &tuneFile,
+                                       std::shared_ptr<DocumentMetaStoreInitializerResult::SP> result) const;
+
+    void
+    setupDocumentMetaStore(DocumentMetaStoreInitializerResult::SP dmsResult);
+
+    void
+    initFeedView(const DocumentDBConfig &configSnapshot);
+
+    virtual IFlushTarget::List
+    getFlushTargetsInternal();
+
+    StoreOnlyFeedView::Context getStoreOnlyFeedViewContext(const DocumentDBConfig &configSnapshot);
+
+    StoreOnlyFeedView::PersistentParams getFeedViewPersistentParams();
+
+    vespalib::string getSubDbName() const {
+        return vespalib::make_string("%s.%s",
+                _owner.getName().c_str(), _subName.c_str());
+    }
+
+    void
+    updateLidReuseDelayer(const DocumentDBConfig *newConfigSnapshot);
+
+    using LidReuseDelayerConfig = documentmetastore::LidReuseDelayerConfig;
+
+    virtual void
+    updateLidReuseDelayer(const LidReuseDelayerConfig &config);
+
+public:
+    StoreOnlyDocSubDB(const Config &cfg,
+                    const Context &ctx);
+
+    virtual
+    ~StoreOnlyDocSubDB();
+
+    virtual uint32_t getSubDbId() const { return _subDbId; }
+
+    virtual vespalib::string getName() const { return _subName; }
+
+    virtual DocumentSubDbInitializer::UP
+    createInitializer(const DocumentDBConfig &configSnapshot,
+                      SerialNum configSerialNum,
+                      const search::index::Schema::SP &unionSchema,
+                      const vespa::config::search::core::
+                      ProtonConfig::Summary &protonSummaryCfg,
+                      const vespa::config::search::core::
+                      ProtonConfig::Index &indexCfg) const override;
+
+    virtual void setup(const DocumentSubDbInitializerResult &initResult)
+        override;
+
+    virtual void
+    initViews(const DocumentDBConfig &configSnapshot,
+              const matching::SessionManager::SP &sessionManager);
+
+    virtual IReprocessingTask::List
+    applyConfig(const DocumentDBConfig &newConfigSnapshot,
+                const DocumentDBConfig &oldConfigSnapshot,
+                SerialNum serialNum,
+                const ReconfigParams params);
+
+    virtual ISearchHandler::SP
+    getSearchView() const
+    {
+        return _iSearchView.get();
+    }
+
+    virtual IFeedView::SP
+    getFeedView() const
+    {
+        return _iFeedView.get();
+    }
+
+    virtual void
+    clearViews()
+    {
+        _iFeedView.clear();
+        _iSearchView.clear();
+    }
+
+    /**
+     * Returns the summary manager that this database uses to manage
+     * document summaries of the corresponding document type.
+     *
+     * @return The summary manager.
+     */
+    virtual const ISummaryManager::SP &
+    getSummaryManager() const
+    {
+        return _iSummaryMgr;
+    }
+
+    virtual proton::IAttributeManager::SP
+    getAttributeManager() const;
+
+    virtual const IIndexManager::SP &
+    getIndexManager() const;
+
+    virtual const ISummaryAdapter::SP &
+    getSummaryAdapter() const
+    {
+        return _summaryAdapter;
+    }
+
+    virtual const IIndexWriter::SP &
+    getIndexWriter() const;
+
+    virtual IDocumentMetaStoreContext &
+    getDocumentMetaStoreContext()
+    {
+        return *_metaStoreCtx;
+    }
+
+    virtual size_t
+    getNumDocs() const;
+
+    virtual size_t
+    getNumActiveDocs() const override;
+
+    virtual bool
+    hasDocument(const document::DocumentId &id);
+
+    virtual void
+    onReplayDone();
+
+    virtual void
+    onReprocessDone(SerialNum serialNum);
+
+    virtual SerialNum
+    getOldestFlushedSerial();
+
+    virtual SerialNum
+    getNewestFlushedSerial();
+
+    virtual void
+    wipeHistory(SerialNum wipeSerial,
+                const search::index::Schema &newHistorySchema,
+                const search::index::Schema &wipeSchema);
+
+    virtual void
+    setIndexSchema(const search::index::Schema::SP &schema,
+                   const search::index::Schema::SP &fusionSchema);
+
+    virtual search::SearchableStats
+    getSearchableStats() const;
+
+    virtual IDocumentRetriever::UP
+    getDocumentRetriever();
+
+    virtual matching::MatchingStats
+    getMatcherStats(const vespalib::string &rankProfile) const;
+
+    void close() override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
new file mode 100644
index 00000000000..f649f43a636
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -0,0 +1,873 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.storeonlyfeedview");
+#include "ireplayconfig.h"
+#include "storeonlyfeedview.h"
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/documentapi/messagebus/messages/documentreply.h>
+#include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
+#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
+#include <vespa/searchcore/proton/common/bucketfactory.h>
+#include <vespa/searchcore/proton/common/commit_time_tracker.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
+#include "forcecommitcontext.h"
+#include "operationdonecontext.h"
+#include "removedonecontext.h"
+#include "updatedonecontext.h"
+#include "putdonecontext.h"
+#include <vespa/searchlib/common/lambdatask.h>
+#include <vespa/searchlib/common/scheduletaskcallback.h>
+
+using document::BucketId;
+using document::DocumentTypeRepo;
+using document::Document;
+using document::DocumentId;
+using document::DocumentUpdate;
+using documentapi::DocumentProtocol;
+using documentapi::DocumentReply;
+using documentapi::RemoveDocumentReply;
+using documentapi::UpdateDocumentReply;
+using search::index::Schema;
+using storage::spi::BucketInfoResult;
+using storage::spi::Timestamp;
+using vespalib::IllegalStateException;
+using vespalib::makeClosure;
+using vespalib::makeTask;
+using vespalib::make_string;
+using search::makeLambdaTask;
+
+namespace proton {
+
+namespace {
+
+bool shouldTrace(StoreOnlyFeedView::OnOperationDoneType onWriteDone,
+                 uint32_t traceLevel)
+{
+    return onWriteDone && onWriteDone->shouldTrace(traceLevel);
+}
+
+FeedToken::UP dupFeedToken(FeedToken *token)
+{
+    // If token is not NULL then a new feed token is created, referencing
+    // same shared state as old token.
+    if (token != NULL) {
+        return FeedToken::UP(new FeedToken(*token));
+    } else {
+        return FeedToken::UP();
+    }
+}
+
+std::shared_ptr<PutDoneContext>
+createPutDoneContext(FeedToken::UP &token,
+                     FeedOperation::Type opType,
+                     PerDocTypeFeedMetrics &metrics,
+                     bool force)
+{
+    std::shared_ptr<PutDoneContext> result;
+    if (token || force) {
+        result = std::make_shared<PutDoneContext>(std::move(token), opType, metrics);
+    }
+    return result;
+}
+
+std::shared_ptr<UpdateDoneContext>
+createUpdateDoneContext(FeedToken::UP &token,
+                        FeedOperation::Type opType,
+                        PerDocTypeFeedMetrics &metrics,
+                        const document::DocumentUpdate::SP &upd)
+{
+    return std::make_shared<UpdateDoneContext>(std::move(token), opType, metrics, upd);
+}
+
+}  // namespace
+
+
+StoreOnlyFeedView::StoreOnlyFeedView(const Context &ctx,
+                                     const PersistentParams &params)
+    : IFeedView(),
+      FeedDebugger(),
+      _summaryAdapter(ctx._summaryAdapter),
+      _schema(ctx._schema),
+      _documentMetaStoreContext(ctx._documentMetaStoreContext),
+      _repo(ctx._repo),
+      _writeService(ctx._writeService),
+      _params(params),
+      _metaStore(_documentMetaStoreContext->get()),
+      _docType(NULL),
+      _lidReuseDelayer(ctx._lidReuseDelayer),
+      _commitTimeTracker(ctx._commitTimeTracker)
+{
+    _docType = _repo->getDocumentType(_params._docTypeName.getName());
+}
+
+
+void
+StoreOnlyFeedView::sync()
+{
+}
+
+void
+StoreOnlyFeedView::forceCommit(SerialNum serialNum)
+{
+    forceCommit(serialNum,
+                std::make_shared<ForceCommitContext>(_writeService.master(),
+                                                     _metaStore));
+}
+
+
+void
+StoreOnlyFeedView::forceCommit(SerialNum serialNum,
+                               OnForceCommitDoneType onCommitDone)
+{
+    (void) serialNum;
+    std::vector<uint32_t> lidsToReuse;
+    lidsToReuse = _lidReuseDelayer.getReuseLids();
+    if (!lidsToReuse.empty()) {
+        onCommitDone->reuseLids(std::move(lidsToReuse));
+    }
+}
+
+
+void
+StoreOnlyFeedView::considerEarlyAck(FeedToken::UP &token,
+                                    FeedOperation::Type opType)
+{
+    if (_commitTimeTracker.hasVisibilityDelay() && token) {
+        token->ack(opType, _params._metrics);
+        token.reset();
+    }
+}
+
+
+void
+StoreOnlyFeedView::putAttributes(SerialNum serialNum,
+                                 search::DocumentIdT lid,
+                                 const Document &doc,
+                                 bool immediateCommit,
+                                 OnPutDoneType onWriteDone)
+{
+    (void) serialNum;
+    (void) lid;
+    (void) doc;
+    (void) immediateCommit;
+    (void) onWriteDone;
+}
+
+void
+StoreOnlyFeedView::putIndexedFields(SerialNum serialNum,
+                                    search::DocumentIdT lid,
+                                    const Document::SP &newDoc,
+                                    bool immediateCommit,
+                                    OnOperationDoneType onWriteDone)
+{
+    (void) serialNum;
+    (void) lid;
+    (void) newDoc;
+    (void) immediateCommit;
+    (void) onWriteDone;
+}
+
+
+namespace {
+void setPrev(DocumentOperation &op,
+             const documentmetastore::IStore::Result &result,
+             uint32_t subDbId, bool markedAsRemoved) {
+    if (result._found) {
+        op.setPrevDbDocumentId(DbDocumentId(subDbId, result._lid));
+        op.setPrevMarkedAsRemoved(markedAsRemoved);
+        op.setPrevTimestamp(result._timestamp);
+    }
+}
+}  // namespace
+
+void
+StoreOnlyFeedView::preparePut(PutOperation &putOp)
+{
+    const DocumentId &docId = putOp.getDocument()->getId();
+    const document::GlobalId &gid = docId.getGlobalId();
+    documentmetastore::IStore::Result inspectResult = _metaStore.inspect(gid);
+    putOp.setDbDocumentId(DbDocumentId(_params._subDbId, inspectResult._lid));
+    assert(_params._subDbType != SubDbType::REMOVED);
+    setPrev(putOp, inspectResult, _params._subDbId, false);
+}
+
+
+void
+StoreOnlyFeedView::handlePut(FeedToken *token,
+                             const PutOperation &putOp)
+{
+    internalPut(dupFeedToken(token), putOp);
+}
+
+
+void
+StoreOnlyFeedView::internalPut(FeedToken::UP token,
+                               const PutOperation &putOp)
+{
+    assert(putOp.getValidDbdId());
+    assert(putOp.notMovingLidInSameSubDb());
+
+    const SerialNum serialNum = putOp.getSerialNum();
+    const Document::SP &doc = putOp.getDocument();
+    const DocumentId &docId = doc->getId();
+    VLOG(getDebugLevel(putOp.getNewOrPrevLid(_params._subDbId), doc->getId()),
+         "database(%s): internalPut: serialNum(%" PRIu64 "), "
+         "docId(%s), lid(%u,%u) prevLid(%u,%u)"
+         " subDbId %u document(%ld) = {\n%s\n}",
+         _params._docTypeName.toString().c_str(),
+         serialNum,
+         doc->getId().toString().c_str(),
+         putOp.getSubDbId(),
+         putOp.getLid(),
+         putOp.getPrevSubDbId(),
+         putOp.getPrevLid(),
+         _params._subDbId,
+         doc->toString(true).size(),
+         doc->toString(true).c_str());
+
+    uint32_t oldDocIdLimit = _metaStore.getCommittedDocIdLimit();
+    adjustMetaStore(putOp, docId);
+    considerEarlyAck(token, putOp.getType());
+
+    bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId);
+    if (putOp.getValidDbdId(_params._subDbId)) {
+        _summaryAdapter->put(serialNum, *doc, putOp.getLid());
+        bool immediateCommit = _commitTimeTracker.needCommit();
+        std::shared_ptr<PutDoneContext> onWriteDone =
+            createPutDoneContext(token, putOp.getType(), _params._metrics,
+                                 immediateCommit &&
+                                 putOp.getLid() >= oldDocIdLimit);
+        putAttributes(serialNum, putOp.getLid(), *doc, immediateCommit, onWriteDone);
+        putIndexedFields(serialNum, putOp.getLid(), doc, immediateCommit, onWriteDone);
+    }
+    if (docAlreadyExists && putOp.changedDbdId()) {
+        assert(!putOp.getValidDbdId(_params._subDbId));
+        internalRemove(std::move(token), serialNum, putOp.getPrevLid(), putOp.getType());
+    }
+    if (token.get() != NULL) {
+        token->ack(putOp.getType(), _params._metrics);
+    }
+}
+
+
+void
+StoreOnlyFeedView::heartBeatIndexedFields(SerialNum serialNum)
+{
+    (void) serialNum;
+}
+
+
+void
+StoreOnlyFeedView::heartBeatAttributes(SerialNum serialNum)
+{
+    (void) serialNum;
+}
+
+
+StoreOnlyFeedView::UpdateScope
+StoreOnlyFeedView::getUpdateScope(const DocumentUpdate &upd)
+{
+    UpdateScope updateScope;
+    size_t m(upd.getUpdates().size());
+    if (m != 0) {
+        updateScope._nonAttributeFields = true;
+    }
+    return updateScope;
+}
+
+
+void
+StoreOnlyFeedView::updateAttributes(SerialNum serialNum,
+                                    search::DocumentIdT lid,
+                                    const document::DocumentUpdate &upd,
+                                    bool immediateCommit,
+                                    OnOperationDoneType onWriteDone)
+{
+    (void) serialNum;
+    (void) lid;
+    (void) upd;
+    (void) immediateCommit;
+    (void) onWriteDone;
+}
+
+
+void
+StoreOnlyFeedView::updateIndexedFields(SerialNum serialNum,
+                                       search::DocumentIdT lid,
+                                       const Document::SP &newDoc,
+                                       bool immediateCommit,
+                                       OnOperationDoneType onWriteDone)
+{
+    (void) serialNum;
+    (void) lid;
+    (void) newDoc;
+    (void) immediateCommit;
+    (void) onWriteDone;
+    abort(); // Should never be called.
+}
+
+void
+StoreOnlyFeedView::prepareUpdate(UpdateOperation &updOp)
+{
+    const DocumentId &docId = updOp.getUpdate()->getId();
+    const document::GlobalId &gid = docId.getGlobalId();
+    documentmetastore::IStore::Result inspectResult = _metaStore.inspect(gid);
+    updOp.setDbDocumentId(DbDocumentId(_params._subDbId, inspectResult._lid));
+    assert(_params._subDbType != SubDbType::REMOVED);
+    setPrev(updOp, inspectResult, _params._subDbId, false);
+}
+
+
+void
+StoreOnlyFeedView::handleUpdate(FeedToken *token,
+                                const UpdateOperation &updOp)
+{
+    internalUpdate(dupFeedToken(token), updOp);
+}
+
+
+void
+StoreOnlyFeedView::internalUpdate(FeedToken::UP token,
+                                  const UpdateOperation &updOp)
+{
+    if (updOp.getUpdate().get() == NULL) {
+        LOG(warning, "database(%s): ignoring invalid update operation",
+            _params._docTypeName.toString().c_str());
+        return;
+    }
+
+    const SerialNum serialNum = updOp.getSerialNum();
+    const DocumentUpdate &upd = *updOp.getUpdate();
+    const DocumentId &docId = upd.getId();
+    const search::DocumentIdT lid = updOp.getLid();
+    VLOG(getDebugLevel(lid, upd.getId()),
+        "database(%s): internalUpdate: serialNum(%" PRIu64
+        "), docId(%s), lid(%d)",
+        _params._docTypeName.toString().c_str(),
+        serialNum,
+        upd.getId().toString().c_str(),
+        lid);
+
+    if (useDocumentMetaStore(serialNum)) {
+        search::DocumentIdT storedLid;
+        bool lookupOk = lookupDocId(docId, storedLid);
+        assert(lookupOk);
+        (void) lookupOk;
+        assert(storedLid == updOp.getLid());
+        bool updateOk = _metaStore.updateMetaData(updOp.getLid(),
+                                                  updOp.getBucketId(),
+                                                  updOp.getTimestamp());
+        assert(updateOk);
+        (void) updateOk;
+        _metaStore.commit(serialNum, serialNum);
+    }
+    considerEarlyAck(token, updOp.getType());
+
+    bool immediateCommit = _commitTimeTracker.needCommit();
+    auto onWriteDone = createUpdateDoneContext(token, updOp.getType(),
+            _params._metrics, updOp.getUpdate());
+    updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone);
+    UpdateScope updateScope(getUpdateScope(upd));
+    /*
+     * XXX: Flushing issue
+     *
+     * If subclass has indexed fields, but no partial updates updates
+     * affect indexed fields then serial number is not updated, for very
+     * specific feed patterns, this can cause flush engine issues.
+     *
+     * Similarly, if subclass has attributes that are not updated.
+     *
+     */
+    if (updateScope._indexedFields || updateScope._nonAttributeFields) {
+        updateIndexAndDocumentStore(updateScope._indexedFields,
+                                    serialNum, lid, upd,
+                                    immediateCommit,
+                                    onWriteDone);
+    }
+    if (!updateScope._indexedFields && onWriteDone) {
+        if (onWriteDone->shouldTrace(1))
+        {
+            token->trace(1, "Partial update applied.");
+        }
+    }
+}
+
+
+void
+StoreOnlyFeedView::updateIndexAndDocumentStore(bool indexedFieldsInScope,
+                                               SerialNum serialNum,
+                                               search::DocumentIdT lid,
+                                               const DocumentUpdate &upd,
+                                               bool immediateCommit,
+                                               OnOperationDoneType onWriteDone)
+{
+    Document::UP prevDoc(_summaryAdapter->get(lid, *_repo));
+    assert(onWriteDone->getToken() == NULL || useDocumentStore(serialNum));
+    if (useDocumentStore(serialNum)) {
+        assert(prevDoc.get() != NULL);
+    }
+    if (prevDoc.get() == NULL) {
+        // Replaying, document removed later before summary was flushed.
+        assert(onWriteDone->getToken() == NULL);
+        // If we've passed serial number for flushed index then we could
+        // also check that this operation is marked for ignore by index
+        // proxy.
+        return;
+    }
+    if (upd.getId() == prevDoc->getId()) {
+        if (shouldTrace(onWriteDone, 1))
+        {
+            FeedToken *token = onWriteDone->getToken();
+            token->trace(1, "The update looks like : " +
+                         upd.toString(token->shouldTrace(2)));
+        }
+        vespalib::nbostream os;
+        prevDoc->serialize(os);
+        Document::SP newDoc(new Document(*_repo, os));
+        if (useDocumentStore(serialNum)) {
+            LOG(spam, "Original document :\n%s", newDoc->toXml("  ").c_str());
+            LOG(spam, "Update\n%s", upd.toXml().c_str());
+            upd.applyTo(*newDoc);
+            LOG(spam, "Updated document :\n%s", newDoc->toXml("  ").c_str());
+            if (shouldTrace(onWriteDone, 1)) {
+                onWriteDone->getToken()->trace(1, "Then we update summary.");
+            }
+            _summaryAdapter->put(serialNum, *newDoc, lid);
+        }
+        if (indexedFieldsInScope) {
+            updateIndexedFields(serialNum, lid, newDoc, immediateCommit, onWriteDone);
+        }
+    } else {
+        // Replaying, document removed and lid reused before summary
+        // was flushed.
+        assert(onWriteDone->getToken() == NULL && !useDocumentStore(serialNum));
+    }
+}
+
+
+bool
+StoreOnlyFeedView::lookupDocId(const DocumentId &docId,
+                               search::DocumentIdT &lid) const
+{
+    // This function should only be called by the updater thread.
+    // Readers need to take a guard on the document meta store
+    // attribute before accessing.
+    if (!_metaStore.getLid(docId.getGlobalId(), lid)) {
+        return false;
+    }
+    if (_params._subDbType == SubDbType::REMOVED)
+        return false;
+    return true;
+}
+
+
+void
+StoreOnlyFeedView::removeAttributes(SerialNum serialNum,
+                                    search::DocumentIdT lid,
+                                    bool immediateCommit,
+                                    OnRemoveDoneType onWriteDone)
+{
+    (void) serialNum;
+    (void) lid;
+    (void) immediateCommit;
+    (void) onWriteDone;
+}
+
+
+void
+StoreOnlyFeedView::removeIndexedFields(SerialNum serialNum,
+                                       search::DocumentIdT lid,
+                                       bool immediateCommit,
+                                       OnRemoveDoneType onWriteDone)
+{
+    (void) serialNum;
+    (void) lid;
+    (void) immediateCommit;
+    (void) onWriteDone;
+}
+
+
+void
+StoreOnlyFeedView::prepareRemove(RemoveOperation &rmOp)
+{
+    const DocumentId &id = rmOp.getDocumentId();
+    const document::GlobalId &gid = id.getGlobalId();
+    documentmetastore::IStore::Result inspectRes = _metaStore.inspect(gid);
+    if (_params._subDbType == SubDbType::REMOVED)
+        rmOp.setDbDocumentId(DbDocumentId(_params._subDbId, inspectRes._lid));
+    setPrev(rmOp, inspectRes, _params._subDbId,
+            _params._subDbType == SubDbType::REMOVED);
+}
+
+
+void
+StoreOnlyFeedView::handleRemove(FeedToken *token,
+                                const RemoveOperation &rmOp)
+{
+    internalRemove(dupFeedToken(token), rmOp);
+}
+
+
+void
+StoreOnlyFeedView::internalRemove(FeedToken::UP token,
+                                  const RemoveOperation &rmOp)
+{
+    assert(rmOp.getValidNewOrPrevDbdId());
+    assert(rmOp.notMovingLidInSameSubDb());
+
+    const SerialNum serialNum = rmOp.getSerialNum();
+    const DocumentId &docId = rmOp.getDocumentId();
+    VLOG(getDebugLevel(rmOp.getNewOrPrevLid(_params._subDbId), docId),
+         "database(%s): internalRemove: serialNum(%" PRIu64 "), "
+         "docId(%s), lid(%u,%u) prevlid(%u,%u), subDbId %u",
+         _params._docTypeName.toString().c_str(),
+         serialNum,
+         docId.toString().c_str(),
+         rmOp.getSubDbId(),
+         rmOp.getLid(),
+         rmOp.getPrevSubDbId(),
+         rmOp.getPrevLid(),
+         _params._subDbId);
+
+    adjustMetaStore(rmOp, docId);
+    considerEarlyAck(token, rmOp.getType());
+    if (rmOp.getValidDbdId(_params._subDbId)) {
+        Document::UP clearDoc(new Document(*_docType, docId));
+        clearDoc->setRepo(*_repo);
+        _summaryAdapter->put(serialNum, *clearDoc, rmOp.getLid());
+    }
+    if (rmOp.getValidPrevDbdId(_params._subDbId)) {
+        if (rmOp.changedDbdId()) {
+            assert(!rmOp.getValidDbdId(_params._subDbId));
+            internalRemove(std::move(token), serialNum, rmOp.getPrevLid(), rmOp.getType());
+        }
+    }
+    if (token.get() != NULL) {
+        token->ack(rmOp.getType(), _params._metrics);
+    }
+}
+
+void
+StoreOnlyFeedView::internalRemove(FeedToken::UP token,
+                                  SerialNum serialNum,
+                                  search::DocumentIdT lid,
+                                  FeedOperation::Type opType)
+{
+    _summaryAdapter->remove(serialNum, lid);
+    bool explicitReuseLid = _lidReuseDelayer.delayReuse(lid);
+    std::shared_ptr<RemoveDoneContext> onWriteDone;
+    if (explicitReuseLid || token) {
+        onWriteDone = std::make_shared<RemoveDoneContext>
+                      (std::move(token), opType, _params._metrics,
+                       _writeService.master(), _metaStore,
+                       explicitReuseLid ? lid : 0u);
+    }
+    bool immediateCommit = _commitTimeTracker.needCommit();
+    removeAttributes(serialNum, lid, immediateCommit, onWriteDone);
+    removeIndexedFields(serialNum, lid, immediateCommit, onWriteDone);
+}
+
+namespace {
+void putMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id,
+                 const DocumentOperation &op, bool is_removed_doc) {
+    documentmetastore::IStore::Result putRes(
+            meta_store.put(doc_id.getGlobalId(),
+                           op.getBucketId(), op.getTimestamp(), op.getLid()));
+    if (!putRes.ok()) {
+        throw IllegalStateException(
+                make_string("Could not put <lid, gid> pair for "
+                            "%sdocument with id '%s' and gid '%s'",
+                            is_removed_doc ? "removed " : "",
+                            doc_id.toString().c_str(),
+                            doc_id.getGlobalId().toString().c_str()));
+    }
+    assert(op.getLid() == putRes._lid);
+}
+
+void removeMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id,
+                    const DocumentOperation &op, bool is_removed_doc) {
+    assert(meta_store.validLid(op.getPrevLid()));
+    assert(is_removed_doc == op.getPrevMarkedAsRemoved());
+    const RawDocumentMetaData &meta(meta_store.getRawMetaData(op.getPrevLid()));
+    assert(meta.getGid() == doc_id.getGlobalId());
+    (void) meta;
+    if (!meta_store.remove(op.getPrevLid())) {
+        throw IllegalStateException(
+                make_string("Could not remove <lid, gid> pair for "
+                            "%sdocument with id '%s' and gid '%s'",
+                            is_removed_doc ? "removed " : "",
+                            doc_id.toString().c_str(),
+                            doc_id.getGlobalId().toString().c_str()));
+    }
+}
+
+void
+moveMetaData(documentmetastore::IStore &meta_store,
+             const DocumentId &doc_id,
+             const DocumentOperation &op)
+{
+    assert(op.getLid() != op.getPrevLid());
+    assert(meta_store.validLid(op.getPrevLid()));
+    assert(!meta_store.validLid(op.getLid()));
+    const RawDocumentMetaData &meta(meta_store.getRawMetaData(op.getPrevLid()));
+    (void) meta;
+    assert(meta.getGid() == doc_id.getGlobalId());
+    assert(meta.getTimestamp() == op.getTimestamp());
+    meta_store.move(op.getPrevLid(), op.getLid());
+}
+
+
+}  // namespace
+
+void
+StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op,
+                                   const DocumentId &docId)
+{
+    const SerialNum serialNum = op.getSerialNum();
+    if (useDocumentMetaStore(serialNum)) {
+        if (op.getValidDbdId(_params._subDbId)) {
+            if (op.getType() == FeedOperation::MOVE &&
+                op.getValidPrevDbdId(_params._subDbId) &&
+                op.getLid() != op.getPrevLid()) {
+                moveMetaData(_metaStore, docId, op);
+            } else {
+                putMetaData(_metaStore, docId, op,
+                            _params._subDbType == SubDbType::REMOVED);
+            }
+        } else if (op.getValidPrevDbdId(_params._subDbId)) {
+            removeMetaData(_metaStore, docId, op,
+                           _params._subDbType == SubDbType::REMOVED);
+        }
+        _metaStore.commit(serialNum, serialNum);
+    }
+}
+
+
+void
+StoreOnlyFeedView::removeAttributes(SerialNum serialNum,
+                                    const LidVector &lidsToRemove,
+                                    bool immediateCommit,
+                                    OnWriteDoneType onWriteDone)
+{
+    (void) serialNum;
+    (void) lidsToRemove;
+    (void) immediateCommit;
+    (void) onWriteDone;
+}
+
+
+void
+StoreOnlyFeedView::removeIndexedFields(SerialNum serialNum,
+                                       const LidVector &lidsToRemove,
+                                       bool immediateCommit,
+                                       OnWriteDoneType onWriteDone)
+{
+    (void) serialNum;
+    (void) lidsToRemove;
+    (void) onWriteDone;
+    (void) immediateCommit;
+}
+
+size_t
+StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op,
+                                   bool remove_index_and_attributes,
+                                   bool immediateCommit)
+{
+    const SerialNum serialNum = op.getSerialNum();
+    const LidVectorContext::LP &ctx = op.getLidsToRemove(_params._subDbId);
+    if (!ctx.get()) {
+        if (useDocumentMetaStore(serialNum)) {
+            _metaStore.commit(serialNum, serialNum);
+        }
+        return 0;
+    }
+    const LidVector &lidsToRemove(ctx->getLidVector());
+    bool useDMS = useDocumentMetaStore(serialNum);
+    bool explicitReuseLids = false;
+    if (useDMS) {
+        _metaStore.removeBatch(lidsToRemove, ctx->getDocIdLimit());
+        _metaStore.commit(serialNum, serialNum);
+        explicitReuseLids = _lidReuseDelayer.delayReuse(lidsToRemove);
+    }
+    std::shared_ptr<search::IDestructorCallback> onWriteDone;
+    if (remove_index_and_attributes) {
+        if (explicitReuseLids) {
+            onWriteDone = std::make_shared<search::ScheduleTaskCallback>(
+                    _writeService.master(),
+                    makeLambdaTask([=]()
+            { _metaStore.removeBatchComplete(lidsToRemove); }));
+        }
+        removeIndexedFields(serialNum, lidsToRemove, immediateCommit, onWriteDone);
+        removeAttributes(serialNum, lidsToRemove, immediateCommit, onWriteDone);
+    }
+    if (useDocumentStore(serialNum + 1)) {
+        for (const auto &lid : lidsToRemove) {
+            _summaryAdapter->remove(serialNum, lid);
+        }
+    }
+    if (explicitReuseLids && !onWriteDone) {
+        _metaStore.removeBatchComplete(lidsToRemove);
+    }
+    return lidsToRemove.size();
+}
+
+void
+StoreOnlyFeedView::prepareDeleteBucket(DeleteBucketOperation &delOp)
+{
+    const BucketId &bucket = delOp.getBucketId();
+    LidVector lidsToRemove;
+    _metaStore.getLids(bucket, lidsToRemove);
+    LOG(debug,
+        "prepareDeleteBucket(): docType(%s), bucket(%s), lidsToRemove(%zu)",
+        _params._docTypeName.toString().c_str(),
+        bucket.toString().c_str(), lidsToRemove.size());
+
+    if (!lidsToRemove.empty()) {
+        LidVectorContext::LP ctx
+            (new LidVectorContext(_metaStore.getCommittedDocIdLimit(),
+                                  lidsToRemove));
+        delOp.setLidsToRemove(_params._subDbId, ctx);
+    }
+}
+
+
+void
+StoreOnlyFeedView::handleDeleteBucket(const DeleteBucketOperation &delOp)
+{
+    internalDeleteBucket(delOp);
+}
+
+
+void
+StoreOnlyFeedView::internalDeleteBucket(const DeleteBucketOperation &delOp)
+{
+    bool immediateCommit = _commitTimeTracker.needCommit();
+    size_t rm_count = removeDocuments(delOp, true, immediateCommit);
+    LOG(debug,
+        "internalDeleteBucket(): docType(%s), bucket(%s), lidsToRemove(%zu)",
+        _params._docTypeName.toString().c_str(),
+        delOp.getBucketId().toString().c_str(), rm_count);
+}
+
+
+// CombiningFeedView calls this only for the subdb we're moving to.
+void
+StoreOnlyFeedView::prepareMove(MoveOperation &moveOp)
+{
+    const DocumentId &docId = moveOp.getDocument()->getId();
+    const document::GlobalId &gid = docId.getGlobalId();
+    documentmetastore::IStore::Result inspectResult = _metaStore.inspect(gid);
+    assert(!inspectResult._found);
+    moveOp.setDbDocumentId(DbDocumentId(_params._subDbId, inspectResult._lid));
+}
+
+
+// CombiningFeedView calls this for both source and target subdb.
+void
+StoreOnlyFeedView::handleMove(const MoveOperation &moveOp)
+{
+    assert(moveOp.getValidDbdId());
+    assert(moveOp.getValidPrevDbdId());
+    assert(moveOp.movingLidIfInSameSubDb());
+
+    const SerialNum serialNum = moveOp.getSerialNum();
+    const Document::SP &doc = moveOp.getDocument();
+    const DocumentId &docId = doc->getId();
+    VLOG(getDebugLevel(moveOp.getNewOrPrevLid(_params._subDbId), doc->getId()),
+         "database(%s): handleMove: serialNum(%" PRIu64 "), "
+         "docId(%s), lid(%u,%u) prevLid(%u,%u)"
+         " subDbId %u document(%ld) = {\n%s\n}",
+         _params._docTypeName.toString().c_str(),
+         serialNum,
+         doc->getId().toString().c_str(),
+         moveOp.getSubDbId(),
+         moveOp.getLid(),
+         moveOp.getPrevSubDbId(),
+         moveOp.getPrevLid(),
+         _params._subDbId,
+         doc->toString(true).size(),
+         doc->toString(true).c_str());
+
+    uint32_t oldDocIdLimit = _metaStore.getCommittedDocIdLimit();
+    adjustMetaStore(moveOp, docId);
+    bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId);
+    if (moveOp.getValidDbdId(_params._subDbId)) {
+        _summaryAdapter->put(serialNum, *doc, moveOp.getLid());
+        bool immediateCommit = _commitTimeTracker.needCommit();
+        FeedToken::UP token;
+        std::shared_ptr<PutDoneContext> onWriteDone =
+            createPutDoneContext(token, moveOp.getType(), _params._metrics,
+                                 immediateCommit &&
+                                 moveOp.getLid() >= oldDocIdLimit);
+        putAttributes(serialNum, moveOp.getLid(), *doc, immediateCommit, onWriteDone);
+        putIndexedFields(serialNum, moveOp.getLid(), doc, immediateCommit, onWriteDone);
+    }
+    if (docAlreadyExists && moveOp.changedDbdId()) {
+        internalRemove(FeedToken::UP(), serialNum, moveOp.getPrevLid(), moveOp.getType());
+    }
+}
+
+
+void
+StoreOnlyFeedView::heartBeat(search::SerialNum serialNum)
+{
+    assert(_writeService.master().isCurrentThread());
+    _metaStore.removeAllOldGenerations();
+    if (serialNum > _metaStore.getLastSerialNum()) {
+        _metaStore.commit(serialNum, serialNum);
+    }
+    _summaryAdapter->heartBeat(serialNum);
+    heartBeatIndexedFields(serialNum);
+    heartBeatAttributes(serialNum);
+}
+
+
+// CombiningFeedView calls this only for the removed subdb.
+void
+StoreOnlyFeedView::
+handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp)
+{
+    assert(_params._subDbType == SubDbType::REMOVED);
+    assert(pruneOp.getSubDbId() == _params._subDbId);
+    uint32_t rm_count = removeDocuments(pruneOp, false, false);
+
+    LOG(debug,
+        "MinimalFeedView::handlePruneRemovedDocuments called,"
+        " doctype(%s)"
+        " %u lids pruned,"
+        " limit %u",
+        _params._docTypeName.toString().c_str(), rm_count,
+        static_cast<uint32_t>(pruneOp.getLidsToRemove()->getDocIdLimit()));
+}
+
+void
+StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+{
+    assert(_params._subDbId == op.getSubDbId());
+    const SerialNum serialNum = op.getSerialNum();
+    if (useDocumentMetaStore(serialNum)) {
+        getDocumentMetaStore()->get().compactLidSpace(op.getLidLimit());
+        std::shared_ptr<ForceCommitContext>
+            commitContext(std::make_shared<ForceCommitContext>
+                          (_writeService.master(),
+                           _metaStore));
+        commitContext->holdUnblockShrinkLidSpace();
+        forceCommit(serialNum, commitContext);
+    }
+}
+
+const ISimpleDocumentMetaStore *
+StoreOnlyFeedView::getDocumentMetaStorePtr() const
+{
+    return &_documentMetaStoreContext->get();
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
new file mode 100644
index 00000000000..11c588eb2ab
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
@@ -0,0 +1,383 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "fileconfigmanager.h"
+#include "ifeedview.h"
+#include "isummaryadapter.h"
+#include "replaypacketdispatcher.h"
+#include "searchcontext.h"
+#include "tlcproxy.h"
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/common/feeddebugger.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
+#include <vespa/searchcore/proton/feedoperation/feedoperation.h>
+#include <vespa/searchcore/proton/persistenceengine/resulthandler.h>
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/vespalib/util/threadstackexecutorbase.h>
+#include <vespa/log/log.h>
+
+namespace search
+{
+
+class IDestructorCallback;
+
+}
+
+namespace proton
+{
+
+class IReplayConfig;
+class PerDocTypeFeedMetrics;
+class ForceCommitContext;
+class OperationDoneContext;
+class PutDoneContext;
+class RemoveDoneContext;
+class CommitTimeTracker;
+
+namespace documentmetastore
+{
+
+class ILidReuseDelayer;
+
+}
+
+
+/**
+ * The feed view used by the store-only sub database.
+ *
+ * Handles inserting/updating/removing of documents to the underlying document store.
+ */
+class StoreOnlyFeedView : public IFeedView,
+                          protected FeedDebugger
+{
+protected:
+    typedef search::transactionlog::Packet Packet;
+public:
+    typedef std::unique_ptr<StoreOnlyFeedView> UP;
+    typedef std::shared_ptr<StoreOnlyFeedView> SP;
+    typedef search::SerialNum SerialNum;
+    typedef LidVectorContext::LidVector LidVector;
+    using OnWriteDoneType =
+        const std::shared_ptr<search::IDestructorCallback> &;
+    using OnForceCommitDoneType =
+        const std::shared_ptr<ForceCommitContext> &;
+    using OnOperationDoneType = const std::shared_ptr<OperationDoneContext> &;
+    using OnPutDoneType = const std::shared_ptr<PutDoneContext> &;
+    using OnRemoveDoneType = const std::shared_ptr<RemoveDoneContext> &;
+
+    struct Context
+    {
+        const ISummaryAdapter::SP               &_summaryAdapter;
+        const search::index::Schema::SP         &_schema;
+        const IDocumentMetaStoreContext::SP     &_documentMetaStoreContext;
+        const document::DocumentTypeRepo::SP    &_repo;
+        searchcorespi::index::IThreadingService &_writeService;
+        documentmetastore::ILidReuseDelayer     &_lidReuseDelayer;
+        CommitTimeTracker                       &_commitTimeTracker;
+
+        Context(const ISummaryAdapter::SP &summaryAdapter,
+                const search::index::Schema::SP &schema,
+                const IDocumentMetaStoreContext::SP &documentMetaStoreContext,
+                const document::DocumentTypeRepo::SP &repo,
+                searchcorespi::index::IThreadingService &writeService,
+                documentmetastore::ILidReuseDelayer &lidReuseDelayer,
+                CommitTimeTracker &commitTimeTracker)
+            : _summaryAdapter(summaryAdapter),
+              _schema(schema),
+              _documentMetaStoreContext(documentMetaStoreContext),
+              _repo(repo),
+              _writeService(writeService),
+              _lidReuseDelayer(lidReuseDelayer),
+              _commitTimeTracker(commitTimeTracker)
+        {
+        }
+    };
+
+    struct PersistentParams
+    {
+        const SerialNum        _flushedDocumentMetaStoreSerialNum;
+        const SerialNum        _flushedDocumentStoreSerialNum;
+        const DocTypeName      _docTypeName;
+        PerDocTypeFeedMetrics &_metrics;
+        const uint32_t         _subDbId;
+        const SubDbType        _subDbType;
+
+        PersistentParams(SerialNum flushedDocumentMetaStoreSerialNum,
+                         SerialNum flushedDocumentStoreSerialNum,
+                         const DocTypeName &docTypeName,
+                         PerDocTypeFeedMetrics &metrics,
+                         uint32_t subDbId,
+                         SubDbType subDbType)
+            : _flushedDocumentMetaStoreSerialNum(
+                      flushedDocumentMetaStoreSerialNum),
+              _flushedDocumentStoreSerialNum(flushedDocumentStoreSerialNum),
+              _docTypeName(docTypeName),
+              _metrics(metrics),
+              _subDbId(subDbId),
+              _subDbType(subDbType)
+        {
+        }
+    };
+
+protected:
+    struct UpdateScope
+    {
+        bool _indexedFields;
+        bool _nonAttributeFields;
+
+        UpdateScope()
+            : _indexedFields(false),
+              _nonAttributeFields(false)
+        {
+        }
+    };
+
+protected:
+    const ISummaryAdapter::SP                _summaryAdapter;
+    const search::index::Schema::SP          _schema;
+    const IDocumentMetaStoreContext::SP      _documentMetaStoreContext;
+    const document::DocumentTypeRepo::SP     _repo;
+    searchcorespi::index::IThreadingService &_writeService;
+    PersistentParams                         _params;
+    IDocumentMetaStore                      &_metaStore;
+    const document::DocumentType            *_docType;
+    documentmetastore::ILidReuseDelayer     &_lidReuseDelayer;
+    CommitTimeTracker                       &_commitTimeTracker;
+
+    bool
+    useDocumentStore(SerialNum replaySerialNum) const
+    {
+        return replaySerialNum > _params._flushedDocumentStoreSerialNum;
+    }
+
+private:
+    bool
+    useDocumentMetaStore(SerialNum replaySerialNum) const
+    {
+        return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum;
+    }
+
+    void
+    adjustMetaStore(const DocumentOperation &op,
+                    const document::DocumentId &docId);
+
+    void
+    internalPut(FeedToken::UP token,
+                const PutOperation &putOp);
+
+    void
+    internalUpdate(FeedToken::UP token,
+                   const UpdateOperation &updOp);
+
+    void
+    updateIndexAndDocumentStore(bool indexedFieldsInScope,
+                                SerialNum serialNum,
+                                search::DocumentIdT lid,
+                                const document::DocumentUpdate &upd,
+                                bool immediateCommit,
+                                OnOperationDoneType onWriteDone);
+
+    bool
+    lookupDocId(const document::DocumentId &gid,
+                search::DocumentIdT & lid) const;
+
+    void
+    internalRemove(FeedToken::UP token,
+                   const RemoveOperation &rmOp);
+
+    // Removes documents from meta store and document store.
+    // returns the number of documents removed.
+    size_t removeDocuments(const RemoveDocumentsOperation &op,
+                           bool remove_index_and_attribute_fields,
+                           bool immediateCommit);
+
+    void internalRemove(FeedToken::UP token,
+                        SerialNum serialNum,
+                        search::DocumentIdT lid,
+                        FeedOperation::Type opType);
+
+    // Ack token early if visibility delay is nonzero
+    void considerEarlyAck(FeedToken::UP &token, FeedOperation::Type opType);
+
+protected:
+    virtual void
+    internalDeleteBucket(const DeleteBucketOperation &delOp);
+
+    virtual void
+    heartBeatIndexedFields(SerialNum serialNum);
+
+    virtual void
+    heartBeatAttributes(SerialNum serialNum);
+
+private:
+    virtual void
+    putAttributes(SerialNum serialNum,
+                  search::DocumentIdT lid,
+                  const document::Document &doc,
+                  bool immediateCommit,
+                  OnPutDoneType onWriteDone);
+
+    virtual void
+    putIndexedFields(SerialNum serialNum,
+                     search::DocumentIdT lid,
+                     const document::Document::SP &newDoc,
+                     bool immediateCommit,
+                     OnOperationDoneType onWriteDone);
+
+    virtual UpdateScope
+    getUpdateScope(const document::DocumentUpdate &upd);
+
+    virtual void
+    updateAttributes(SerialNum serialNum,
+                     search::DocumentIdT lid,
+                     const document::DocumentUpdate &upd,
+                     bool immediateCommit,
+                     OnOperationDoneType onWriteDone);
+
+    virtual void
+    updateIndexedFields(SerialNum serialNum,
+                        search::DocumentIdT lid,
+                        const document::Document::SP &newDoc,
+                        bool immediateCommit,
+                        OnOperationDoneType onWriteDone);
+
+    virtual void
+    removeAttributes(SerialNum serialNum,
+                     search::DocumentIdT lid,
+                     bool immediateCommit,
+                     OnRemoveDoneType onWriteDone);
+
+    virtual void
+    removeIndexedFields(SerialNum serialNum,
+                        search::DocumentIdT lid,
+                        bool immediateCommit,
+                        OnRemoveDoneType onWriteDone);
+
+protected:
+    virtual void
+    removeAttributes(SerialNum serialNum,
+                     const LidVector &lidsToRemove,
+                     bool immediateCommit,
+                     OnWriteDoneType onWriteDone);
+
+    virtual void
+    removeIndexedFields(SerialNum serialNum,
+                        const LidVector &lidsToRemove,
+                        bool immediateCommit,
+                        OnWriteDoneType onWriteDone);
+
+public:
+    StoreOnlyFeedView(const Context &ctx,
+                      const PersistentParams &params);
+
+    virtual ~StoreOnlyFeedView() {}
+
+    const ISummaryAdapter::SP &
+    getSummaryAdapter() const { return _summaryAdapter; }
+
+    const search::index::Schema::SP &
+    getSchema() const { return _schema; }
+
+    const PersistentParams &
+    getPersistentParams() const { return _params; }
+
+    const search::IDocumentStore &
+    getDocumentStore() const { return _summaryAdapter->getDocumentStore(); }
+
+    const IDocumentMetaStoreContext::SP &
+    getDocumentMetaStore() const { return _documentMetaStoreContext; }
+
+    searchcorespi::index::IThreadingService &getWriteService() {
+        return _writeService;
+    }
+
+    documentmetastore::ILidReuseDelayer &
+    getLidReuseDelayer()
+    {
+        return _lidReuseDelayer;
+    }
+
+    CommitTimeTracker &getCommitTimeTracker() { return _commitTimeTracker; }
+
+    /**
+     * Implements IFeedView.
+     */
+    virtual const document::DocumentTypeRepo::SP &
+    getDocumentTypeRepo() const { return _repo; }
+
+    /**
+     * Implements IFeedView.
+     */
+    virtual const ISimpleDocumentMetaStore *
+    getDocumentMetaStorePtr() const;
+
+    /**
+     * Similar to IPersistenceHandler functions.
+     * Takes pointer to feed token instead of instance because
+     * when replaying the spooler we don't have a feed token.
+     */
+
+    virtual void
+    preparePut(PutOperation &putOp);
+
+    virtual void
+    handlePut(FeedToken *token,
+              const PutOperation &putOp);
+
+    virtual void
+    prepareUpdate(UpdateOperation &updOp);
+
+    virtual void
+    handleUpdate(FeedToken *token,
+                 const UpdateOperation &updOp);
+
+    virtual void
+    prepareRemove(RemoveOperation &rmOp);
+
+    virtual void
+    handleRemove(FeedToken *token,
+                 const RemoveOperation &rmOp);
+
+    virtual void
+    prepareDeleteBucket(DeleteBucketOperation &delOp);
+
+    virtual void
+    handleDeleteBucket(const DeleteBucketOperation &delOp);
+
+    virtual void
+    prepareMove(MoveOperation &putOp);
+
+    virtual void
+    handleMove(const MoveOperation &putOp);
+
+    virtual void
+    heartBeat(search::SerialNum serialNum);
+
+    virtual void
+    sync();
+
+    virtual void forceCommit(SerialNum serialNum) override;
+
+    virtual void forceCommit(SerialNum serialNum,
+                             OnForceCommitDoneType onCommitDone);
+
+    /**
+     * Prune lids present in operation.  Caller must call doneSegment()
+     * on prune operation after this call.
+     *
+     * Called by writer thread.
+     */
+    virtual void
+    handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp);
+
+    virtual void
+    handleCompactLidSpace(const CompactLidSpaceOperation &op);
+
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp
new file mode 100644
index 00000000000..adb8423c68e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.summaryadapter");
+
+#include "summaryadapter.h"
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+
+using namespace document;
+
+namespace proton {
+
+SummaryAdapter::SummaryAdapter(const SummaryManager::SP &mgr)
+    : _mgr(mgr),
+      _imgr(mgr),
+      _lastSerial(_mgr->getBackingStore().lastSyncToken())
+{
+    // empty
+}
+
+bool SummaryAdapter::ignore(search::SerialNum serialNum) const
+{
+    assert(serialNum != 0);
+    return serialNum <= _lastSerial;
+}
+
+void
+SummaryAdapter::put(search::SerialNum serialNum,
+                    const document::Document &doc,
+                    const search::DocumentIdT lid)
+{
+    if ( ! ignore(serialNum) ) {
+        LOG(spam, "SummaryAdapter::put(docId = '%s', lid = %u, document = '%s')",
+            doc.getId().toString().c_str(), lid, doc.toString(true).c_str());
+        _mgr->putDocument(serialNum, doc, lid);
+        _lastSerial = serialNum;
+    }
+}
+
+void
+SummaryAdapter::remove(search::SerialNum serialNum,
+                       const search::DocumentIdT lid)
+{
+    if ( ! ignore(serialNum + 1) ) {
+        _mgr->removeDocument(serialNum, lid);
+        _lastSerial = serialNum;
+    }
+}
+
+void
+SummaryAdapter::update(search::SerialNum serialNum,
+                       const document::DocumentUpdate &upd,
+                       const search::DocumentIdT lid,
+                       const document::DocumentTypeRepo &repo)
+{
+    if ( ! ignore(serialNum) ) {
+        search::IDocumentStore &store = _mgr->getBackingStore();
+        document::Document::UP doc = store.read(lid, repo);
+        if (doc.get() == NULL) {
+            LOG(warning, "Did not find document '%s' to update.",
+                upd.getId().toString().c_str());
+            return;
+        }
+        upd.applyTo(*doc);
+
+        LOG(spam, "SummaryAdapter::update(docId = '%s', lid = %u, update = '%s')",
+            upd.getId().toString().c_str(), lid, upd.toString().c_str());
+        _mgr->putDocument(serialNum, *doc, lid);
+        _lastSerial = serialNum;
+    }
+}
+
+
+void
+SummaryAdapter::heartBeat(search::SerialNum serialNum)
+{
+    if (serialNum > _lastSerial) {
+        remove(serialNum, 0u); // XXX: Misused lid 0
+    }
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h
new file mode 100644
index 00000000000..8e266e7df5a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/docsummary/summarymanager.h>
+#include "isummaryadapter.h"
+
+namespace proton {
+
+class SummaryAdapter : public ISummaryAdapter {
+private:
+    SummaryManager::SP _mgr;
+    ISummaryManager::SP _imgr;
+    search::SerialNum _lastSerial;
+
+    bool ignore(search::SerialNum serialNum) const;
+
+public:
+    SummaryAdapter(const SummaryManager::SP &mgr);
+
+    /**
+     * Implements ISummaryAdapter.
+     */
+    virtual void put(search::SerialNum serialNum,
+                     const document::Document &doc,
+                     const search::DocumentIdT lid);
+    virtual void remove(search::SerialNum serialNum,
+                        const search::DocumentIdT lid);
+    virtual void update(search::SerialNum serialNum,
+                        const document::DocumentUpdate &upd,
+                        const search::DocumentIdT lid,
+                        const document::DocumentTypeRepo &repo);
+
+    virtual void
+    heartBeat(search::SerialNum serialNum);
+
+    virtual const search::IDocumentStore &
+    getDocumentStore(void) const
+    {
+        return _imgr->getBackingStore();
+    }
+
+    virtual std::unique_ptr<document::Document>
+    get(const search::DocumentIdT lid,
+        const document::DocumentTypeRepo &repo)
+    {
+        return _imgr->getBackingStore().read(lid, repo);
+    }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp
new file mode 100644
index 00000000000..b4c509744f7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.tlcproxy");
+
+#include "tlcproxy.h"
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/searchlib/common/bitvector.h>
+#include <vespa/document/bucket/bucketid.h>
+
+using vespalib::nbostream;
+
+namespace proton {
+
+void TlcProxy::commit(search::SerialNum serialNum, search::transactionlog::Type type, const vespalib::nbostream &buf)
+{
+    search::transactionlog::Packet::Entry
+        entry(serialNum, type, vespalib::ConstBufferRef(buf.c_str(), buf.size()));
+    search::transactionlog::Packet packet;
+    packet.add(entry);
+    packet.close();
+    if (_tlsDirectWriter != NULL) {
+        _tlsDirectWriter->commit(_session.getDomain(), packet);
+    } else {
+        if (!_session.commit(vespalib::ConstBufferRef(packet.getHandle().c_str(), packet.getHandle().size()))) {
+            throw vespalib::IllegalStateException(vespalib::make_string(
+                        "Failed to commit packet %" PRId64
+                        " to TLS (type = %d, size = %d).",
+                        entry.serial(), type, (uint32_t)buf.size()));
+        }
+    }
+}
+
+void
+TlcProxy::storeOperation(const FeedOperation &op)
+{
+    nbostream stream;
+    op.serialize(stream);
+    LOG(debug, "storeOperation(): serialNum(%" PRIu64 "), type(%u), size(%zu)",
+        op.getSerialNum(), (uint32_t)op.getType(), stream.size());
+    commit(op.getSerialNum(), (uint32_t)op.getType(), stream);
+}
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h
new file mode 100644
index 00000000000..342db7d7cf0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/searchcore/proton/feedoperation/feedoperation.h>
+#include <vespa/searchlib/query/base.h>
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
+#include "fileconfigmanager.h"
+#include <persistence/spi/types.h>
+
+namespace proton {
+
+class TlcProxy {
+    search::transactionlog::TransLogClient::Session & _session;
+    search::transactionlog::Writer                  * _tlsDirectWriter;
+
+    void commit( search::SerialNum serialNum, search::transactionlog::Type type, const vespalib::nbostream &buf);
+public:
+    typedef std::unique_ptr<TlcProxy> UP;
+
+    TlcProxy(search::transactionlog::TransLogClient::Session &session, search::transactionlog::Writer * writer = NULL)
+        : _session(session), _tlsDirectWriter(writer) {}
+
+    void storeOperation(const FeedOperation &op);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/tls_replay_progress.h b/searchcore/src/vespa/searchcore/proton/server/tls_replay_progress.h
new file mode 100644
index 00000000000..a4e5c86b84d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/tls_replay_progress.h
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+class TlsReplayProgress
+{
+private:
+    const vespalib::string  _domainName;
+    const search::SerialNum _first;
+    const search::SerialNum _last;
+    search::SerialNum       _current;
+
+public:
+    typedef std::unique_ptr<TlsReplayProgress> UP;
+
+    TlsReplayProgress(const vespalib::string &domainName,
+                      search::SerialNum first,
+                      search::SerialNum last)
+        : _domainName(domainName),
+          _first(first),
+          _last(last),
+          _current(first)
+    {
+    }
+    const vespalib::string &getDomainName() const { return _domainName; }
+    search::SerialNum getFirst() const { return _first; }
+    search::SerialNum getLast() const { return _last; }
+    search::SerialNum getCurrent() const { return _current; }
+    float getProgress() const {
+        if (_first == _last) {
+            return 1.0;
+        } else {
+            return ((float)(_current - _first)/float(_last - _first));
+        }
+    }
+    void updateCurrent(search::SerialNum current) { _current = current; }
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp b/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp
new file mode 100644
index 00000000000..f17538836bc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "tlssyncer.h"
+#include "igetserialnum.h"
+#include <vespa/vespalib/util/threadexecutor.h>
+#include <vespa/searchlib/common/lambdatask.h>
+#include <vespa/searchlib/transactionlog/syncproxy.h>
+#include <future>
+
+using search::makeLambdaTask;
+using search::SerialNum;
+
+namespace proton
+{
+
+TlsSyncer::TlsSyncer(vespalib::ThreadExecutor &executor,
+                     const IGetSerialNum &getSerialNum,
+                     search::transactionlog::SyncProxy &proxy)
+    : _executor(executor),
+      _getSerialNum(getSerialNum),
+      _proxy(proxy)
+{
+}
+
+
+void
+TlsSyncer::sync()
+{
+    std::promise<SerialNum> promise;
+    std::future<SerialNum> future = promise.get_future();
+    _executor.execute(makeLambdaTask([&]() { promise.set_value(_getSerialNum.getSerialNum()); }));
+    SerialNum serialNum = future.get();
+    _proxy.sync(serialNum);
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/tlssyncer.h b/searchcore/src/vespa/searchcore/proton/server/tlssyncer.h
new file mode 100644
index 00000000000..1010755b493
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/tlssyncer.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "itlssyncer.h"
+
+namespace vespalib { class ThreadExecutor; }
+namespace search { namespace transactionlog { class SyncProxy; } }
+
+namespace proton
+{
+
+class IGetSerialNum;
+
+/*
+ * Class for syncing transaction log server in a safe manner.  The
+ * serial number is retrieved by running a task in the document db
+ * master thread to ensure that it reflects changes performed to data
+ * structures as of now.
+ */
+class TlsSyncer : public ITlsSyncer
+{
+    vespalib::ThreadExecutor &_executor;
+    const IGetSerialNum &_getSerialNum;
+    search::transactionlog::SyncProxy &_proxy;
+public:
+    virtual ~TlsSyncer(void) = default;
+
+    TlsSyncer(vespalib::ThreadExecutor &executor,
+              const IGetSerialNum &getSerialNum,
+              search::transactionlog::SyncProxy &proxy);
+
+    virtual void sync() override;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/tlswriter.h b/searchcore/src/vespa/searchcore/proton/server/tlswriter.h
new file mode 100644
index 00000000000..554cd9c77d9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/tlswriter.h
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/serialnum.h>
+
+namespace proton {
+class FeedOperation;
+
+/**
+ * Interface for writing to the TransactionLogServer.
+ */
+struct TlsWriter {
+    virtual ~TlsWriter() {}
+
+    virtual void storeOperation(const FeedOperation &op) = 0;
+    virtual bool erase(search::SerialNum oldest_to_keep) = 0;
+
+    virtual search::SerialNum
+    sync(search::SerialNum syncTo) = 0;
+};
+
+}  // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp
new file mode 100644
index 00000000000..d35d9a41f52
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp
@@ -0,0 +1,168 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.transactionlogmanager");
+#include "configstore.h"
+#include "transactionlogmanager.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+using vespalib::IllegalStateException;
+using vespalib::make_string;
+using search::transactionlog::TransLogClient;
+
+namespace proton
+{
+
+
+void
+TransactionLogManager::doLogReplayComplete(const vespalib::string &domainName,
+                                           int64_t elapsedTime) const
+{
+    EventLogger::transactionLogReplayComplete(domainName, elapsedTime);
+}
+
+
+TransactionLogManager::TransactionLogManager(const vespalib::string &tlsSpec,
+        const vespalib::string &domainName)
+    : TransactionLogManagerBase(tlsSpec, domainName),
+      _visitor()
+{
+}
+
+
+void
+TransactionLogManager::init(SerialNum oldestConfigSerial,
+                            SerialNum &prunedSerialNum,
+                            SerialNum &serialNum)
+{
+    StatusResult res = TransactionLogManagerBase::init();
+    prunedSerialNum = res.serialBegin > 0 ? (res.serialBegin - 1) : 0;
+    serialNum = res.serialEnd;
+    if (oldestConfigSerial != 0) {
+        prunedSerialNum = std::max(prunedSerialNum, oldestConfigSerial);
+    }
+}
+
+namespace {
+
+void getStatus(TransLogClient::Session & session,
+               search::SerialNum & serialBegin,
+               search::SerialNum & serialEnd,
+               size_t & count)
+{
+    if (!session.status(serialBegin, serialEnd, count)) {
+        throw IllegalStateException(
+                make_string(
+                    "Could not get status from session with"
+                    " domain '%s' on TLS '%s'",
+                    session.getDomain().c_str(),
+                    session.getTLC().getRPCTarget().c_str()));
+    }
+}
+
+void getStatus(TransLogClient & client,
+               const vespalib::string & domainName,
+               search::SerialNum & serialBegin,
+               search::SerialNum & serialEnd,
+               size_t & count)
+{
+    TransLogClient::Session::UP session = client.open(domainName);
+    if (session.get() == NULL) {
+        throw IllegalStateException(
+                make_string(
+                    "Could not open session with domain '%s' on TLS '%s'",
+                    session->getDomain().c_str(),
+                    session->getTLC().getRPCTarget().c_str()));
+    }
+    getStatus(*session, serialBegin, serialEnd, count);
+}
+
+}
+
+void
+TransactionLogManager::prepareReplay(TransLogClient &client,
+                                     const vespalib::string &domainName,
+                                     SerialNum flushedIndexMgrSerial,
+                                     SerialNum flushedSummaryMgrSerial,
+                                     ConfigStore &config_store)
+{
+    SerialNum oldestConfigSerial = config_store.getOldestSerialNum();
+    SerialNum from = flushedIndexMgrSerial;
+    SerialNum to = flushedSummaryMgrSerial;
+    assert(oldestConfigSerial != 0);
+    from = std::max(from, oldestConfigSerial);
+    if (from < to) {
+        SerialNum serialBegin = 0;
+        SerialNum serialEnd = 0;
+        size_t count = 0;
+        getStatus(client, domainName, serialBegin, serialEnd, count);
+        SerialNum prunedToken = serialBegin > 0 ? (serialBegin - 1) : 0;
+        from = std::max(from, prunedToken);
+        if (serialEnd < flushedSummaryMgrSerial) {
+            throw IllegalStateException(
+                    make_string("SummaryStore '%ld' is more recent than "
+                                "transactionlog '%ld'. Immpossible !!",
+                                flushedSummaryMgrSerial, serialEnd));
+        }
+        if (serialEnd < flushedIndexMgrSerial) {
+            throw IllegalStateException(
+                    make_string("IndexStore '%ld' is more recent than "
+                                "transactionlog '%ld'. Immpossible !!",
+                                flushedIndexMgrSerial, serialEnd));
+        }
+    }
+}
+
+
+TlsReplayProgress::UP
+TransactionLogManager::startReplay(SerialNum first,
+                                   SerialNum syncToken,
+                                   TransLogClient::Session::Callback &callback)
+{
+    assert(_visitor.get() == NULL);
+    _visitor = createTlcVisitor(callback);
+    if (_visitor.get() == NULL) {
+        throw IllegalStateException(
+                make_string(
+                    "Could not create visitor for "
+                    "replaying domain '%s' on TLS '%s'",
+                    getDomainName().c_str(), getRpcTarget().c_str()));
+    }
+    TransactionLogManagerBase::internalStartReplay();
+
+    if (LOG_WOULD_LOG(event)) {
+        EventLogger::transactionLogReplayStart(
+                getDomainName(), first, syncToken);
+    }
+    if (!_visitor->visit(first, syncToken)) {
+        throw IllegalStateException(
+                make_string(
+                    "Could not start visitor for "
+                    "replaying domain '%s<%ld, %ld]' on TLS '%s'",
+                    getDomainName().c_str(),
+                    first, syncToken, getRpcTarget().c_str()));
+    }
+    return TlsReplayProgress::UP(new TlsReplayProgress(getDomainName(), first, syncToken));
+}
+
+
+void
+TransactionLogManager::replayDone()
+{
+    assert(_visitor.get() != NULL);
+    LOG(debug,
+        "Transaction log replayed for domain '%s'", getDomainName().c_str());
+    changeReplayDone();
+    LOG(debug,
+        "Broadcasted replay done for domain '%s'", getDomainName().c_str());
+    if (LOG_WOULD_LOG(event)) {
+        logReplayComplete();
+    }
+    _visitor.reset();
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.h b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.h
new file mode 100644
index 00000000000..0afa55bea24
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.h
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documentdbconfig.h"
+#include "tls_replay_progress.h"
+#include "transactionlogmanagerbase.h"
+#include <vespa/searchcore/proton/index/i_index_writer.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
+
+namespace proton {
+class ConfigStore;
+
+/**
+ * Class managing the initialization and replay of the transaction log.
+ **/
+class TransactionLogManager : public TransactionLogManagerBase
+{
+    TransLogClient::Visitor::UP _visitor;
+
+    virtual void doLogReplayComplete(const vespalib::string &domainName,
+                                     int64_t elapsedTime) const;
+
+public:
+    /**
+     * Create a new manager.
+     *
+     * @param tlsSpec the spec of the transaction log server.
+     * @param domainName the name of the domain this manager should handle.
+     **/
+    TransactionLogManager(const vespalib::string &tlsSpec,
+                          const vespalib::string &domainName);
+
+    /**
+     * Init the transaction log.
+     *
+     * @param oldestConfigSerial the serial num of the oldest config.
+     * @param the pruned serial num will be set to 1 lower than
+     *        the serial num of the first entry in the transaction log.
+     * @param the current serial num will be set to 1 higher than
+     *        the serial num of the last entry in the transaction log.
+     **/
+    void
+    init(SerialNum oldestConfigSerial,
+         SerialNum &prunedSerialNum,
+         SerialNum &serialNum);
+
+    /**
+     * Prepare replay of the transaction log.
+     **/
+    static void
+    prepareReplay(TransLogClient &client,
+                  const vespalib::string &domainName,
+                  SerialNum flushedIndexMgrSerial,
+                  SerialNum flushedSummaryMgrSerial,
+                  ConfigStore &config_store);
+
+    /**
+     * Start replay of the transaction log.
+     **/
+    TlsReplayProgress::UP
+    startReplay(SerialNum first,
+                SerialNum syncToken,
+                TransLogClient::Session::Callback &callback);
+
+    /**
+     * Indicate that replay is done.
+     * Should be called when session callback handles eof().
+     **/
+    void replayDone();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp
new file mode 100644
index 00000000000..afab869e177
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp
@@ -0,0 +1,147 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.transactionlogmanagerbase");
+#include "transactionlogmanagerbase.h"
+#include <vespa/searchcore/proton/common/eventlogger.h>
+#include <stdexcept>
+
+using search::transactionlog::TransLogClient;
+
+namespace proton {
+
+
+TransactionLogManagerBase::TransactionLogManagerBase(
+        const vespalib::string &tlsSpec, const vespalib::string &domainName) :
+    _tlc(tlsSpec),
+    _tlcSession(),
+    _domainName(domainName),
+    _replayMonitor(),
+    _replayDone(false),
+    _replayStarted(false),
+    _replayStartTime(0)
+{
+}
+
+TransactionLogManagerBase::~TransactionLogManagerBase()
+{
+}
+
+
+TransactionLogManagerBase::StatusResult
+TransactionLogManagerBase::init()
+{
+    TransLogClient::Session::UP session = _tlc.open(_domainName);
+    if (session.get() == NULL) {
+        if (!_tlc.create(_domainName)) {
+            vespalib::string str = vespalib::make_string(
+                    "Failed creating domain '%s' on TLS '%s'",
+                    _domainName.c_str(), _tlc.getRPCTarget().c_str());
+            throw std::runtime_error(str);
+        }
+        LOG(debug, "Created domain '%s' on TLS '%s'",
+            _domainName.c_str(), _tlc.getRPCTarget().c_str());
+        session = _tlc.open(_domainName);
+        if (session.get() == NULL) {
+            vespalib::string str = vespalib::make_string(
+                    "Could not open session for domain '%s' on TLS '%s'",
+                    _domainName.c_str(), _tlc.getRPCTarget().c_str());
+            throw std::runtime_error(str);
+        }
+    }
+    LOG(debug, "Opened domain '%s' on TLS '%s'",
+        _domainName.c_str(), _tlc.getRPCTarget().c_str());
+    StatusResult res;
+    if (!session->status(res.serialBegin, res.serialEnd, res.count)) {
+        vespalib::string str = vespalib::make_string(
+                "Could not get status from session with domain '%s' on TLS '%s'",
+                _domainName.c_str(), _tlc.getRPCTarget().c_str());
+        throw std::runtime_error(str);
+    }
+    LOG(debug,
+        "Status for domain '%s': serialBegin(%" PRIu64 "), serialEnd(%" PRIu64 "), count(%zu)",
+        _domainName.c_str(), res.serialBegin, res.serialEnd, res.count);
+    _tlcSession = std::move(session);
+    return res;
+}
+
+
+void
+TransactionLogManagerBase::internalStartReplay()
+{
+    vespalib::MonitorGuard guard(_replayMonitor);
+    _replayStarted = true;
+    _replayDone = false;
+    FastOS_Time timer;
+    timer.SetNow();
+    _replayStartTime = timer.MilliSecs();
+}
+
+
+void
+TransactionLogManagerBase::markReplayStarted(void)
+{
+    vespalib::MonitorGuard guard(_replayMonitor);
+    _replayStarted = true;
+}
+
+
+void TransactionLogManagerBase::changeReplayDone()
+{
+    vespalib::MonitorGuard guard(_replayMonitor);
+    _replayDone = true;
+    guard.broadcast();
+}
+
+
+void
+TransactionLogManagerBase::waitForReplayDone() const
+{
+    vespalib::MonitorGuard guard(_replayMonitor);
+    while (_replayStarted && !_replayDone) {
+        guard.wait();
+    }
+}
+
+
+void
+TransactionLogManagerBase::close()
+{
+    if (_tlcSession.get() != NULL) {
+        _tlcSession->close();
+    }
+    // Delay destruction until replay is not active.
+    waitForReplayDone();
+    if (_tlcSession.get() != NULL) {
+        _tlcSession->clear();
+    }
+}
+
+TransLogClient::Subscriber::UP TransactionLogManagerBase::createTlcSubscriber(
+        TransLogClient::Session::Callback &callback) {
+    return _tlc.createSubscriber(_domainName, callback);
+}
+
+TransLogClient::Visitor::UP TransactionLogManagerBase::createTlcVisitor(
+        TransLogClient::Session::Callback &callback) {
+    return _tlc.createVisitor(_domainName, callback);
+}
+
+bool TransactionLogManagerBase::getReplayDone() const {
+    vespalib::MonitorGuard guard(_replayMonitor);
+    return _replayDone;
+}
+
+bool TransactionLogManagerBase::isDoingReplay() const {
+    vespalib::MonitorGuard guard(_replayMonitor);
+    return _replayStarted && !_replayDone;
+}
+
+void TransactionLogManagerBase::logReplayComplete() const {
+    FastOS_Time timer;
+    timer.SetMilliSecs(_replayStartTime);
+    doLogReplayComplete(_domainName, static_cast<int64_t>(timer.MilliSecsToNow()));
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h
new file mode 100644
index 00000000000..4a140f0caed
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/transactionlog/translogclient.h>
+#include <boost/noncopyable.hpp>
+
+namespace proton {
+
+/**
+ * Base class managing the initialization and replay of a transaction log.
+ **/
+class TransactionLogManagerBase : public boost::noncopyable {
+
+    search::transactionlog::TransLogClient _tlc;
+    search::transactionlog::TransLogClient::Session::UP _tlcSession;
+    vespalib::string            _domainName;
+    vespalib::Monitor           _replayMonitor;
+    volatile bool               _replayDone;
+    bool                        _replayStarted;
+    double                      _replayStartTime;
+
+protected:
+    typedef search::transactionlog::TransLogClient TransLogClient;
+    typedef search::SerialNum SerialNum;
+
+    struct StatusResult {
+        SerialNum serialBegin;
+        SerialNum serialEnd;
+        size_t    count;
+        StatusResult() : serialBegin(0), serialEnd(0), count(0) {}
+    };
+
+    StatusResult init();
+
+    void internalStartReplay();
+    virtual void doLogReplayComplete(const vespalib::string &domainName,
+                                     int64_t elapsedTime) const = 0;
+
+public:
+    /**
+     * Create a new manager.
+     *
+     * @param tlsSpec the spec of the transaction log server.
+     * @param domainName the name of the domain this manager should handle.
+     **/
+    TransactionLogManagerBase(const vespalib::string &tlsSpec,
+                              const vespalib::string &domainName);
+    virtual ~TransactionLogManagerBase();
+
+    void changeReplayDone();
+    void close();
+    TransLogClient::Subscriber::UP createTlcSubscriber(
+            TransLogClient::Session::Callback &callback);
+    TransLogClient::Visitor::UP createTlcVisitor(
+            TransLogClient::Session::Callback &callback);
+
+    void waitForReplayDone() const;
+
+    TransLogClient &getClient() { return _tlc; }
+    TransLogClient::Session *getSession() { return _tlcSession.get(); }
+    const vespalib::string &getDomainName() const { return _domainName; }
+    bool getReplayDone() const;
+    bool isDoingReplay() const;
+    void logReplayComplete() const;
+    const vespalib::string &getRpcTarget() const
+    { return _tlc.getRPCTarget(); }
+
+    void
+    markReplayStarted(void);
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp
new file mode 100644
index 00000000000..82db11aa0f8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "updatedonecontext.h"
+#include <vespa/searchcore/proton/common/feedtoken.h>
+
+namespace proton
+{
+
+
+UpdateDoneContext::UpdateDoneContext(std::unique_ptr<FeedToken> token,
+                                     const FeedOperation::Type opType,
+                                     PerDocTypeFeedMetrics &metrics,
+                                     const document::DocumentUpdate::SP &upd)
+    : OperationDoneContext(std::move(token), opType, metrics),
+      _upd(upd)
+{
+}
+
+
+UpdateDoneContext::~UpdateDoneContext()
+{
+}
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.h b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.h
new file mode 100644
index 00000000000..d763b1d270e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "operationdonecontext.h"
+#include <vespa/document/update/documentupdate.h>
+
+namespace proton
+{
+
+/**
+ * Context class for document update operations that acks operation when
+ * instance is destroyed. Typically a shared pointer to an instance is
+ * passed around to multiple worker threads that performs portions of
+ * a larger task before dropping the shared pointer, triggering the
+ * ack when all worker threads have completed.
+ */
+class UpdateDoneContext : public OperationDoneContext
+{
+    document::DocumentUpdate::SP _upd;
+public:
+    UpdateDoneContext(std::unique_ptr<FeedToken> token,
+                      const FeedOperation::Type opType,
+                      PerDocTypeFeedMetrics &metrics,
+                      const document::DocumentUpdate::SP &upd);
+
+    virtual ~UpdateDoneContext();
+
+    const document::DocumentUpdate &getUpdate() { return *_upd; }
+};
+
+
+}  // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp
new file mode 100644
index 00000000000..885828b649f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "visibilityhandler.h"
+#include <vespa/vespalib/util/closuretask.h>
+
+using vespalib::makeTask;
+using vespalib::makeClosure;
+
+namespace proton {
+
+VisibilityHandler::VisibilityHandler(const IGetSerialNum & serial,
+                                     IThreadingService &writeService,
+                                     const FeedViewHolder & feedView)
+    : _serial(serial),
+      _writeService(writeService),
+      _feedView(feedView),
+      _visibilityDelay(0),
+      _lastCommitSerialNum(0),
+      _lock()
+{
+}
+
+void VisibilityHandler::commit()
+{
+    if (_visibilityDelay != 0) {
+        if (_writeService.master().isCurrentThread()) {
+            performCommit();
+        } else {
+            LockGuard guard(_lock);
+            startCommit(guard);
+        }
+    }
+}
+
+void VisibilityHandler::commitAndWait()
+{
+    if (_visibilityDelay != 0) {
+        if (_writeService.master().isCurrentThread()) {
+            performCommit();
+        } else {
+            LockGuard guard(_lock);
+            if (startCommit(guard)) {
+                _writeService.master().sync();
+            }
+        }
+    }
+    // Always sync attribute writer threads so attribute vectors are
+    // properly updated when document retriver rebuilds document
+    _writeService.attributeFieldWriter().sync();
+}
+
+bool VisibilityHandler::startCommit(const LockGuard & unused)
+{
+    (void) unused;
+    SerialNum current = _serial.getSerialNum();
+    if (current > _lastCommitSerialNum) {
+        _writeService.master().execute(makeTask(makeClosure(this,
+             &VisibilityHandler::performCommit)));
+        return true;
+    }
+    return false;
+}
+
+void VisibilityHandler::performCommit()
+{
+    // Called by master thread
+    SerialNum current = _serial.getSerialNum();
+    if (current > _lastCommitSerialNum) {
+        IFeedView::SP feedView(_feedView.get());
+        feedView->forceCommit(current);
+        _lastCommitSerialNum = current;
+    }
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.h b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.h
new file mode 100644
index 00000000000..ddecfb34db5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.h
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/server/ifeedview.h>
+#include <vespa/searchcore/proton/server/icommitable.h>
+#include <vespa/searchcore/proton/server/igetserialnum.h>
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/vespalib/util/varholder.h>
+
+namespace proton {
+
+/**
+ * Handle commit of changes withing the allowance of visibilitydelay.
+ * It will both handle background commit jobs and the necessary commit and wait for sequencing.
+ **/
+class VisibilityHandler : public ICommitable
+{
+    typedef vespalib::LockGuard       LockGuard;
+    typedef fastos::TimeStamp         TimeStamp;
+    using IThreadingService = searchcorespi::index::IThreadingService;
+    typedef vespalib::ThreadExecutor  ThreadExecutor;
+    typedef vespalib::Lock            Lock;
+    typedef vespalib::VarHolder<IFeedView::SP> FeedViewHolder;
+public:
+    typedef search::SerialNum         SerialNum;
+    VisibilityHandler(const IGetSerialNum &serial,
+                      IThreadingService &threadingService,
+                      const FeedViewHolder &feedView);
+    void setVisibilityDelay(TimeStamp visibilityDelay) { _visibilityDelay = visibilityDelay; }
+    TimeStamp getVisibilityDelay() const { return _visibilityDelay; } 
+    void commit() override;
+    virtual void commitAndWait() override;
+private:
+    bool startCommit(const LockGuard & unused);
+    void performCommit();
+    const IGetSerialNum  & _serial;
+    IThreadingService    & _writeService;
+    const FeedViewHolder & _feedView;
+    TimeStamp              _visibilityDelay;
+    SerialNum              _lastCommitSerialNum;
+    Lock                   _lock;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp b/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp
new file mode 100644
index 00000000000..dff3ff57375
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "wipe_old_removed_fields_job.h"
+#include <vespa/fastos/timestamp.h>
+
+using fastos::ClockSystem;
+using fastos::TimeStamp;
+
+namespace proton {
+
+WipeOldRemovedFieldsJob::WipeOldRemovedFieldsJob(IWipeOldRemovedFieldsHandler &handler,
+                                                 const DocumentDBWipeOldRemovedFieldsConfig &config)
+    : IMaintenanceJob("wipe_old_removed_fields", config.getInterval(), config.getInterval()),
+      _handler(handler),
+      _ageLimitSeconds(config.getAge())
+{
+}
+
+bool
+WipeOldRemovedFieldsJob::run()
+{
+    TimeStamp wipeTimeLimit = TimeStamp(ClockSystem::now()) -
+        TimeStamp(int64_t(_ageLimitSeconds) * TimeStamp::SEC);
+    _handler.wipeOldRemovedFields(wipeTimeLimit);
+    return true;
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h b/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h
new file mode 100644
index 00000000000..0a2b5fe3b6f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "document_db_maintenance_config.h"
+#include "i_maintenance_job.h"
+#include "iwipeoldremovedfieldshandler.h"
+
+namespace proton {
+
+/**
+ * Job that regularly wipes old removed fields from a document database.
+ */
+class WipeOldRemovedFieldsJob : public IMaintenanceJob
+{
+private:
+    IWipeOldRemovedFieldsHandler &_handler;
+    const double                  _ageLimitSeconds;
+
+public:
+    WipeOldRemovedFieldsJob(IWipeOldRemovedFieldsHandler &handler,
+                            const DocumentDBWipeOldRemovedFieldsConfig &config);
+
+    // Implements IMaintenanceJob
+    virtual bool run();
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/.gitignore b/searchcore/src/vespa/searchcore/proton/summaryengine/.gitignore
new file mode 100644
index 00000000000..5dae353d999
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/.gitignore
@@ -0,0 +1,2 @@
+.depend
+Makefile
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/summaryengine/CMakeLists.txt
new file mode 100644
index 00000000000..417eb9aa57a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_summaryengine STATIC
+    SOURCES
+    summaryengine.cpp
+    docsum_by_slime.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/OWNERS b/searchcore/src/vespa/searchcore/proton/summaryengine/OWNERS
new file mode 100644
index 00000000000..7ae1acb1be9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/OWNERS
@@ -0,0 +1 @@
+geirst
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp
new file mode 100644
index 00000000000..e3e1cf89c88
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "docsum_by_slime.h"
+#include <vespa/document/util/compressor.h>
+#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
+
+#include <vespa/log/log.h>
+
+LOG_SETUP(".proton.summaryengine.docsum_by_slime");
+
+namespace proton {
+
+using search::engine::DocsumRequest;
+using search::engine::DocsumReply;
+using vespalib::slime::Inspector;
+using vespalib::slime::Cursor;
+using vespalib::slime::ObjectSymbolInserter;
+using vespalib::slime::Memory;
+using vespalib::slime::Symbol;
+using vespalib::slime::BinaryFormat;
+using vespalib::slime::ArrayTraverser;
+using vespalib::slime::SimpleBuffer;
+using vespalib::DataBuffer;
+using vespalib::ConstBufferRef;
+using document::CompressionConfig;
+
+namespace {
+
+Memory SUMMARYCLASS("class");
+Memory GIDS("gids");
+Memory DOCSUM("docsum");
+Memory DOCSUMS("docsums");
+
+class GidTraverser : public ArrayTraverser
+{
+public:
+    GidTraverser(std::vector<DocsumRequest::Hit> & hits) : _hits(hits) { }
+    void entry(size_t idx, const Inspector &inspector) override {
+        (void) idx;
+        Memory data(inspector.asData());
+        assert(data.size >= document::GlobalId::LENGTH);
+        _hits.emplace_back(document::GlobalId(data.data));
+    }
+private:
+    std::vector<DocsumRequest::Hit> & _hits;
+};
+
+CompressionConfig
+getCompressionConfig()
+{
+    using search::fs4transport::FS4PersistentPacketStreamer;
+    const FS4PersistentPacketStreamer & streamer = FS4PersistentPacketStreamer::Instance;
+    return CompressionConfig(streamer.getCompressionType(), streamer.getCompressionLevel(), 80, streamer.getCompressionLimit());
+}
+
+}
+
+DocsumRequest::UP
+DocsumBySlime::slimeToRequest(const Inspector & request)
+{
+    DocsumRequest::UP docsumRequest(std::make_unique<DocsumRequest>(true));
+
+    docsumRequest->_flags = search::fs4transport::GDFLAG_ALLOW_SLIME;
+    docsumRequest->resultClassName = request[SUMMARYCLASS].asString().make_string();
+    Inspector & gids = request[GIDS];
+    docsumRequest->hits.reserve(gids.entries());
+    GidTraverser gidFiller(docsumRequest->hits);
+    gids.traverse(gidFiller);
+
+    return docsumRequest;
+}
+
+vespalib::Slime::UP
+DocsumBySlime::getDocsums(const Inspector & req)
+{
+    DocsumReply::UP reply = _docsumServer.getDocsums(slimeToRequest(req));
+    if (reply && reply->_root) {
+        return std::move(reply->_root);
+    } else {
+        LOG(warning, "got <null> docsum reply from back-end");
+    }
+    return std::make_unique<vespalib::Slime>();
+}
+
+DocsumByRPC::DocsumByRPC(DocsumBySlime & slimeDocsumServer) :
+    _slimeDocsumServer(slimeDocsumServer)
+{
+}
+
+void
+DocsumByRPC::getDocsums(FRT_RPCRequest & req)
+{
+    FRT_Values &arg = *req.GetParams();
+    uint8_t encoding = arg[0]._intval8;
+    uint32_t uncompressedSize = arg[1]._intval32;
+    DataBuffer uncompressed(arg[2]._data._buf, arg[2]._data._len);
+    ConstBufferRef blob(arg[2]._data._buf, arg[2]._data._len);
+    document::decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true);
+    assert(uncompressedSize == uncompressed.getDataLen());
+    vespalib::Slime summariesToGet;
+    BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), summariesToGet);
+
+    vespalib::Slime::UP summaries = _slimeDocsumServer.getDocsums(summariesToGet.get());
+    assert(summaries);  // Mandatory, not optional.
+
+    search::RawBuf rbuf(4096);
+    search::SlimeOutputRawBufAdapter output(rbuf);
+    BinaryFormat::encode(*summaries, output);
+    ConstBufferRef buf(rbuf.GetDrainPos(), rbuf.GetUsedLen());
+    DataBuffer compressed(rbuf.GetWritableDrainPos(0), rbuf.GetUsedLen());
+    CompressionConfig::Type type = document::compress(getCompressionConfig(), buf, compressed, true);
+
+    FRT_Values &ret = *req.GetReturn();
+    ret.AddInt8(type);
+    ret.AddInt32(buf.size());
+    ret.AddData(compressed.getData(), compressed.getDataLen());
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.h b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.h
new file mode 100644
index 00000000000..920facf76a2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/engine/docsumapi.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/fnet/frt/frt.h>
+
+namespace proton {
+
+class DocsumBySlime {
+    using DocsumServer = search::engine::DocsumServer;
+    using DocsumRequest = search::engine::DocsumRequest;
+    using DocsumReply = search::engine::DocsumReply;
+    using Inspector = vespalib::slime::Inspector;
+public:
+    typedef std::unique_ptr<DocsumBySlime> UP;
+    DocsumBySlime(DocsumServer & docsumServer) : _docsumServer(docsumServer) { }
+    vespalib::Slime::UP getDocsums(const Inspector & req);
+    static DocsumRequest::UP slimeToRequest(const Inspector & req);
+private:
+    DocsumServer & _docsumServer;
+};
+
+class DocsumByRPC
+{
+public:
+    DocsumByRPC(DocsumBySlime & slimeDocsumServer);
+    void getDocsums(FRT_RPCRequest & req);
+private:
+    DocsumBySlime & _slimeDocsumServer;
+};
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h b/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h
new file mode 100644
index 00000000000..b8fe2b658ff
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/utility.hpp>
+#include <vespa/searchlib/engine/docsumreply.h>
+#include <vespa/searchlib/engine/docsumrequest.h>
+#include <vespa/searchlib/engine/searchreply.h>
+#include <vespa/searchlib/engine/searchrequest.h>
+#include <vespa/vespalib/util/thread_bundle.h>
+
+namespace proton {
+
+/**
+ * This interface describes a sync summary operation handler. It is
+ * implemented by the DocumentDB class, and used by the SummaryEngine
+ * class to delegate operations to the appropriate db.
+ */
+class ISearchHandler {
+public:
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::unique_ptr<ISearchHandler> UP;
+    typedef std::shared_ptr<ISearchHandler> SP;
+
+    /**
+     * Virtual destructor to allow inheritance.
+     */
+    virtual ~ISearchHandler() { }
+
+    /**
+     * @return Use the request and produce the document summary result.
+     */
+    virtual search::engine::DocsumReply::UP getDocsums(const search::engine::DocsumRequest & request) = 0;
+
+    virtual search::engine::SearchReply::UP match(
+            const ISearchHandler::SP &self,
+            const search::engine::SearchRequest &req,
+            vespalib::ThreadBundle &threadBundle) const = 0;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp
new file mode 100644
index 00000000000..8f56aa90a98
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp
@@ -0,0 +1,133 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.summaryengine.summaryengine");
+
+#include "summaryengine.h"
+#include <vespa/vespalib/util/exceptions.h>
+
+using namespace search::engine;
+using namespace proton;
+
+namespace {
+
+class DocsumTask : public vespalib::Executor::Task {
+private:
+    SummaryEngine       & _engine;
+    DocsumClient        & _client;
+    DocsumRequest::Source _request;
+
+public:
+    DocsumTask(SummaryEngine & engine,
+               DocsumRequest::Source request,
+               DocsumClient & client)
+        : _engine(engine),
+          _client(client),
+          _request(std::move(request))
+    {
+    }
+
+    void run() {
+        _client.getDocsumsDone(_engine.getDocsums(_request.release()));
+    }
+};
+
+} // namespace anonymous
+
+namespace proton {
+
+SummaryEngine::SummaryEngine(size_t numThreads)
+    : _lock(),
+      _closed(false),
+      _handlers(),
+      _executor(numThreads, 128 * 1024)
+{
+    // empty
+}
+
+SummaryEngine::~SummaryEngine()
+{
+    _executor.shutdown();
+}
+
+void
+SummaryEngine::close()
+{
+    LOG(debug, "Closing summary engine");
+    {
+        vespalib::LockGuard guard(_lock);
+        _closed = true;
+    }
+    LOG(debug, "Handshaking with task manager");
+    _executor.sync();
+}
+
+ISearchHandler::SP
+SummaryEngine::putSearchHandler(const DocTypeName &docTypeName,
+                                 const ISearchHandler::SP & searchHandler)
+{
+    vespalib::LockGuard guard(_lock);
+    return _handlers.putHandler(docTypeName, searchHandler);
+}
+
+ISearchHandler::SP
+SummaryEngine::getSearchHandler(const DocTypeName &docTypeName)
+{
+    return _handlers.getHandler(docTypeName);
+}
+
+ISearchHandler::SP
+SummaryEngine::removeSearchHandler(const DocTypeName &docTypeName)
+{
+    vespalib::LockGuard guard(_lock);
+    return _handlers.removeHandler(docTypeName);
+}
+
+DocsumReply::UP
+SummaryEngine::getDocsums(DocsumRequest::Source request, DocsumClient & client)
+{
+    if (_closed) {
+        LOG(warning, "Receiving docsumrequest after engine has been shutdown");
+        DocsumReply::UP ret(new DocsumReply());
+
+        // TODO: Notify closed.
+
+        return ret;
+    }
+    vespalib::Executor::Task::UP task(new DocsumTask(*this, std::move(request), client));
+    _executor.execute(std::move(task));
+    return DocsumReply::UP();
+}
+
+DocsumReply::UP
+SummaryEngine::getDocsums(DocsumRequest::UP req)
+{
+    DocsumReply::UP reply;
+    reply.reset(new DocsumReply());
+
+    if (req) {
+        ISearchHandler::SP searchHandler;
+        { // try to find the summary handler corresponding to the specified search doc type
+            vespalib::LockGuard guard(_lock);
+            DocTypeName docTypeName(*req);
+            searchHandler = _handlers.getHandler(docTypeName);
+        }
+        if (searchHandler.get() != NULL) {
+            reply = searchHandler->getDocsums(*req);
+        } else {
+            vespalib::Sequence<ISearchHandler*>::UP snapshot;
+            {
+                vespalib::LockGuard guard(_lock);
+                snapshot = _handlers.snapshot();
+            }
+            if (snapshot->valid()) {
+                reply = snapshot->get()->getDocsums(*req); // use the first handler
+            }
+        }
+    }
+    reply->request = std::move(req);
+    return reply;
+}
+
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h
new file mode 100644
index 00000000000..888fbc82ec8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/common/handlermap.hpp>
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
+#include <vespa/searchlib/engine/docsumapi.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+
+namespace proton {
+
+class SummaryEngine : public boost::noncopyable,
+                      public search::engine::DocsumServer
+{
+private:
+    using DocsumReply = search::engine::DocsumReply;
+    using DocsumRequest = search::engine::DocsumRequest;
+    using DocsumClient = search::engine::DocsumClient;
+
+    vespalib::Lock                _lock;
+    bool                          _closed;
+    HandlerMap<ISearchHandler>    _handlers;
+    vespalib::ThreadStackExecutor _executor;
+
+public:
+    /**
+     * Convenience typedefs.
+     */
+    typedef std::unique_ptr<SummaryEngine> UP;
+    typedef std::shared_ptr<SummaryEngine> SP;
+
+    /**
+     * Constructs a new summary engine. This does the necessary setup of the
+     * internal structures, without actually starting any threads. Before
+     * calling start(), you should register handlers for all know document types
+     * using the putSearchHandler() method.
+     *
+     * @param numThreads Number of threads allocated for handling summary requests.
+     */
+    SummaryEngine(size_t numThreads);
+
+    /**
+     * Frees any allocated resources. This will also stop all internal threads
+     * and wait for them to finish. All pending docsum requests are deleted.
+     */
+    ~SummaryEngine();
+
+    /**
+     * Observe and reset internal executor stats
+     *
+     * @return executor stats
+     **/
+    vespalib::ThreadStackExecutor::Stats getExecutorStats() { return _executor.getStats(); }
+
+    /**
+     * Starts the underlying threads. This will throw a vespalib::Exception if
+     * it failed to start for any reason.
+     */
+    void start();
+
+    /**
+     * Closes the request handler interface. This will prevent any more data
+     * from entering this object, allowing you to flush all pending operations
+     * without having to safe-guard against input.
+     */
+    void close();
+
+    /**
+     * Registers a new summary handler for the given document type. If another
+     * handler was already registered under the same type, this method will
+     * return a pointer to that handler.
+     *
+     * @param docType        The document type to register a handler for.
+     * @param searchHandler The handler to register.
+     * @return The replaced handler, if any.
+     */
+    ISearchHandler::SP putSearchHandler(const DocTypeName &docTypeName, const ISearchHandler::SP &searchHandler);
+
+    /**
+     * Returns the summary handler for the given document type. If no handler was
+     * registered, this method returns an empty shared pointer.
+     *
+     * @param docType The document type whose handler to return.
+     * @return The registered handler, if any.
+     */
+    ISearchHandler::SP getSearchHandler(const DocTypeName &docTypeName);
+
+    /**
+     * Removes and returns the summary handler for the given document type. If no
+     * handler was registered, this method returns an empty shared pointer.
+     *
+     * @param docType The document type whose handler to remove.
+     * @return The removed handler, if any.
+     */
+    ISearchHandler::SP removeSearchHandler(const DocTypeName &docTypeName);
+
+    // Implements DocsumServer.
+    DocsumReply::UP getDocsums(DocsumRequest::Source request, DocsumClient & client) override;
+
+    /**
+     * Performs the given docsum request in the current thread and returns the reply.
+     *
+     * @param req    The docsum request to perform.
+     */
+    DocsumReply::UP getDocsums(DocsumRequest::UP req) override;
+};
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt
new file mode 100644
index 00000000000..10cc8e60b07
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_test STATIC
+    SOURCES
+    buckethandler.cpp
+    clusterstatehandler.cpp
+    userdocumentsbuilder.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/test/OWNERS b/searchcore/src/vespa/searchcore/proton/test/OWNERS
new file mode 100644
index 00000000000..7ae1acb1be9
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/OWNERS
@@ -0,0 +1 @@
+geirst
diff --git a/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h b/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h
new file mode 100644
index 00000000000..e3af44da9e8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+
+namespace proton {
+
+namespace test {
+
+struct AttributeUtils
+{
+    static void fillAttribute(const search::AttributeVector::SP &attr,
+                              uint32_t numDocs, int64_t value, uint64_t lastSyncToken) {
+        search::IntegerAttribute &ia = static_cast<search::IntegerAttribute &>(*attr);
+        ia.addDocs(numDocs);
+        for (uint32_t i = 1; i < ia.getNumDocs(); ++i) {
+            ia.update(i, value);
+        }
+        ia.commit(lastSyncToken, lastSyncToken);
+    }
+    static void fillAttribute(const search::AttributeVector::SP &attr,
+                              uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) {
+        search::IntegerAttribute &ia = static_cast<search::IntegerAttribute &>(*attr);
+        while (ia.getNumDocs() < to) {
+            uint32_t docId;
+            if (!ia.addDoc(docId)) { abort(); }
+        }
+        for (uint32_t i = from; i < to; ++i) {
+            ia.update(i, value);
+        }
+        ia.commit(lastSyncToken, lastSyncToken);
+    }
+    static search::attribute::Config getInt32Config() {
+        return search::attribute::Config(search::attribute::BasicType::INT32);
+    }
+    static search::attribute::Config getInt32ArrayConfig() {
+        return search::attribute::Config(search::attribute::BasicType::INT32,
+         search::attribute::CollectionType::ARRAY);
+    }
+    static search::attribute::Config getStringConfig() {
+        return search::attribute::Config(search::attribute::BasicType::STRING);
+    }
+    static search::attribute::Config getPredicateConfig() {
+        return search::attribute::Config(search::attribute::BasicType::PREDICATE);
+    }
+    static search::attribute::Config getTensorConfig() {
+        return search::attribute::Config(search::attribute::BasicType::TENSOR);
+    }
+};
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/attribute_vectors.h b/searchcore/src/vespa/searchcore/proton/test/attribute_vectors.h
new file mode 100644
index 00000000000..7010fef620d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/attribute_vectors.h
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "attribute_utils.h"
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
+
+namespace proton {
+
+namespace test {
+
+typedef search::SingleValueNumericAttribute<search::IntegerAttributeTemplate<int32_t> > Int32AttributeBase;
+
+struct Int32Attribute : public Int32AttributeBase
+{
+    Int32Attribute(const vespalib::string &name) :
+        Int32AttributeBase(name, AttributeUtils::getInt32Config())
+    {
+    }
+};
+
+} // namespace test
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h b/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h
new file mode 100644
index 00000000000..b2fc3ff6dbc
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "document.h"
+
+namespace proton {
+
+namespace test {
+
+/**
+ * Collection of documents contained in the same bucket.
+ */
+class BucketDocuments
+{
+private:
+    DocumentVector _docs;
+public:
+    BucketDocuments()
+        : _docs()
+    {
+    }
+    document::BucketId getBucket() const {
+        if (!_docs.empty()) {
+            return _docs.back().getBucket();
+        }
+        return document::BucketId();
+    }
+    const DocumentVector &getDocs() const { return _docs; }
+    DocumentVector getGidOrderDocs() const {
+        DocumentVector retval = _docs;
+        std::sort(retval.begin(), retval.end(), DocumentGidOrderCmp());
+        return retval;
+    }
+    void addDoc(const Document &doc) {
+        if (!_docs.empty()) {
+            assert(_docs.back().getBucket() == doc.getBucket());
+        }
+        _docs.push_back(doc);
+    }
+};
+
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/test/buckethandler.cpp
new file mode 100644
index 00000000000..f7b8ce691da
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/buckethandler.cpp
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+
+#include "buckethandler.h"
+
+
+namespace proton
+{
+
+namespace test
+{
+
+
+BucketHandler::BucketHandler()
+    : IBucketStateChangedNotifier(),
+      _handlers()
+{
+}
+
+
+BucketHandler::~BucketHandler()
+{
+    assert(_handlers.empty());
+}
+
+
+void
+BucketHandler::addBucketStateChangedHandler(IBucketStateChangedHandler *handler)
+{
+    _handlers.insert(handler);
+}
+
+
+void
+BucketHandler::removeBucketStateChangedHandler(IBucketStateChangedHandler *
+                                               handler)
+{
+    _handlers.erase(handler);
+}
+
+
+void
+BucketHandler::notifyBucketStateChanged(const document::BucketId &bucketId,
+                                        storage::spi::BucketInfo::ActiveState
+                                        newState)
+{
+    for (auto &handler : _handlers) {
+        handler->notifyBucketStateChanged(bucketId, newState);
+    }
+}
+
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/buckethandler.h b/searchcore/src/vespa/searchcore/proton/test/buckethandler.h
new file mode 100644
index 00000000000..4b256bb1443
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/buckethandler.h
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/server/ibucketstatechangednotifier.h>
+#include <vespa/searchcore/proton/server/ibucketstatechangedhandler.h>
+#include <set>
+
+namespace proton
+{
+
+namespace test
+{
+
+/**
+ * Test bucket handler that forwards bucket state change notifications
+ * as appropriate.
+ */
+class BucketHandler : public IBucketStateChangedNotifier
+{
+    std::set<IBucketStateChangedHandler *> _handlers;
+public:
+    BucketHandler();
+
+    virtual
+    ~BucketHandler();
+
+    virtual void
+    addBucketStateChangedHandler(IBucketStateChangedHandler *handler);
+
+    virtual void
+    removeBucketStateChangedHandler(IBucketStateChangedHandler *handler);
+
+    void
+    notifyBucketStateChanged(const document::BucketId &bucketId,
+                             storage::spi::BucketInfo::ActiveState newState);
+};
+
+
+} // namespace test
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h
new file mode 100644
index 00000000000..b3fc146d9c2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/searchcore/proton/server/ibucketstatecalculator.h>
+
+namespace proton {
+
+namespace test {
+
+typedef document::BucketId::List BucketIdVector;
+typedef std::set<document::BucketId>    BucketIdSet;
+
+class BucketStateCalculator : public IBucketStateCalculator
+{
+private:
+    BucketIdSet            _ready;
+    mutable BucketIdVector _asked;
+    bool                   _clusterUp;
+    bool                   _nodeUp;
+
+public:
+    typedef std::shared_ptr<BucketStateCalculator> SP;
+    BucketStateCalculator() :
+        _ready(),
+        _asked(),
+        _clusterUp(true),
+        _nodeUp(true)
+    {
+    }
+    BucketStateCalculator &addReady(const document::BucketId &bucket) {
+        _ready.insert(bucket);
+        return *this;
+    }
+    BucketStateCalculator &remReady(const document::BucketId &bucket) {
+        _ready.erase(bucket);
+        return *this;
+    }
+    BucketStateCalculator &setClusterUp(bool value) {
+        _clusterUp = value;
+        return *this;
+    }
+
+    BucketStateCalculator &
+    setNodeUp(bool value)
+    {
+        _nodeUp = value;
+        return *this;
+    }
+
+    const BucketIdVector &asked() const { return _asked; }
+    void resetAsked() { _asked.clear(); }
+
+    // Implements IBucketStateCalculator
+    virtual bool shouldBeReady(const document::BucketId &bucket) const {
+        _asked.push_back(bucket);
+        return _ready.count(bucket) == 1;
+    }
+    virtual bool clusterUp() const {
+        return _clusterUp;
+    }
+    
+    virtual bool
+    nodeUp(void) const
+    {
+        return _nodeUp;
+    }
+
+    virtual bool
+    nodeInitializing(void) const
+    {
+        return false;
+    }
+};
+
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.cpp b/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.cpp
new file mode 100644
index 00000000000..e241e92806a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.cpp
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+
+#include "clusterstatehandler.h"
+
+
+namespace proton
+{
+
+namespace test
+{
+
+
+ClusterStateHandler::ClusterStateHandler()
+    : IClusterStateChangedNotifier(),
+      _handlers()
+{
+}
+
+
+ClusterStateHandler::~ClusterStateHandler()
+{
+    assert(_handlers.empty());
+}
+
+
+void
+ClusterStateHandler::
+addClusterStateChangedHandler(IClusterStateChangedHandler *handler)
+{
+    _handlers.insert(handler);
+}
+
+
+void
+ClusterStateHandler::
+removeClusterStateChangedHandler(IClusterStateChangedHandler *handler)
+{
+    _handlers.erase(handler);
+}
+
+
+void
+ClusterStateHandler::
+notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc)
+{
+    for (auto &handler : _handlers) {
+        handler->notifyClusterStateChanged(newCalc);
+    }
+}
+
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.h b/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.h
new file mode 100644
index 00000000000..78e1c73b35a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/server/iclusterstatechangednotifier.h>
+#include <vespa/searchcore/proton/server/iclusterstatechangedhandler.h>
+#include <set>
+
+namespace proton
+{
+
+namespace test
+{
+
+/**
+ * Test cluster state handler that forwards cluster state change
+ * notifications as appropriate.
+ */
+class ClusterStateHandler : public IClusterStateChangedNotifier
+{
+    std::set<IClusterStateChangedHandler *> _handlers;
+public:
+    ClusterStateHandler();
+
+    virtual
+    ~ClusterStateHandler();
+
+    virtual void
+    addClusterStateChangedHandler(IClusterStateChangedHandler *handler);
+
+    virtual void
+    removeClusterStateChangedHandler(IClusterStateChangedHandler *handler);
+
+    void
+    notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc);
+};
+
+
+} // namespace test
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/test/directory_handler.h b/searchcore/src/vespa/searchcore/proton/test/directory_handler.h
new file mode 100644
index 00000000000..11d41e99809
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/directory_handler.h
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+namespace test {
+
+class DirectoryHandler
+{
+private:
+    vespalib::string _mkdir;
+    vespalib::string _rmdir;
+    bool             _cleanup;
+
+public:
+    DirectoryHandler(const vespalib::string &mkdir)
+        : _mkdir(mkdir),
+          _rmdir(mkdir),
+          _cleanup(true)
+    {
+        vespalib::mkdir(_mkdir);
+    }
+    DirectoryHandler(const vespalib::string &mkdir,
+                     const vespalib::string &rmdir)
+        : _mkdir(mkdir),
+          _rmdir(rmdir),
+          _cleanup(true)
+    {
+        vespalib::mkdir(_mkdir);
+    }
+    ~DirectoryHandler() {
+        if (_cleanup) {
+            vespalib::rmdir(_rmdir, true);
+        }
+    }
+    void cleanup(bool v) { _cleanup = v; }
+};
+
+}
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/document.h b/searchcore/src/vespa/searchcore/proton/test/document.h
new file mode 100644
index 00000000000..aacf67c45bd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/document.h
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <persistence/spi/types.h>
+#include <vespa/searchlib/query/base.h>
+
+namespace proton {
+
+namespace test {
+
+/**
+ * Representation of a test document.
+ */
+class Document
+{
+private:
+    document::Document::SP  _doc;
+    search::DocumentIdT     _lid;
+    storage::spi::Timestamp _tstamp;
+    uint32_t                _numUsedBits;
+public:
+    Document(document::Document::SP doc,
+             search::DocumentIdT lid,
+             storage::spi::Timestamp tstamp,
+             uint32_t numUsedBits = 8u)
+        : _doc(doc),
+          _lid(lid),
+          _tstamp(tstamp),
+          _numUsedBits(numUsedBits)
+    {
+    }
+    const document::Document::SP &getDoc() const { return _doc; }
+    const document::DocumentId &getDocId() const { return _doc->getId(); }
+    const document::GlobalId &getGid() const { return getDocId().getGlobalId(); }
+    document::BucketId getBucket() const {
+        document::BucketId retval = getGid().convertToBucketId();
+        retval.setUsedBits(_numUsedBits);
+        return retval;
+    }
+    search::DocumentIdT getLid() const { return _lid; }
+    storage::spi::Timestamp getTimestamp() const { return _tstamp; }
+};
+
+typedef std::vector<Document> DocumentVector;
+
+struct DocumentGidOrderCmp
+{
+    bool operator()(const Document &lhs, const Document &rhs) const {
+        document::GlobalId::BucketOrderCmp cmp;
+        return cmp(lhs.getGid(), rhs.getGid());
+    }
+};
+
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_context_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_context_observer.h
new file mode 100644
index 00000000000..e3d70c9a926
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_context_observer.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "document_meta_store_observer.h"
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h>
+
+namespace proton {
+namespace test {
+
+struct DocumentMetaStoreContextObserver : public IDocumentMetaStoreContext
+{
+    typedef std::shared_ptr<DocumentMetaStoreContextObserver> SP;
+
+    IDocumentMetaStoreContext &_context;
+    DocumentMetaStoreObserver::SP _observer;
+
+    DocumentMetaStoreContextObserver(IDocumentMetaStoreContext &context)
+        : _context(context),
+          _observer(std::make_shared<DocumentMetaStoreObserver>(_context.get()))
+    {
+    }
+    const DocumentMetaStoreObserver &getObserver() const {
+        return * dynamic_cast<const DocumentMetaStoreObserver *>(_observer.get());
+    }
+
+    // Implements IDocumentMetaStoreContext
+    proton::IDocumentMetaStore::SP getSP() const override { return _observer; }
+    proton::IDocumentMetaStore &     get()       override { return *_observer; }
+    IReadGuard::UP          getReadGuard() const override { return _context.getReadGuard(); }
+    void               constructFreeList()       override { return _context.constructFreeList(); }
+};
+
+} // namespace test
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h
new file mode 100644
index 00000000000..c2a461eb90b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h
@@ -0,0 +1,185 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
+
+namespace proton {
+namespace test {
+
+struct DocumentMetaStoreObserver : public IDocumentMetaStore
+{
+    IDocumentMetaStore &_store;
+    uint32_t _removeCompleteCnt;
+    DocId _removeCompleteLid;
+    DocId _compactLidSpaceLidLimit;
+    uint32_t _holdUnblockShrinkLidSpaceCnt;
+
+    DocumentMetaStoreObserver(IDocumentMetaStore &store)
+        : _store(store),
+          _removeCompleteCnt(0),
+          _removeCompleteLid(0),
+          _compactLidSpaceLidLimit(0),
+          _holdUnblockShrinkLidSpaceCnt(0)
+    {
+    }
+
+    /**
+     * Implements search::IDocumentMetaStore
+     **/
+    virtual bool getGid(DocId lid, GlobalId &gid) const override {
+        return _store.getGid(lid, gid);
+    }
+    virtual bool getLid(const GlobalId &gid, DocId &lid) const override {
+        return _store.getLid(gid, lid);
+    }
+    virtual search::DocumentMetaData getMetaData(const GlobalId &gid) const override {
+        return _store.getMetaData(gid);
+    }
+    virtual void getMetaData(const BucketId &bucketId,
+                             search::DocumentMetaData::Vector &result) const override {
+        _store.getMetaData(bucketId, result);
+    }
+    virtual search::LidUsageStats getLidUsageStats() const override {
+        return _store.getLidUsageStats();
+    }
+    virtual search::queryeval::Blueprint::UP createBlackListBlueprint() const override {
+        return _store.createBlackListBlueprint();
+    }
+
+
+    /**
+     * Implements documentmetastore::IStore.
+     */
+    virtual Result inspectExisting(const GlobalId &gid) const override {
+        return _store.inspectExisting(gid);
+    }
+    virtual Result inspect(const GlobalId &gid) override {
+        return _store.inspect(gid);
+    }
+    virtual Result put(const GlobalId &gid,
+                       const BucketId &bucketId,
+                       const Timestamp &timestamp,
+                       DocId lid) override {
+        return _store.put(gid, bucketId, timestamp, lid);
+    }
+    virtual bool updateMetaData(DocId lid,
+                                const BucketId &bucketId,
+                                const Timestamp &timestamp) override {
+        return _store.updateMetaData(lid, bucketId, timestamp);
+    }
+    virtual bool remove(DocId lid) override {
+        return _store.remove(lid);
+    }
+    virtual void removeComplete(DocId lid) override {
+        ++_removeCompleteCnt;
+        _removeCompleteLid = lid;
+        _store.removeComplete(lid);
+    }
+    virtual void move(DocId fromLid, DocId toLid) override {
+        _store.move(fromLid, toLid);
+    }
+    virtual bool validLid(DocId lid) const override {
+        return _store.validLid(lid);
+    }
+    virtual void removeBatch(const std::vector<DocId> &lidsToRemove,
+                             const DocId docIdLimit) override {
+        _store.removeBatch(lidsToRemove, docIdLimit);
+    }
+    virtual void removeBatchComplete(const std::vector<DocId> &lidsToRemove) override {
+        _store.removeBatchComplete(lidsToRemove);
+    }
+    virtual const RawDocumentMetaData &getRawMetaData(DocId lid) const override {
+        return _store.getRawMetaData(lid);
+    }
+
+    /**
+     * Implements documentmetastore::IBucketHandler.
+     */
+    virtual BucketDBOwner &getBucketDB() const override {
+        return _store.getBucketDB();
+    }
+    virtual bucketdb::BucketDeltaPair
+    handleSplit(const bucketdb::SplitBucketSession &session) override {
+        return _store.handleSplit(session);
+    }
+    virtual bucketdb::BucketDeltaPair
+    handleJoin(const bucketdb::JoinBucketsSession &session) override {
+        return _store.handleJoin(session);
+    }
+    virtual void updateActiveLids(const BucketId &bucketId, bool active) override {
+        _store.updateActiveLids(bucketId, active);
+    }
+    virtual void setBucketState(const BucketId &bucketId, bool active) override {
+        _store.setBucketState(bucketId, active);
+    }
+    virtual void populateActiveBuckets(const BucketId::List &buckets) override {
+        _store.populateActiveBuckets(buckets);
+    }
+
+
+    /**
+     * Implements proton::IDocumentMetaStore
+     */
+    virtual void constructFreeList() override {
+        _store.constructFreeList();
+    }
+    virtual Iterator begin() const override {
+        return _store.begin();
+    }
+    virtual Iterator lowerBound(const BucketId &bucketId) const override {
+        return _store.lowerBound(bucketId);
+    }
+    virtual Iterator upperBound(const BucketId &bucketId) const override {
+        return _store.upperBound(bucketId);
+    }
+    virtual Iterator lowerBound(const GlobalId &gid) const override {
+        return _store.lowerBound(gid);
+    }
+    virtual Iterator upperBound(const GlobalId &gid) const override {
+        return _store.upperBound(gid);
+    }
+    virtual void getLids(const BucketId &bucketId, std::vector<DocId> &lids) override {
+        _store.getLids(bucketId, lids);
+    }
+    virtual DocId getNumUsedLids() const override {
+        return _store.getNumUsedLids();
+    }
+    virtual DocId getNumActiveLids() const override {
+        return _store.getNumActiveLids();
+    }
+    virtual search::AttributeGuard getActiveLidsGuard() const override {
+        return _store.getActiveLidsGuard();
+    }
+    virtual bool getFreeListActive() const override {
+        return _store.getFreeListActive();
+    }
+    virtual void compactLidSpace(DocId wantedLidLimit) override {
+        _compactLidSpaceLidLimit = wantedLidLimit;
+        _store.compactLidSpace(wantedLidLimit);
+    }
+    virtual void holdUnblockShrinkLidSpace() override {
+        ++_holdUnblockShrinkLidSpaceCnt;
+        _store.holdUnblockShrinkLidSpace();
+    }
+    virtual void commit(search::SerialNum firstSerialNum,
+                        search::SerialNum lastSerialNum) override {
+        _store.commit(firstSerialNum, lastSerialNum);
+    }
+    virtual DocId getCommittedDocIdLimit() const override {
+        return _store.getCommittedDocIdLimit();
+    }
+    virtual void removeAllOldGenerations() override {
+        _store.removeAllOldGenerations();
+    }
+    virtual bool canShrinkLidSpace() const override {
+        return _store.canShrinkLidSpace();
+    }
+    virtual search::SerialNum getLastSerialNum() const override {
+        return _store.getLastSerialNum();
+    }
+};
+
+}
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h
new file mode 100644
index 00000000000..6b165649f1a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchlib/docstore/cachestats.h>
+#include <vespa/searchlib/docstore/idocumentstore.h>
+
+namespace proton {
+
+namespace test {
+
+struct DummyDocumentStore : public search::IDocumentStore
+{
+    vespalib::string _baseDir;
+
+    DummyDocumentStore()
+        : _baseDir("")
+    {}
+    DummyDocumentStore(const vespalib::string &baseDir)
+        : _baseDir(baseDir)
+    {}
+    virtual document::Document::UP read(search::DocumentIdT,
+                                        const document::DocumentTypeRepo &) const {
+        return document::Document::UP();
+    }
+    virtual void write(uint64_t, const document::Document &, search::DocumentIdT) {}
+    virtual void remove(uint64_t, search::DocumentIdT) {}
+    virtual void flush(uint64_t) {}
+    virtual uint64_t initFlush(uint64_t) { return 0; }
+    virtual void compact(uint64_t) {}
+    virtual uint64_t lastSyncToken() const { return 0; }
+    virtual uint64_t tentativeLastSyncToken() const override { return 0; }
+    virtual fastos::TimeStamp getLastFlushTime() const { return fastos::TimeStamp(); }
+    virtual uint64_t nextId() const { return 0; }
+    virtual size_t memoryUsed() const { return 0; }
+    virtual size_t memoryMeta() const { return 0; }
+    virtual size_t getDiskFootprint() const { return 0; }
+    virtual size_t getDiskBloat() const { return 0; }
+    virtual size_t getMaxCompactGain() const { return getDiskBloat(); }
+    virtual search::CacheStats getCacheStats() const { return search::CacheStats(); }
+    virtual const vespalib::string &getBaseDir() const { return _baseDir; }
+    virtual void accept(search::IDocumentStoreReadVisitor &,
+                        search::IDocumentStoreVisitorProgress &,
+                        const document::DocumentTypeRepo &) {}
+
+    virtual void accept(search::IDocumentStoreRewriteVisitor &,
+                        search::IDocumentStoreVisitorProgress &,
+                        const document::DocumentTypeRepo &) {}
+
+    virtual double
+    getVisitCost() const
+    {
+        return 1.0;
+    }
+    virtual search::DataStoreStorageStats getStorageStats() const override {
+        return search::DataStoreStorageStats(0, 0, 0.0, 0, 0);
+    }
+    virtual std::vector<search::DataStoreFileChunkStats>
+    getFileChunkStats() const override {
+        std::vector<search::DataStoreFileChunkStats> result;
+        return result;
+    }
+};
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
new file mode 100644
index 00000000000..454e0dc01fb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/server/idocumentsubdb.h>
+#include <vespa/searchcore/proton/server/executorthreadingservice.h>
+
+namespace proton {
+
+namespace test {
+
+struct DummyDocumentSubDb : public IDocumentSubDB
+{
+    uint32_t                 _subDbId;
+    DocumentMetaStoreContext _metaStoreCtx;
+    ISummaryManager::SP      _summaryManager;
+    IIndexManager::SP        _indexManager;
+    ISummaryAdapter::SP      _summaryAdapter;
+    IIndexWriter::SP         _indexWriter;
+    std::unique_ptr<ExecutorThreadingService> _writeService;
+
+    DummyDocumentSubDb(std::shared_ptr<BucketDBOwner> bucketDB,
+                       uint32_t subDbId)
+        : _subDbId(subDbId),
+          _metaStoreCtx(bucketDB),
+          _summaryManager(),
+          _indexManager(),
+          _summaryAdapter(),
+          _indexWriter(),
+          _writeService(std::make_unique<ExecutorThreadingService>(1))
+    {
+    }
+    void close() override { }
+    virtual uint32_t getSubDbId() const override { return _subDbId; }
+    virtual vespalib::string getName() const override { return "dummysubdb"; }
+    virtual DocumentSubDbInitializer::UP
+    createInitializer(const DocumentDBConfig &,
+                      SerialNum,
+                      const search::index::Schema::SP &,
+                      const vespa::config::search::core::ProtonConfig::
+                      Summary &,
+                      const vespa::config::search::core::
+                      ProtonConfig::Index &) const override {
+        return std::make_unique<DocumentSubDbInitializer>
+            (const_cast<DummyDocumentSubDb &>(*this),
+             _writeService->master());
+    }
+    virtual void setup(const DocumentSubDbInitializerResult &) override {}
+    virtual void initViews(const DocumentDBConfig &,
+                           const proton::matching::SessionManager::SP &) override {}
+    virtual IReprocessingTask::List applyConfig(const DocumentDBConfig &,
+                                                const DocumentDBConfig &,
+                                                SerialNum,
+                                                const ReconfigParams) override {
+        return IReprocessingTask::List();
+    }
+    virtual ISearchHandler::SP getSearchView() const override { return ISearchHandler::SP(); }
+    virtual IFeedView::SP getFeedView() const override { return IFeedView::SP(); }
+    virtual void clearViews() override {}
+    virtual const ISummaryManager::SP &getSummaryManager() const override { return _summaryManager; }
+    virtual proton::IAttributeManager::SP getAttributeManager() const override {
+        return proton::IAttributeManager::SP();
+    }
+    virtual const IIndexManager::SP &getIndexManager() const override { return _indexManager; }
+    virtual const ISummaryAdapter::SP &getSummaryAdapter() const override { return _summaryAdapter; }
+    virtual const IIndexWriter::SP &getIndexWriter() const override { return _indexWriter; }
+    virtual IDocumentMetaStoreContext &getDocumentMetaStoreContext() override { return _metaStoreCtx; }
+    virtual IFlushTarget::List getFlushTargets() override { return IFlushTarget::List(); }
+    virtual size_t getNumDocs() const override { return 0; }
+    virtual size_t getNumActiveDocs() const override { return 0; }
+    virtual bool hasDocument(const document::DocumentId &) override { return false; }
+    virtual void onReplayDone() override {}
+    virtual void onReprocessDone(SerialNum) override { }
+    virtual SerialNum getOldestFlushedSerial() override { return 0; }
+    virtual SerialNum getNewestFlushedSerial() override { return 0; }
+    virtual void wipeHistory(SerialNum,
+                             const search::index::Schema &,
+                             const search::index::Schema &) override {}
+    virtual void setIndexSchema(const search::index::Schema::SP &,
+                                const search::index::Schema::SP &) override {}
+    virtual search::SearchableStats getSearchableStats(void) const override {
+        return search::SearchableStats();
+    }
+    virtual IDocumentRetriever::UP getDocumentRetriever() override {
+        return IDocumentRetriever::UP();
+    }
+    virtual matching::MatchingStats getMatcherStats(const vespalib::string &) const override {
+        return matching::MatchingStats();
+    }
+
+};
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h
new file mode 100644
index 00000000000..a86cbbbc5f6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/server/ifeedview.h>
+
+namespace proton {
+
+namespace test {
+
+struct DummyFeedView : public IFeedView
+{
+    document::DocumentTypeRepo::SP _docTypeRepo;
+
+    DummyFeedView()
+        : _docTypeRepo()
+    {}
+    DummyFeedView(const document::DocumentTypeRepo::SP &docTypeRepo)
+        : _docTypeRepo(docTypeRepo)
+    {}
+    virtual const document::DocumentTypeRepo::SP &getDocumentTypeRepo() const {
+        return _docTypeRepo;
+    }
+    virtual const ISimpleDocumentMetaStore *getDocumentMetaStorePtr() const {
+        return std::nullptr_t();
+    }
+    virtual void preparePut(PutOperation &) {}
+    virtual void handlePut(FeedToken *,
+                           const PutOperation &) {}
+    virtual void prepareUpdate(UpdateOperation &) {}
+    virtual void handleUpdate(FeedToken *,
+                              const UpdateOperation &) {}
+    virtual void prepareRemove(RemoveOperation &) {}
+    virtual void handleRemove(FeedToken *,
+                              const RemoveOperation &) {}
+    virtual void prepareDeleteBucket(DeleteBucketOperation &) {}
+    virtual void handleDeleteBucket(const DeleteBucketOperation &) {}
+    virtual void handleSplit(FeedToken *, const SplitBucketOperation &) {}
+    virtual void handleJoin(FeedToken *, const JoinBucketsOperation &) {}
+    virtual void prepareMove(MoveOperation &) {}
+    virtual void handleMove(const MoveOperation &) {}
+    virtual void heartBeat(search::SerialNum) {}
+    virtual void sync() {}
+    virtual void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &) {}
+    virtual void handleCompactLidSpace(const CompactLidSpaceOperation &) {}
+    void forceCommit(search::SerialNum) override { }
+};
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_handler.h b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_handler.h
new file mode 100644
index 00000000000..cf5d3f1148f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_handler.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/flushengine/iflushhandler.h>
+
+namespace proton {
+
+namespace test {
+
+/**
+ * Default implementation used for testing.
+ */
+struct DummyFlushHandler : public IFlushHandler
+{
+    DummyFlushHandler(const vespalib::string &name)
+        : IFlushHandler(name)
+    {}
+
+    // Implements IFlushHandler
+    virtual std::vector<IFlushTarget::SP> getFlushTargets() override {
+        return std::vector<IFlushTarget::SP>();
+    }
+
+    virtual SerialNum getCurrentSerialNumber() const override {
+        return 0;
+    }
+
+    virtual void flushDone(SerialNum oldestSerial) override {
+        (void) oldestSerial;
+    }
+
+    virtual void syncTls(SerialNum syncTo) override {
+        (void) syncTo;
+    }
+};
+
+} // namespace test
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h
new file mode 100644
index 00000000000..f2217d1e1bd
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/flush/iflushtarget.h>
+
+namespace proton {
+
+namespace test {
+
+struct DummyFlushTarget : public searchcorespi::IFlushTarget
+{
+    DummyFlushTarget(const vespalib::string &name)
+        : searchcorespi::IFlushTarget(name)
+    {}
+    DummyFlushTarget(const vespalib::string &name,
+                     const Type &type,
+                     const Component &component)
+        : searchcorespi::IFlushTarget(name, type, component)
+    {}
+    // Implements searchcorespi::IFlushTarget
+    virtual MemoryGain getApproxMemoryGain() const override { return MemoryGain(0, 0); }
+    virtual DiskGain getApproxDiskGain() const override { return DiskGain(0, 0); }
+    virtual SerialNum getFlushedSerialNum() const override { return 0; }
+    virtual Time getLastFlushTime() const override { return Time(); }
+    virtual bool needUrgentFlush() const override { return false; }
+    virtual searchcorespi::FlushTask::UP initFlush(SerialNum) override {
+        return searchcorespi::FlushTask::UP();
+    }
+    virtual searchcorespi::FlushStats getLastFlushStats() const override {
+        return searchcorespi::FlushStats();
+    }
+
+    virtual uint64_t getApproxBytesToWriteToDisk() const override {
+        return 0;
+    }
+};
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_summary_manager.h b/searchcore/src/vespa/searchcore/proton/test/dummy_summary_manager.h
new file mode 100644
index 00000000000..e0b478b7de0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_summary_manager.h
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/docsummary/isummarymanager.h>
+
+namespace proton {
+
+namespace test {
+
+struct DummySummaryManager : public ISummaryManager
+{
+    virtual ISummarySetup::SP
+    createSummarySetup(const vespa::config::search::SummaryConfig &,
+                       const vespa::config::search::SummarymapConfig &,
+                       const vespa::config::search::summary::JuniperrcConfig &,
+                       const document::DocumentTypeRepo::SP &,
+                       const std::shared_ptr<search::IAttributeManager> &) {
+        return ISummarySetup::SP();
+    }
+};
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/executor_observer.h b/searchcore/src/vespa/searchcore/proton/test/executor_observer.h
new file mode 100644
index 00000000000..e68f32a14c1
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/executor_observer.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/util/executor.h>
+
+namespace proton {
+namespace test {
+
+class ExecutorObserver : public vespalib::Executor
+{
+private:
+    vespalib::Executor &_executor;
+    uint32_t _executeCnt;
+
+public:
+    ExecutorObserver(vespalib::Executor &executor)
+        : _executor(executor),
+          _executeCnt(0)
+    {}
+
+    uint32_t getExecuteCnt() const { return _executeCnt; }
+
+    // Implements vespalib::Executor
+    virtual Task::UP execute(Task::UP task) {
+        ++_executeCnt;
+        return _executor.execute(std::move(task));
+    }
+};
+
+} // namespace test
+} // namespace proton
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h
new file mode 100644
index 00000000000..c6fe10ffec2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/index/iindexmanager.h>
+
+namespace proton {
+
+namespace test {
+
+/**
+ * Mock of the IIndexManager interface used for unit testing.
+ */
+struct MockIndexManager : public IIndexManager
+{
+    virtual void putDocument(uint32_t, const Document &, SerialNum) override {}
+    virtual void removeDocument(uint32_t, SerialNum) override {}
+    virtual void commit(SerialNum, OnWriteDoneType) override {}
+    virtual SerialNum getCurrentSerialNum() const override { return 0; }
+    virtual SerialNum getFlushedSerialNum() const override { return 0; }
+    virtual searchcorespi::IndexSearchable::SP getSearchable() const override {
+        return searchcorespi::IndexSearchable::SP();
+    }
+    virtual search::SearchableStats getSearchableStats() const override {
+        return search::SearchableStats();
+    }
+    virtual searchcorespi::IFlushTarget::List getFlushTargets() override {
+        return searchcorespi::IFlushTarget::List();
+    }
+    virtual void setSchema(const Schema &, const Schema &) override {}
+    virtual void wipeHistory(SerialNum, const Schema &) override {}
+    virtual void heartBeat(SerialNum) override {}
+};
+
+} // namespace test
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h
new file mode 100644
index 00000000000..5cf0d3f35d6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/index/i_index_writer.h>
+
+namespace proton {
+
+namespace test {
+
+/**
+ * Mock of the IIndexWriter interface used for unit testing.
+ */
+struct MockIndexWriter : public IIndexWriter
+{
+    IIndexManager::SP _idxMgr;
+    MockIndexWriter() : _idxMgr() {}
+    MockIndexWriter(const IIndexManager::SP &idxMgr) : _idxMgr(idxMgr) {}
+    virtual const IIndexManager::SP &getIndexManager() const override { return _idxMgr; }
+    virtual void put(search::SerialNum, const document::Document &, const search::DocumentIdT) override {}
+    virtual void remove(search::SerialNum, const search::DocumentIdT) override {}
+    virtual void commit(search::SerialNum, OnWriteDoneType) override {}
+    virtual void heartBeat(search::SerialNum) override {}
+};
+
+} // namespace test
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/test/resulthandler.h b/searchcore/src/vespa/searchcore/proton/test/resulthandler.h
new file mode 100644
index 00000000000..59c407d2bfa
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/resulthandler.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/persistenceengine/resulthandler.h>
+
+namespace proton {
+
+namespace test {
+
+class GenericResultHandler : public IGenericResultHandler
+{
+private:
+    std::unique_ptr<storage::spi::Result> _result;
+public:
+    virtual void handle(const storage::spi::Result &result) {
+        _result.reset(new storage::spi::Result(result));
+    }
+    bool valid() const { return _result.get() != NULL; }
+    const storage::spi::Result &getResult() const { return *_result; }
+};
+
+
+class BucketInfoResultHandler : public IBucketInfoResultHandler
+{
+private:
+    std::unique_ptr<storage::spi::BucketInfoResult> _result;
+public:
+    virtual void handle(const storage::spi::BucketInfoResult &result) {
+        _result.reset(new storage::spi::BucketInfoResult(result));
+    }
+    bool valid() const { return _result.get() != NULL; }
+    const storage::spi::BucketInfoResult &getResult() const { return *_result; }
+    const storage::spi::BucketInfo &getInfo() const { return getResult().getBucketInfo(); }
+};
+
+
+class BucketIdListResultHandler : public IBucketIdListResultHandler
+{
+private:
+    std::unique_ptr<storage::spi::BucketIdListResult> _result;
+public:
+    virtual void handle(const storage::spi::BucketIdListResult &result) {
+        _result.reset(new storage::spi::BucketIdListResult(result));
+    }
+    bool valid() const { return _result.get() != NULL; }
+    const storage::spi::BucketIdListResult &getResult() const { return *_result; }
+    const storage::spi::BucketIdListResult::List &getList() const { return getResult().getList(); }
+};
+
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/simple_job_tracker.h b/searchcore/src/vespa/searchcore/proton/test/simple_job_tracker.h
new file mode 100644
index 00000000000..aad61a9024c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/simple_job_tracker.h
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/metrics/i_job_tracker.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace proton {
+namespace test {
+
+struct SimpleJobTracker : public IJobTracker
+{
+    typedef std::shared_ptr<SimpleJobTracker> SP;
+    vespalib::CountDownLatch _started;
+    vespalib::CountDownLatch _ended;
+    SimpleJobTracker(uint32_t numJobTrackings)
+        : _started(numJobTrackings),
+          _ended(numJobTrackings)
+    {}
+
+    // Implements IJobTracker
+    virtual void start() { _started.countDown(); }
+    virtual void end() { _ended.countDown(); }
+};
+
+} // namespace test
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/simple_thread_service.h b/searchcore/src/vespa/searchcore/proton/test/simple_thread_service.h
new file mode 100644
index 00000000000..adb02e70365
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/simple_thread_service.h
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/index/i_thread_service.h>
+
+namespace proton {
+namespace test {
+
+/**
+ * Implementation of IThreadService that overrides isCurrentThread() to true.
+ * Can be used by unit tests that do not care about that functions are executed in the correct
+ * thread.
+ */
+class SimpleThreadService : public searchcorespi::index::IThreadService
+{
+private:
+    searchcorespi::index::IThreadService &_service;
+
+public:
+    SimpleThreadService(searchcorespi::index::IThreadService &service)
+        : _service(service)
+    {
+    }
+    virtual vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) {
+        return _service.execute(std::move(task));
+    }
+    virtual void run(vespalib::Runnable &runnable) {
+        _service.run(runnable);
+        // sync() because underlying implementation can use isCurrentThread()
+        // to determine to run it directly.
+        if (!_service.isCurrentThread()) {
+            sync();
+        }
+    }
+    virtual vespalib::Syncable &sync() {
+        _service.sync();
+        return *this;
+    }
+    virtual bool isCurrentThread() const {
+        return true;
+    }
+};
+
+} // namespace test
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/simple_threading_service.h b/searchcore/src/vespa/searchcore/proton/test/simple_threading_service.h
new file mode 100644
index 00000000000..3e78d0807d8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/simple_threading_service.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "simple_thread_service.h"
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+
+namespace proton {
+namespace test {
+
+/**
+ * Implementation of IThreadingService that overrides IThreadService::isCurrentThread() to true.
+ * Can be used by unit tests that do not care about that functions are executed in the correct
+ * thread.
+ */
+class SimpleThreadingService : public searchcorespi::index::IThreadingService
+{
+private:
+    searchcorespi::index::IThreadingService &_service;
+    SimpleThreadService _master;
+    SimpleThreadService _index;
+
+public:
+    SimpleThreadingService(searchcorespi::index::IThreadingService &service)
+        : _service(service),
+          _master(_service.master()),
+          _index(_service.index())
+    {
+    }
+    virtual vespalib::Syncable &sync() {
+        return _service.sync();
+    }
+    virtual searchcorespi::index::IThreadService &master() {
+        return _master;
+    }
+    virtual searchcorespi::index::IThreadService &index() {
+        return _index;
+    }
+    virtual search::ISequencedTaskExecutor &indexFieldInverter() {
+        return _service.indexFieldInverter();
+    }
+    virtual search::ISequencedTaskExecutor &indexFieldWriter() {
+        return _service.indexFieldWriter();
+    }
+
+    virtual search::ISequencedTaskExecutor &attributeFieldWriter() {
+        return _service.attributeFieldWriter();
+    }
+};
+
+} // namespace test
+} // namespace proton
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/test.h b/searchcore/src/vespa/searchcore/proton/test/test.h
new file mode 100644
index 00000000000..9aa02f74ace
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/test.h
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "attribute_utils.h"
+#include "bucketdocuments.h"
+#include "bucketstatecalculator.h"
+#include "document.h"
+#include "dummy_document_store.h"
+#include "dummy_document_sub_db.h"
+#include "dummy_feed_view.h"
+#include "dummy_summary_manager.h"
+#include "directory_handler.h"
+#include "resulthandler.h"
+#include "userdocuments.h"
+#include "userdocumentsbuilder.h"
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h
new file mode 100644
index 00000000000..f81026aaed7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/index/i_thread_service.h>
+
+namespace proton {
+namespace test {
+
+class ThreadServiceObserver : public searchcorespi::index::IThreadService
+{
+private:
+    searchcorespi::index::IThreadService &_service;
+    uint32_t _executeCnt;
+
+public:
+    ThreadServiceObserver(searchcorespi::index::IThreadService &service)
+        : _service(service),
+          _executeCnt(0)
+    {
+    }
+
+    uint32_t getExecuteCnt() const { return _executeCnt; }
+
+    /**
+     * Implements IThreadService
+     */
+    virtual vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) {
+        ++_executeCnt;
+        return _service.execute(std::move(task));
+    }
+    virtual void run(vespalib::Runnable &runnable) {
+        _service.run(runnable);
+    }
+    virtual vespalib::Syncable &sync() {
+        _service.sync();
+        return *this;
+    }
+    virtual bool isCurrentThread() const {
+        return _service.isCurrentThread();
+    }
+};
+
+} // namespace test
+} // namespace proton
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h
new file mode 100644
index 00000000000..041cb204c0a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/vespalib/util/closuretask.h>
+
+namespace proton {
+
+namespace test {
+
+template <typename FunctionType>
+void
+runFunction(FunctionType *func)
+{
+    (*func)();
+}
+
+/**
+ * Run the given function in the master thread and wait until done.
+ */
+template <typename FunctionType>
+void
+runInMaster(searchcorespi::index::IThreadingService &writeService,
+            FunctionType func)
+{
+    writeService.master().execute(vespalib::makeTask
+            (vespalib::makeClosure(&runFunction<FunctionType>, &func)));
+    writeService.sync();
+}
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h
new file mode 100644
index 00000000000..88a224447c6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "executor_observer.h"
+#include "thread_service_observer.h"
+#include <vespa/searchcorespi/index/ithreadingservice.h>
+#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h>
+
+namespace proton {
+namespace test {
+
+class ThreadingServiceObserver : public searchcorespi::index::IThreadingService
+{
+private:
+    searchcorespi::index::IThreadingService &_service;
+    ThreadServiceObserver _master;
+    ThreadServiceObserver _index;
+    search::SequencedTaskExecutorObserver _indexFieldInverter;
+    search::SequencedTaskExecutorObserver _indexFieldWriter;
+    search::SequencedTaskExecutorObserver _attributeFieldWriter;
+
+public:
+    ThreadingServiceObserver(searchcorespi::index::IThreadingService &service)
+        : _service(service),
+          _master(_service.master()),
+          _index(service.index()),
+          _indexFieldInverter(_service.indexFieldInverter()),
+          _indexFieldWriter(_service.indexFieldWriter()),
+          _attributeFieldWriter(_service.attributeFieldWriter())
+    {
+    }
+    const ThreadServiceObserver &masterObserver() const {
+        return _master;
+    }
+    const ThreadServiceObserver &indexObserver() const {
+        return _index;
+    }
+    const search::SequencedTaskExecutorObserver &indexFieldInverterObserver()
+        const
+    {
+        return _indexFieldInverter;
+    }
+    const search::SequencedTaskExecutorObserver &indexFieldWriterObserver()
+        const
+    {
+        return _indexFieldWriter;
+    }
+
+    const search::SequencedTaskExecutorObserver &attributeFieldWriterObserver()
+        const
+    {
+        return _attributeFieldWriter;
+    }
+
+    /**
+     * Implements vespalib::Syncable
+     */
+    virtual vespalib::Syncable &sync() {
+        return _service.sync();
+    }
+
+    /**
+     * Implements IThreadingService
+     */
+    virtual searchcorespi::index::IThreadService &master() {
+        return _master;
+    }
+    virtual searchcorespi::index::IThreadService &index() {
+        return _index;
+    }
+    virtual search::ISequencedTaskExecutor &indexFieldInverter()
+    {
+        return _indexFieldInverter;
+    }
+    virtual search::ISequencedTaskExecutor &indexFieldWriter()
+    {
+        return _indexFieldWriter;
+    }
+
+    virtual search::ISequencedTaskExecutor &attributeFieldWriter()
+    {
+        return _attributeFieldWriter;
+    }
+};
+
+} // namespace test
+} // namespace proton
+
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/userdocuments.h b/searchcore/src/vespa/searchcore/proton/test/userdocuments.h
new file mode 100644
index 00000000000..d1ea6f6ab4c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/userdocuments.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "bucketdocuments.h"
+
+namespace proton {
+
+namespace test {
+
+/**
+ * Collection of documents for a set of users,
+ * where each user is located in the same bucket.
+ */
+class UserDocuments
+{
+public:
+    typedef std::map<uint32_t, BucketDocuments> DocMap;
+    typedef DocMap::const_iterator Iterator;
+private:
+    DocMap _docs;
+public:
+    UserDocuments()
+        : _docs()
+    {
+    }
+    void merge(const UserDocuments &rhs) {
+        _docs.insert(rhs._docs.begin(), rhs._docs.end());
+    }
+    void addDoc(uint32_t userId, const Document &userDoc) {
+        _docs[userId].addDoc(userDoc);
+    }
+    const BucketDocuments &getUserDocs(uint32_t userId) const {
+        DocMap::const_iterator itr = _docs.find(userId);
+        assert(itr != _docs.end());
+        return itr->second;
+    }
+    document::BucketId getBucket(uint32_t userId) const {
+        return getUserDocs(userId).getBucket();
+    }
+    const DocumentVector &getDocs(uint32_t userId) const {
+        return getUserDocs(userId).getDocs();
+    }
+    DocumentVector getGidOrderDocs(uint32_t userId) const {
+        return getUserDocs(userId).getGidOrderDocs();
+    }
+    Iterator begin() const { return _docs.begin(); }
+    Iterator end() const { return _docs.end(); }
+    void clear() { _docs.clear(); }
+};
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/test/userdocumentsbuilder.cpp b/searchcore/src/vespa/searchcore/proton/test/userdocumentsbuilder.cpp
new file mode 100644
index 00000000000..71f5f21038b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/userdocumentsbuilder.cpp
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.test.userdocumentsbuilder");
+
+#include "userdocumentsbuilder.h"
+
+namespace proton {
+
+namespace test {
+
+UserDocumentsBuilder::UserDocumentsBuilder()
+    : _schema(),
+      _builder(_schema),
+      _docs()
+{
+}
+
+
+UserDocumentsBuilder &
+UserDocumentsBuilder::createDoc(uint32_t userId, search::DocumentIdT lid)
+{
+    vespalib::string docId = vespalib::make_string("userdoc:test:%u:%u", userId, lid);
+    document::Document::SP doc(_builder.startDocument(docId).endDocument().release());
+    _docs.addDoc(userId, Document(doc, lid, storage::spi::Timestamp(lid)));
+    return *this;
+}
+
+
+UserDocumentsBuilder &
+UserDocumentsBuilder::createDocs(uint32_t userId,
+                                 search::DocumentIdT begin,
+                                 search::DocumentIdT end)
+{
+    for (search::DocumentIdT lid = begin; lid < end; ++lid) {
+        createDoc(userId, lid);
+    }
+    return *this;
+}
+
+} // namespace test
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/test/userdocumentsbuilder.h b/searchcore/src/vespa/searchcore/proton/test/userdocumentsbuilder.h
new file mode 100644
index 00000000000..7c795c69580
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/userdocumentsbuilder.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "userdocuments.h"
+#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+namespace proton {
+
+namespace test {
+
+/**
+ * Builder for creating documents for a set of users.
+ */
+class UserDocumentsBuilder
+{
+private:
+    search::index::Schema     _schema;
+    search::index::DocBuilder _builder;
+    UserDocuments             _docs;
+public:
+    UserDocumentsBuilder();
+    const document::DocumentTypeRepo::SP &getRepo() const {
+        return _builder.getDocumentTypeRepo();
+    }
+    UserDocumentsBuilder &createDoc(uint32_t userId, search::DocumentIdT lid);
+    UserDocumentsBuilder &createDocs(uint32_t userId, search::DocumentIdT begin,
+                                     search::DocumentIdT end);
+    UserDocumentsBuilder &clearDocs() {
+        _docs.clear();
+        return *this;
+    }
+    const UserDocuments &getDocs() const { return _docs; }
+};
+
+
+} // namespace test
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/util/.gitignore b/searchcore/src/vespa/searchcore/util/.gitignore
new file mode 100644
index 00000000000..51f1b16bde5
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/.gitignore
@@ -0,0 +1,14 @@
+*.exp
+*.ilk
+*.lib
+*.pdb
+.depend
+ID
+Makefile
+extcase
+extcase.exe
+extprop
+extprop.exe
+mklicense
+result
+util.lib
diff --git a/searchcore/src/vespa/searchcore/util/CMakeLists.txt b/searchcore/src/vespa/searchcore/util/CMakeLists.txt
new file mode 100644
index 00000000000..98fce6782ca
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_util STATIC
+    SOURCES
+    base64encoder.cpp
+    eventloop.cpp
+    log.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/util/OWNERS b/searchcore/src/vespa/searchcore/util/OWNERS
new file mode 100644
index 00000000000..1037590124e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/OWNERS
@@ -0,0 +1 @@
+balder
diff --git a/searchcore/src/vespa/searchcore/util/autoptr.h b/searchcore/src/vespa/searchcore/util/autoptr.h
new file mode 100644
index 00000000000..9fe58babe3a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/autoptr.h
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+template <class T>
+class FastS_AutoPtr
+{
+private:
+    FastS_AutoPtr(const FastS_AutoPtr &);
+    FastS_AutoPtr& operator=(const FastS_AutoPtr &);
+
+    T *_val;
+    void Clean(void) {
+        if (_val != NULL) {
+            delete _val;
+            _val = NULL;
+        }
+    }
+public:
+    FastS_AutoPtr(void) : _val(NULL) { }
+    explicit FastS_AutoPtr(T *val)
+        : _val(val)
+    {
+    }
+    ~FastS_AutoPtr(void) { Clean(); }
+    void Set(T *val) { Clean(); _val = val; }
+    T *Get(void) const { return _val; }
+    T *HandOver(void) { T *ret = _val; _val = NULL; return ret; }
+    void Drop(void) {
+        if (_val != NULL) {
+            delete _val;
+            _val = NULL;
+        }
+    }
+};
+
+
+template <class T>
+class FastS_AutoRefCntPtr
+{
+private:
+    FastS_AutoRefCntPtr(const FastS_AutoRefCntPtr &);
+    FastS_AutoRefCntPtr& operator=(const FastS_AutoRefCntPtr &);
+
+    T *_val;
+    void Clean(void) {
+        if (_val != NULL)
+            _val->subRef();
+    }
+public:
+    FastS_AutoRefCntPtr(void) : _val(NULL) { }
+    explicit FastS_AutoRefCntPtr(T *val) {_val = val; }
+    ~FastS_AutoRefCntPtr(void) { Clean(); }
+    void Set(T *val) { Clean(); _val = val; }
+    void SetDup(T *val) {
+        Clean();
+        if (val != NULL)
+            val->addRef();
+        _val = val;
+    }
+    T *Get(void) const { return _val; }
+    T *GetDup(void) {
+        if (_val != NULL)
+            _val->addRef();
+        return _val;
+    }
+    T *HandOver(void) { T *ret = _val; _val = NULL; return ret; }
+    void Drop(void) {
+        if (_val != NULL) {
+            _val->subRef();
+            _val = NULL;
+        }
+    }
+};
+
+
+class FastS_AutoCharPtr
+{
+private:
+    FastS_AutoCharPtr(const FastS_AutoCharPtr &);
+    FastS_AutoCharPtr& operator=(const FastS_AutoCharPtr &);
+
+    char *_val;
+    void Clean(void) {
+        if (_val != NULL)
+            free(_val);
+    }
+public:
+    FastS_AutoCharPtr(void)
+        : _val(NULL)
+    {
+    }
+    explicit FastS_AutoCharPtr(char *val)
+        : _val(val)
+    {
+    }
+    ~FastS_AutoCharPtr(void) { Clean(); }
+    void Set(char *val) { Clean(); _val = val; }
+    char *Get(void) const { return _val; }
+    char *HandOver(void)  { char *ret = _val; _val = NULL; return ret; }
+    void Drop(void) {
+        if (_val != NULL) {
+            free(_val);
+            _val = NULL;
+        }
+    }
+};
+
diff --git a/searchcore/src/vespa/searchcore/util/base64encoder.cpp b/searchcore/src/vespa/searchcore/util/base64encoder.cpp
new file mode 100644
index 00000000000..7b3938533a7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/base64encoder.cpp
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2001-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/searchcore/util/base64encoder.h>
+
+char FastS_Base64Encoder::_base64Padding = '=';
+
+char FastS_Base64Encoder::_base64Table[] =
+{
+    'A','B','C','D','E','F','G','H','I','J','K','L','M',
+    'N','O','P','Q','R','S','T','U','V','W','X','Y','Z',
+    'a','b','c','d','e','f','g','h','i','j','k','l','m',
+    'n','o','p','q','r','s','t','u','v','w','x','y','z',
+    '0','1','2','3','4','5','6','7','8','9','+','/'
+};
diff --git a/searchcore/src/vespa/searchcore/util/base64encoder.h b/searchcore/src/vespa/searchcore/util/base64encoder.h
new file mode 100644
index 00000000000..10fcd85abca
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/base64encoder.h
@@ -0,0 +1,120 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2001-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+
+/**
+ * Simple class that may be used to base 64 encode a continuous data buffer.
+ **/
+class FastS_Base64Encoder
+{
+private:
+    FastS_Base64Encoder(const FastS_Base64Encoder &);
+    FastS_Base64Encoder& operator=(const FastS_Base64Encoder &);
+
+    /**
+     * The char used for padding in base 64 encoding.
+     **/
+    static char _base64Padding;
+
+    /**
+     * Table containing the 64 chars used to represent numbers from 0-63
+     * in base 64 encoding.
+     **/
+    static char _base64Table[];
+
+
+    const unsigned char  *_data;
+    const unsigned char  *_dataPos;
+    const unsigned char  *_dataEnd;
+
+public:
+
+
+    /**
+     * Create a base 64 encoder object with the task of encoding the
+     * given buffer.
+     *
+     * @param data the data to encode.
+     * @param datalen the byte-count of the data to encode.
+     **/
+    FastS_Base64Encoder(const void *data, unsigned int datalen)
+        : _data(static_cast<const unsigned char *>(data)),
+          _dataPos(_data),
+          _dataEnd(_data + datalen)
+    {
+    }
+
+
+    /**
+     * @return number of bytes left in the input buffer.
+     **/
+    unsigned int InputBytesLeft()
+    {
+        return (_dataEnd - _dataPos);
+    }
+
+
+    /**
+     * This method determines how much output space is needed to encode
+     * the rest of the input buffer referenced by this object.
+     *
+     * @return the space needed to encode the rest of the input.
+     **/
+    unsigned int OutputBytesNeeded()
+    {
+        unsigned int groups = (_dataEnd - _dataPos) / 3;
+        if (((_dataEnd - _dataPos) % 3) != 0)
+            groups++;
+
+        return (groups << 2);
+    }
+
+
+    /**
+     * Encode data from the buffer referenced by this object into the
+     * buffer given to this method. NOTE: dstLen should be at least 4
+     * since this method only encodes in complete groups.
+     *
+     * @return the number of bytes of output generated.
+     * @param dst where to generate output.
+     * @param dstLen maximum output to generate.
+     **/
+    unsigned int Encode(char *dst, unsigned int dstLen)
+    {
+        unsigned int groups = dstLen >> 2;
+
+        char *dstPos = dst;
+        for (;groups > 0 && InputBytesLeft() >= 3; groups--) {
+            dstPos[0] = _base64Table[_dataPos[0] >> 2];
+            dstPos[1] = _base64Table[((_dataPos[0] & 0x03) << 4) + (_dataPos[1] >> 4)];
+            dstPos[2] = _base64Table[((_dataPos[1] & 0x0f) << 2) + (_dataPos[2] >> 6)];
+            dstPos[3] = _base64Table[_dataPos[2] & 0x3f];
+            dstPos   += 4;
+            _dataPos += 3;
+        }
+
+        if (groups > 0 && InputBytesLeft() > 0) { // handle padding
+            dstPos[0] = _base64Table[_dataPos[0] >> 2];
+
+            if (InputBytesLeft() == 2) { // 2 bytes left
+                dstPos[1] = _base64Table[((_dataPos[0] & 0x03) << 4) + (_dataPos[1] >> 4)];
+                dstPos[2] = _base64Table[(_dataPos[1] & 0x0f) << 2];
+                dstPos[3] = _base64Padding;
+
+            } else {                     // 1 byte left
+                dstPos[1] = _base64Table[(_dataPos[0] & 0x03) << 4];
+                dstPos[2] = _base64Padding;
+                dstPos[3] = _base64Padding;
+            }
+            dstPos     += 4;
+            _dataPos    = _dataEnd;
+        }
+
+        return (dstPos - dst);
+    }
+
+};
+
diff --git a/searchcore/src/vespa/searchcore/util/description.html b/searchcore/src/vespa/searchcore/util/description.html
new file mode 100644
index 00000000000..025ae212095
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/description.html
@@ -0,0 +1 @@
+<!-- Short description for make kdoc. -->
diff --git a/searchcore/src/vespa/searchcore/util/eventloop.cpp b/searchcore/src/vespa/searchcore/util/eventloop.cpp
new file mode 100644
index 00000000000..fc249a2369a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/eventloop.cpp
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/searchcore/util/eventloop.h>
+
+
+double FastS_TimeOut::_val[FastS_TimeOut::valCnt];
+
+
+void
+FastS_TimeOut::WriteTime(char* buffer, size_t bufsize, double xtime)
+{
+    snprintf(buffer, bufsize, "%.3fs ", xtime);
+}
diff --git a/searchcore/src/vespa/searchcore/util/eventloop.h b/searchcore/src/vespa/searchcore/util/eventloop.h
new file mode 100644
index 00000000000..2bbb437675e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/eventloop.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1999-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+
+class FastS_TimeOut
+{
+public:
+    enum ValName {
+        maxSockSilent,		// 0
+        valCnt			// 1 - Must be last, used as array size:
+    };
+    static double _val[valCnt];
+
+    static void WriteTime(char* buffer, size_t bufsize, double xtime);
+};
+
+
diff --git a/searchcore/src/vespa/searchcore/util/log.cpp b/searchcore/src/vespa/searchcore/util/log.cpp
new file mode 100644
index 00000000000..064d0ef3e92
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/log.cpp
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2000-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/searchcore/util/log.h>
+LOG_SETUP(".searchcore.util.log");
+
+/**
+ * assert and abort functions.
+ */
+void __FastS_assert_fail(const char *assertion,
+			 const char *file,
+			 unsigned int line,
+			 const char * function)
+{
+    const char *vtag = V_TAG;
+    if (function != NULL) {
+        LOG(error, "FATAL: %s:%d (%s) %s: Failed assertion: '%s'",
+            file, line, vtag, function, assertion);
+        fprintf(stderr, "%s:%d (%s) %s: Failed assertion: '%s'\n",
+                file, line, vtag, function, assertion);
+    } else {
+        LOG(error, "FATAL: %s:%d (%s): Failed assertion: '%s'",
+            file, line, vtag, assertion);
+        fprintf(stderr, "%s:%d (%s): Failed assertion: '%s'\n",
+                file, line, vtag, assertion);
+    }
+    EV_STOPPING("", "assert failed");
+    abort();
+}
+
+void __FastS_abort(const char *message,
+		   const char *file,
+		   unsigned int line,
+		   const char * function)
+{
+    const char *vtag = V_TAG;
+    if (function != NULL) {
+        LOG(error, "FATAL: %s:%d (%s) %s: Abort called. Reason: %s",
+            file, line, vtag, function, message);
+        fprintf(stderr, "%s:%d (%s) %s: Abort called. Reason: %s\n",
+                file, line, vtag, function, message);
+    } else {
+        LOG(error, "FATAL: %s:%d (%s): Abort called. Reason: %s",
+            file, line, vtag, message);
+        fprintf(stderr, "%s:%d (%s): Abort called. Reason: %s\n",
+                file, line, vtag, message);
+    }
+    EV_STOPPING("", "aborted");
+    abort();
+}
diff --git a/searchcore/src/vespa/searchcore/util/log.h b/searchcore/src/vespa/searchcore/util/log.h
new file mode 100644
index 00000000000..873339a3253
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/log.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 2000-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+
+/*
+ * Define FastS_abort and FastS_assert macro's.
+ */
+
+/**
+ * This logs an "assertion failed" message and aborts.
+ */
+extern void __FastS_assert_fail(const char *assertion,
+                                const char *file,
+                                unsigned int line,
+                                const char * function);
+
+/**
+ * This logs an "abort" message and aborts.
+ */
+extern void __FastS_abort(const char *message,
+                          const char *file,
+                          unsigned int line,
+                          const char * function);
+
+# ifndef __STRING
+#  define __STRING(x) #x
+# endif
+
+# ifndef V_TAG
+#  define V_TAG "NOTAG"
+# endif
+
+# ifndef __ASSERT_FUNCTION
+#  define __ASSERT_FUNCTION NULL
+# endif
+
+
+# define FastS_abort(msg) \
+  (__FastS_abort(msg, __FILE__, __LINE__, __ASSERT_FUNCTION), abort())
+
+# ifndef NDEBUG
+#  define FastS_assert(expr) \
+  ((void) ((expr) ? 0 : \
+           (__FastS_assert_fail (__STRING(expr), \
+                                 __FILE__, __LINE__, \
+                                 __ASSERT_FUNCTION), 0)))
+# else
+#  define FastS_assert(expr)
+# endif // #ifndef NDEBUG
+
diff --git a/searchcore/src/vespa/searchcore/util/stlishheap.h b/searchcore/src/vespa/searchcore/util/stlishheap.h
new file mode 100644
index 00000000000..45b6ac0185d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/util/stlishheap.h
@@ -0,0 +1,398 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright (C) 1998-2003 Fast Search & Transfer ASA
+// Copyright (C) 2003 Overture Services Norway AS
+
+#pragma once
+
+/* These algorithms only work for standard C-type arrays, not generic
+   iterators the way STL works.  To make them work for stl-type
+   iterators, we need to create a set of traits classes, but that is
+   kind of overkill until we need that functionality.  */
+
+/* You can use these functions to customize the heap */
+
+template<typename T>
+inline bool
+FastS_min(T a, T b)
+{
+    return a < b;
+}
+
+template<typename T>
+inline bool
+FastS_max(T a, T b)
+{
+    return b < a;
+}
+
+/* Push obj onto the heap first of length len.  len must be large
+ * enough to include the new object.  For example if you have a heap
+ * with 3 elements and want to push a new element onto the heap, len
+ * should be 4.
+ */
+
+template <typename T, typename Comp>
+inline void
+FastS_push_heap(T *first, int len, T obj, Comp comp)
+{
+    int x = len - 1;
+    int parent = (x - 1)/2;
+    while (x > 0 && comp(*(first + parent), obj)) {
+        *(first + x) = *(first + parent);
+        x = parent;
+        parent = (x - 1)/2;
+    }
+    *(first + x) = obj;
+}
+
+/* Pop the largest element off the heap, reducing the size of the heap
+ * by 1. (Note: it is the responsibility of the caller to keep track
+ * of the size of the heap.)
+ */
+
+template<typename T, typename Comp>
+inline T
+FastS_pop_heap(T *first, int len, Comp comp)
+{
+    /* The algorithm we use is a variation of the textbook algorithm.
+       We first remove the first element, then instead of putting the
+       last element at the top of the heap and heapify(), we propagate
+       the "hole" left by the removed first element to the bottom.  Then
+       we copy the last element into the hole and push this element
+       upwards. Since the last element has a high probability of being
+       pushed down to the bottom anyways, this reduces the number of
+       comparisons we need to do. */
+    T ret = *first;
+    /* right child */
+    int topidx = 0;
+    int childidx = 2;
+    /* while both right and left child exist.. */
+    while (childidx < len) {
+        /* compare right to left child */
+        if (comp(*(first + childidx), *(first + childidx - 1)))
+            childidx--;
+        *(first + topidx) = *(first + childidx);
+        topidx = childidx;
+        childidx = 2 * (topidx + 1);
+    }
+    /* if only left child exists.. */
+    if (childidx == len) {
+        *(first + topidx) = *(first + childidx - 1);
+        topidx = childidx - 1;
+    }
+    /* now topidx is the hole.. */
+    FastS_push_heap(first, topidx + 1, *(first + len - 1), comp);
+    return ret;
+}
+
+/* Pop the largest element off the heap, and push a new element on the
+ * heap in the same operation.
+ */
+
+template<typename T, typename Comp>
+inline T
+FastS_pop_push_heap(T *first, int len, T obj, Comp comp)
+{
+    T ret = *first;
+    /* right child */
+    int topidx = 0;
+    int childidx = 2;
+    /* while both right and left child exist.. */
+    while (childidx < len) {
+        /* compare right to left child */
+        if (comp(*(first + childidx), *(first + childidx - 1)))
+            childidx--;
+        *(first + topidx) = *(first + childidx);
+        topidx = childidx;
+        childidx = 2 * (topidx + 1);
+    }
+    /* if only left child exists.. */
+    if (childidx == len) {
+        *(first + topidx) = *(first + childidx - 1);
+        topidx = childidx - 1;
+    }
+    /* now topidx is the hole.. */
+    FastS_push_heap(first, topidx + 1, obj, comp);
+    return ret;
+}
+
+/* Similar to FastS_pop_heap, this function, given a "hole" in the
+ * heap, heapify()es the heap downwards.  It then inserts obj and
+ * adjusts the heap upwards.
+ */
+
+template<typename T, typename Comp>
+inline void
+FastS__adjust_heap(T *first, int len, int hole, T obj, Comp comp)
+{
+    /* right child */
+    int topidx = hole;
+    int childidx = 2 * (hole + 1);
+    /* while both right and left child exist.. */
+    while (childidx < len) {
+        /* compare right to left child */
+        if (comp(*(first + childidx), *(first + childidx - 1)))
+            childidx--;
+        *(first + topidx) = *(first + childidx);
+        topidx = childidx;
+        childidx = 2 * (topidx + 1);
+    }
+    /* if only left child exists.. */
+    if (childidx == len) {
+        *(first + topidx) = *(first + childidx - 1);
+        topidx = childidx - 1;
+    }
+    /* now first[topidx] is the hole.. */
+    FastS_push_heap(first, topidx + 1, obj, comp);
+}
+
+template <typename T, typename Comp>
+inline void
+FastS_make_heap(T *first, int len, Comp comp)
+{
+    if (len < 2)
+        return;
+    int parent = (len - 2)/2;
+    for (/**/; parent >= 0; parent--) {
+        int holeidx = parent;
+        T obj = *(first + parent);
+        int childidx = 2 * (parent + 1);
+        while (childidx < len) {
+            if (comp(*(first + childidx), *(first + childidx - 1)))
+                childidx--;
+            if (comp(*(first + childidx), obj)) {
+                *(first + holeidx) = obj;
+                goto nextparent;
+            } else {
+                *(first + holeidx) = *(first + childidx);
+                holeidx = childidx;
+                childidx = 2* (holeidx + 1);
+            }
+        }
+        if (childidx == len) {
+            if (comp(*(first + childidx - 1), obj)) {
+                *(first + holeidx) = obj;
+            } else {
+                *(first + holeidx) = *(first + childidx - 1);
+                *(first + childidx - 1) = obj;
+            }
+        } else /* childidx > len */
+            *(first + holeidx) = obj;
+    nextparent:
+        ;
+    }
+}
+
+template <typename T, typename Comp>
+inline void
+FastS_sort_heap(T *first, int len, Comp comp)
+{
+    while (len > 0) {
+        *(first + len - 1) = FastS_pop_heap(first, len, comp);
+        len--;
+    }
+}
+
+template <typename T, typename Comp>
+inline bool
+FastS_is_heap(T *first, int len, Comp comp)
+{
+    for (int i = 0; i < len; i++) {
+        int left = 2 * i + 1;
+        int right = 2 * i + 2;
+        if (left < len && comp(*(first + i), *(first + left)))
+            return false;
+        if (right < len && comp(*(first + i), *(first + right)))
+            return false;
+    }
+    return true;
+}
+
+////////////////////////////////////////////////////////
+// Similar to the above, but without comparator support
+////////////////////////////////////////////////////////
+
+template <typename T>
+inline void
+FastS_push_heap(T *first, int len, T obj)
+{
+    int x = len - 1;
+    int parent = (x - 1)/2;
+    while (x > 0 && *(first + parent) < obj) {
+        *(first + x) = *(first + parent);
+        x = parent;
+        parent = (x - 1)/2;
+    }
+    *(first + x) = obj;
+}
+
+/* Pop the largest element off the heap, reducing the size of the heap
+ * by 1. (Note: it is the responsibility of the caller to keep track
+ * of the size of the heap.)
+ */
+
+template<typename T>
+inline T
+FastS_pop_heap(T *first, int len)
+{
+    /* The algorithm we use is a variation of the textbook algorithm.
+       We first remove the first element, then instead of putting the
+       last element at the top of the heap and heapify(), we propagate
+       the "hole" left by the removed first element to the bottom.  Then
+       we copy the last element into the hole and push this element
+       upwards. Since the last element has a high probability of being
+       pushed down to the bottom anyways, this reduces the number of
+       comparisons we need to do. */
+    T ret = *first;
+    /* right child */
+    int topidx = 0;
+    int childidx = 2;
+    /* while both right and left child exist.. */
+    while (childidx < len) {
+        /* compare right to left child */
+        if (*(first + childidx) < *(first + childidx - 1))
+            childidx--;
+        *(first + topidx) = *(first + childidx);
+        topidx = childidx;
+        childidx = 2 * (topidx + 1);
+    }
+    /* if only left child exists.. */
+    if (childidx == len) {
+        *(first + topidx) = *(first + childidx - 1);
+        topidx = childidx - 1;
+    }
+    /* now topidx is the hole.. */
+    FastS_push_heap(first, topidx + 1, *(first + len - 1));
+    return ret;
+}
+
+
+/* Pop the largest element off the heap, and push a new element on the
+ * heap in the same operation.
+ */
+
+template<typename T>
+inline T
+FastS_pop_push_heap(T *first, int len, T obj)
+{
+    T ret = *first;
+    /* right child */
+    int topidx = 0;
+    int childidx = 2;
+    /* while both right and left child exist.. */
+    while (childidx < len) {
+        /* compare right to left child */
+        if (*(first + childidx) < *(first + childidx - 1))
+            childidx--;
+        *(first + topidx) = *(first + childidx);
+        topidx = childidx;
+        childidx = 2 * (topidx + 1);
+    }
+    /* if only left child exists.. */
+    if (childidx == len) {
+        *(first + topidx) = *(first + childidx - 1);
+        topidx = childidx - 1;
+    }
+    /* now topidx is the hole.. */
+    FastS_push_heap(first, topidx + 1, obj);
+    return ret;
+}
+
+
+/* Similar to FastS_pop_heap, this function, given a "hole" in the
+ * heap, heapify()es the heap downwards.  It then inserts obj and
+ * adjusts the heap upwards.
+ */
+
+template<typename T>
+inline void
+FastS__adjust_heap(T *first, int len, int hole, T obj)
+{
+    /* right child */
+    int topidx = hole;
+    int childidx = 2 * (hole + 1);
+    /* while both right and left child exist.. */
+    while (childidx < len) {
+        /* compare right to left child */
+        if (*(first + childidx) < *(first + childidx - 1))
+            childidx--;
+        *(first + topidx) = *(first + childidx);
+        topidx = childidx;
+        childidx = 2 * (topidx + 1);
+    }
+    /* if only left child exists.. */
+    if (childidx == len) {
+        *(first + topidx) = *(first + childidx - 1);
+        topidx = childidx - 1;
+    }
+    /* now first[topidx] is the hole.. */
+    FastS_push_heap(first, topidx + 1, obj);
+}
+
+template <typename T>
+inline void
+FastS_make_heap(T *first, int len)
+{
+    if (len < 2)
+        return;
+    int parent = (len - 2)/2;
+    for (/**/; parent >= 0; parent--) {
+        int holeidx = parent;
+        T obj = *(first + parent);
+        int childidx = 2 * (parent + 1);
+        while (childidx < len) {
+            // Find largest of left, right child of holeidx, and object.
+            if (*(first + childidx) < *(first + childidx - 1))
+                childidx--;
+            if (*(first + childidx) < obj) {
+                *(first + holeidx) = obj;
+                goto nextparent;
+            } else {
+                // If child is largest, put it at holeidx, and
+                // look further down.
+                *(first + holeidx) = *(first + childidx);
+                holeidx = childidx;
+                childidx = 2* (holeidx + 1);
+            }
+        }
+        if (childidx == len) {
+            // Only left child exists
+            if (*(first + childidx - 1) < obj) {
+                *(first + holeidx) = obj;
+            } else {
+                *(first + holeidx) = *(first + childidx - 1);
+                *(first + childidx - 1) = obj;
+            }
+        } else /* childidx > len */
+            *(first + holeidx) = obj;
+    nextparent:
+        ;
+    }
+}
+
+template <typename T>
+inline void
+FastS_sort_heap(T *first, int len)
+{
+    while (len > 0) {
+        *(first + len - 1) = FastS_pop_heap(first, len);
+        len--;
+    }
+}
+
+template <typename T>
+inline bool
+FastS_is_heap(T *first, int len)
+{
+    for (int i = 0; i < len; i++) {
+        int left = 2 * i + 1;
+        int right = 2 * i + 2;
+        if (left < len && *(first + i) < *(first + left))
+            return false;
+        if (right < len && *(first + i) < *(first + right))
+            return false;
+    }
+    return true;
+}
+
+