Publish

44 files changed, 9538 insertions, 0 deletions

diff --git a/memfilepersistence/src/tests/.gitignore b/memfilepersistence/src/tests/.gitignore
new file mode 100644
index 00000000000..b8a959a31c5
--- /dev/null
+++ b/memfilepersistence/src/tests/.gitignore
@@ -0,0 +1,8 @@
+/.depend
+/Makefile
+/dirconfig.tmp
+/test.vlog
+/testfile.0
+/testrunner
+/vdsroot
+memfilepersistence_testrunner_app
diff --git a/memfilepersistence/src/tests/CMakeLists.txt b/memfilepersistence/src/tests/CMakeLists.txt
new file mode 100644
index 00000000000..ee0cea9e1a5
--- /dev/null
+++ b/memfilepersistence/src/tests/CMakeLists.txt
@@ -0,0 +1,14 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(memfilepersistence_testrunner_app
+    SOURCES
+    testhelper.cpp
+    testrunner.cpp
+    DEPENDS
+    memfilepersistence_testconformance
+    memfilepersistence_testdevices
+    memfilepersistence_testinit
+    memfilepersistence_testspi
+    memfilepersistence_testtools
+    memfilepersistence
+)
+vespa_add_test(NAME memfilepersistence_testrunner_app COMMAND memfilepersistence_testrunner_app)
diff --git a/memfilepersistence/src/tests/conformance/.gitignore b/memfilepersistence/src/tests/conformance/.gitignore
new file mode 100644
index 00000000000..7e7c0fe7fae
--- /dev/null
+++ b/memfilepersistence/src/tests/conformance/.gitignore
@@ -0,0 +1,2 @@
+/.depend
+/Makefile
diff --git a/memfilepersistence/src/tests/conformance/CMakeLists.txt b/memfilepersistence/src/tests/conformance/CMakeLists.txt
new file mode 100644
index 00000000000..378f5751931
--- /dev/null
+++ b/memfilepersistence/src/tests/conformance/CMakeLists.txt
@@ -0,0 +1,6 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(memfilepersistence_testconformance
+    SOURCES
+    memfileconformancetest.cpp
+    DEPENDS
+)
diff --git a/memfilepersistence/src/tests/conformance/memfileconformancetest.cpp b/memfilepersistence/src/tests/conformance/memfileconformancetest.cpp
new file mode 100644
index 00000000000..18a12788945
--- /dev/null
+++ b/memfilepersistence/src/tests/conformance/memfileconformancetest.cpp
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/memfilepersistence/spi/memfilepersistence.h>
+#include <vespa/persistence/conformancetest/conformancetest.h>
+
+LOG_SETUP(".test.conformance");
+
+using namespace storage::spi;
+
+namespace storage {
+namespace memfile {
+
+    /*
+struct MemFileConformanceTest : public ConformanceTest {
+    struct Factory : public PersistenceFactory {
+
+        PersistenceSPI::UP getPersistenceImplementation() {
+            return PersistenceSPI::UP(new MemFilePersistence);
+        }
+    };
+
+    MemFileConformanceTest()
+        : ConformanceTest(PersistenceFactory::UP(new Factory)) {}
+
+    CPPUNIT_TEST_SUITE(MemFileConformanceTest);
+    DEFINE_CONFORMANCE_TESTS();
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MemFileConformanceTest);
+*/
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/device/.gitignore b/memfilepersistence/src/tests/device/.gitignore
new file mode 100644
index 00000000000..7e7c0fe7fae
--- /dev/null
+++ b/memfilepersistence/src/tests/device/.gitignore
@@ -0,0 +1,2 @@
+/.depend
+/Makefile
diff --git a/memfilepersistence/src/tests/device/CMakeLists.txt b/memfilepersistence/src/tests/device/CMakeLists.txt
new file mode 100644
index 00000000000..845c70ae8e3
--- /dev/null
+++ b/memfilepersistence/src/tests/device/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(memfilepersistence_testdevices
+    SOURCES
+    mountpointlisttest.cpp
+    devicemanagertest.cpp
+    devicestest.cpp
+    devicemappertest.cpp
+    partitionmonitortest.cpp
+    DEPENDS
+)
diff --git a/memfilepersistence/src/tests/device/devicemanagertest.cpp b/memfilepersistence/src/tests/device/devicemanagertest.cpp
new file mode 100644
index 00000000000..eeb5007f452
--- /dev/null
+++ b/memfilepersistence/src/tests/device/devicemanagertest.cpp
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/device/devicemanager.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/util/exception.h>
+#include <sys/errno.h>
+#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
+
+namespace storage {
+
+namespace memfile {
+
+class DeviceManagerTest : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(DeviceManagerTest);
+    CPPUNIT_TEST(testEventClass);
+    CPPUNIT_TEST(testEventSending);
+    CPPUNIT_TEST(testXml);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testEventClass();
+    void testEventSending();
+    void testXml();
+
+    framework::defaultimplementation::FakeClock _clock;
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DeviceManagerTest);
+
+void DeviceManagerTest::testEventClass()
+{
+    // Test that creation various IO events through common errno errors
+    // generates understandable errors.
+    {
+        IOEvent e(IOEvent::createEventFromErrno(1, ENOTDIR, "/mypath"));
+        CPPUNIT_ASSERT_EQUAL(
+            std::string("IOEvent(PATH_FAILURE, Not a directory: /mypath, time 1)"),
+            e.toString(true));
+        CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE, e.getState());
+    }
+    {
+        IOEvent e(IOEvent::createEventFromErrno(2, EACCES, "/mypath"));
+        CPPUNIT_ASSERT_EQUAL(
+            std::string("IOEvent(NO_PERMISSION, Permission denied: /mypath, time 2)"),
+            e.toString(true));
+        CPPUNIT_ASSERT_EQUAL(Device::NO_PERMISSION, e.getState());
+    }
+    {
+        IOEvent e(IOEvent::createEventFromErrno(3, EIO, "/mypath"));
+        CPPUNIT_ASSERT_EQUAL(
+            std::string("IOEvent(IO_FAILURE, Input/output error: /mypath, time 3)"),
+            e.toString(true));
+        CPPUNIT_ASSERT_EQUAL(Device::IO_FAILURE, e.getState());
+    }
+    {
+        IOEvent e(
+                IOEvent::createEventFromErrno(4, EBADF, "/mypath", VESPA_STRLOC));
+        CPPUNIT_ASSERT_PREFIX(
+            std::string("IOEvent(INTERNAL_FAILURE, Bad file descriptor: /mypath"
+                        ", testEventClass in"),
+            e.toString(true));
+        CPPUNIT_ASSERT_EQUAL(Device::INTERNAL_FAILURE, e.getState());
+    }
+}
+
+namespace {
+
+    struct Listener : public IOEventListener {
+        std::ostringstream ost;
+
+        Listener() : ost() { ost << "\n"; }
+        virtual ~Listener() {}
+
+        virtual void handleDirectoryEvent(Directory& dir, const IOEvent& e) {
+            ost << "Dir " << dir.getPath() << ": " << e.toString(true) << "\n";
+        }
+        virtual void handlePartitionEvent(Partition& part, const IOEvent& e) {
+            ost << "Partition " << part.getMountPoint() << ": "
+                << e.toString(true) << "\n";
+        }
+        virtual void handleDiskEvent(Disk& disk, const IOEvent& e) {
+            ost << "Disk " << disk.getId() << ": " << e.toString(true) << "\n";
+        }
+
+    };
+
+}
+
+void DeviceManagerTest::testEventSending()
+{
+        // Test that adding events to directories in the manager actually sends
+        // these events on to listeners.
+    DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock);
+    Listener l;
+    manager.addIOEventListener(l);
+    Directory::LP dir(manager.getDirectory("/home/foo/var", 0));
+        // IO failures are disk events. Will mark all partitions and
+        // directories on that disk bad
+    dir->addEvent(IOEvent::createEventFromErrno(1, EIO, "/home/foo/var/foo"));
+    dir->addEvent(IOEvent::createEventFromErrno(2, EBADF, "/home/foo/var/bar"));
+    dir->addEvent(IOEvent::createEventFromErrno(3, EACCES, "/home/foo/var/car"));
+    dir->addEvent(IOEvent::createEventFromErrno(4, EISDIR, "/home/foo/var/var"));
+    std::string expected("\n"
+                         "Disk 1: IOEvent(IO_FAILURE, Input/output error: "
+                         "/home/foo/var/foo, time 1)\n"
+                         "Dir /home/foo/var: IOEvent(INTERNAL_FAILURE, Bad file "
+                         "descriptor: /home/foo/var/bar, time 2)\n"
+                         "Dir /home/foo/var: IOEvent(NO_PERMISSION, Permission denied: "
+                         "/home/foo/var/car, time 3)\n"
+                         "Dir /home/foo/var: IOEvent(PATH_FAILURE, Is a directory: "
+                         "/home/foo/var/var, time 4)\n"
+    );
+    CPPUNIT_ASSERT_EQUAL(expected, l.ost.str());
+}
+
+void DeviceManagerTest::testXml()
+{
+    DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock);
+    Directory::LP dir(manager.getDirectory("/home/", 0));
+    dir->getPartition().initializeMonitor();
+    std::string xml = manager.toXml("  ");
+    CPPUNIT_ASSERT_MSG(xml,
+                       xml.find("<partitionmonitor>") != std::string::npos);
+}
+
+}
+
+}
diff --git a/memfilepersistence/src/tests/device/devicemappertest.cpp b/memfilepersistence/src/tests/device/devicemappertest.cpp
new file mode 100644
index 00000000000..a78554a6342
--- /dev/null
+++ b/memfilepersistence/src/tests/device/devicemappertest.cpp
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/device/devicemapper.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <sys/errno.h>
+
+namespace storage {
+
+namespace memfile {
+
+class DeviceMapperTest : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(DeviceMapperTest);
+    CPPUNIT_TEST(testSimpleDeviceMapper);
+    CPPUNIT_TEST(testAdvancedDeviceMapper);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testSimpleDeviceMapper();
+    void testAdvancedDeviceMapper();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DeviceMapperTest);
+
+void DeviceMapperTest::testSimpleDeviceMapper()
+{
+    SimpleDeviceMapper mapper;
+    CPPUNIT_ASSERT_EQUAL(uint64_t(1), mapper.getDeviceId("whatever&�"));
+    CPPUNIT_ASSERT_EQUAL(uint64_t(1), mapper.getDeviceId("whatever&�"));
+    CPPUNIT_ASSERT_EQUAL(uint64_t(2), mapper.getDeviceId("whatnot"));
+    std::string expected("Whatever& �=)/%#)=");
+    CPPUNIT_ASSERT_EQUAL(expected, mapper.getMountPoint(expected));
+}
+
+void DeviceMapperTest::testAdvancedDeviceMapper()
+{
+    AdvancedDeviceMapper mapper;
+    try{
+        mapper.getDeviceId("/doesnotexist");
+        CPPUNIT_FAIL("Expected exception");
+    } catch (vespalib::Exception& e) {
+        std::string what(e.what());
+        CPPUNIT_ASSERT_CONTAIN(
+                "Failed to run stat to find data on file /doesnotexist", what);
+    }
+}
+
+}
+
+} // storage
diff --git a/memfilepersistence/src/tests/device/devicestest.cpp b/memfilepersistence/src/tests/device/devicestest.cpp
new file mode 100644
index 00000000000..bd6898cb7ac
--- /dev/null
+++ b/memfilepersistence/src/tests/device/devicestest.cpp
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/device/devicemanager.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <sys/errno.h>
+#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
+
+namespace storage {
+
+namespace memfile {
+
+class DevicesTest : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(DevicesTest);
+    CPPUNIT_TEST(testDisk);
+    CPPUNIT_TEST(testPartition);
+    CPPUNIT_TEST(testDirectory);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testDisk();
+    void testPartition();
+    void testDirectory();
+
+    framework::defaultimplementation::FakeClock _clock;
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DevicesTest);
+
+void DevicesTest::testDisk()
+{
+    DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock);
+    Disk::LP disk1(manager.getDisk("/something/on/disk"));
+    Disk::LP disk2(manager.getDisk("/something/on/disk"));
+    CPPUNIT_ASSERT_EQUAL(disk1->getId(), disk2->getId());
+    CPPUNIT_ASSERT_EQUAL(disk1.get(), disk2.get());
+    Disk::LP disk3(manager.getDisk("/something/on/disk2"));
+    CPPUNIT_ASSERT(disk2->getId() != disk3->getId());
+    disk3->toString(); // Add code coverage
+}
+
+void DevicesTest::testPartition()
+{
+    DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock);
+    Partition::LP part(manager.getPartition("/etc"));
+    CPPUNIT_ASSERT_EQUAL(std::string("/etc"), part->getMountPoint());
+    part->toString(); // Add code coverage
+}
+
+void DevicesTest::testDirectory()
+{
+    DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock);
+    Directory::LP dir1(manager.getDirectory("/on/disk", 0));
+    CPPUNIT_ASSERT_EQUAL(std::string("/on/disk"), dir1->getPath());
+    CPPUNIT_ASSERT(dir1->getLastEvent() == 0);
+    CPPUNIT_ASSERT_EQUAL(Device::OK, dir1->getState());
+    CPPUNIT_ASSERT(dir1->isOk());
+    CPPUNIT_ASSERT_EQUAL(std::string("/on/disk 0"), dir1->toString());
+
+    dir1->addEvent(Device::IO_FAILURE, "Ouch", "");
+    CPPUNIT_ASSERT(!dir1->isOk());
+    CPPUNIT_ASSERT(dir1->getLastEvent() != 0);
+    CPPUNIT_ASSERT_EQUAL(std::string("/on/disk 5 0 Ouch"), dir1->toString());
+    dir1->toString(); // Add code coverage
+}
+
+}
+
+} // storage
diff --git a/memfilepersistence/src/tests/device/mountpointlisttest.cpp b/memfilepersistence/src/tests/device/mountpointlisttest.cpp
new file mode 100644
index 00000000000..4cb5822ceb7
--- /dev/null
+++ b/memfilepersistence/src/tests/device/mountpointlisttest.cpp
@@ -0,0 +1,255 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <fstream>
+#include <vespa/memfilepersistence/device/mountpointlist.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
+
+using vespalib::LinkedPtr;
+using vespalib::fileExists;
+using vespalib::isDirectory;
+using vespalib::isSymLink;
+using vespalib::readLink;
+
+namespace storage {
+
+namespace memfile {
+
+class MountPointList_Test : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(MountPointList_Test);
+    CPPUNIT_TEST(testScanning);
+    CPPUNIT_TEST(testStatusFile);
+    CPPUNIT_TEST(testInitDisks);
+    CPPUNIT_TEST_SUITE_END();
+
+    static const std::string _prefix;
+
+public:
+    void testScanning();
+    void testStatusFile();
+    void testInitDisks();
+
+    void init();
+    void tearDown();
+
+    framework::defaultimplementation::FakeClock _clock;
+
+private:
+    LinkedPtr<DeviceManager> newDeviceManager() {
+        return LinkedPtr<DeviceManager>(
+                new DeviceManager(
+                        DeviceMapper::UP(new SimpleDeviceMapper),
+                        _clock));
+    }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MountPointList_Test);
+
+const std::string MountPointList_Test::_prefix("./vdsroot");
+
+namespace {
+    void run(const std::string& cmd) {
+        CPPUNIT_ASSERT_MESSAGE(cmd, system(cmd.c_str()) == 0);
+    }
+}
+
+void MountPointList_Test::init()
+{
+    tearDown();
+    run("rm -rf "+_prefix);
+    run("mkdir -p "+_prefix+"/disks");
+
+    run("mkdir "+_prefix+"/disks/d0");    // Regular dir
+    run("mkdir "+_prefix+"/disks/d1");    // Inaccessible dir
+    run("chmod 000 "+_prefix+"/disks/d1");
+    run("mkdir "+_prefix+"/disks/D2");    // Wrongly named dir
+    run("mkdir "+_prefix+"/disks/d3");    // Regular non-empty dir
+    run("touch "+_prefix+"/disks/d3/foo");
+    run("touch "+_prefix+"/disks/d4");    // Not a dir
+    run("ln -s D2 "+_prefix+"/disks/d5"); // Symlink to dir
+    run("ln -s d4 "+_prefix+"/disks/d6"); // Symlink to file
+}
+
+void MountPointList_Test::tearDown()
+{
+    try{
+        if (fileExists(_prefix+"/disks/d1")) {
+            run("chmod 755 "+_prefix+"/disks/d1");
+        }
+    } catch (std::exception& e) {
+        std::cerr << "Failed to clean up: " << e.what() << "\n";
+    }
+}
+
+void MountPointList_Test::testScanning()
+{
+    init();
+    MountPointList list(_prefix,
+                        std::vector<vespalib::string>(),
+                        vespalib::LinkedPtr<DeviceManager>(
+                                new DeviceManager(
+                                        DeviceMapper::UP(new SimpleDeviceMapper),
+                                        _clock)));
+    list.scanForDisks();
+
+        // Check that we got the expected entries.
+    CPPUNIT_ASSERT_EQUAL(7u, list.getSize());
+
+    for (uint32_t i=0; i<7u; ++i) {
+        std::ostringstream ost;
+        ost << _prefix << "/disks/d" << i;
+        CPPUNIT_ASSERT_EQUAL(ost.str(), list[i].getPath());
+    }
+
+    // Note.. scanForDisks() should not in any circumstances access the
+    // disks. Thus it should not know that d1 is inaccessible, or that d6
+    // is actually a symlink to a file
+    CPPUNIT_ASSERT_EQUAL(Device::OK,           list[0].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::OK,           list[1].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND,    list[2].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::OK,           list[3].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE, list[4].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::OK,           list[5].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::OK,           list[6].getState());
+
+    list.verifyHealthyDisks(-1);
+    CPPUNIT_ASSERT_EQUAL(Device::OK,               list[0].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::NO_PERMISSION,    list[1].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND,        list[2].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::INTERNAL_FAILURE, list[3].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE,     list[4].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::OK,               list[5].getState());
+    CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE,     list[6].getState());
+}
+
+void MountPointList_Test::testStatusFile()
+{
+    init();
+    std::string statusFileName(_prefix + "/disks.status");
+
+    // Try reading non-existing file, and writing a file
+    {
+        MountPointList list(_prefix,
+                            std::vector<vespalib::string>(),
+                            vespalib::LinkedPtr<DeviceManager>(
+                                    new DeviceManager(
+                                            DeviceMapper::UP(new SimpleDeviceMapper),
+                                            _clock)));
+
+        _clock.setAbsoluteTimeInSeconds(5678);
+        list.scanForDisks();
+
+        // File does not currently exist, that should be ok though.
+        list.readFromFile();
+        list.verifyHealthyDisks(-1);
+        CPPUNIT_ASSERT_EQUAL(7u, list.getSize());
+        list[5].addEvent(IOEvent(1234, Device::IO_FAILURE, "Argh", "Hmm"));
+        CPPUNIT_ASSERT_EQUAL(Device::IO_FAILURE, list[5].getState());
+
+        // Write to file.
+        list.writeToFile();
+    }
+
+    // Check contents of file.
+    {
+        std::ifstream in(statusFileName.c_str());
+        std::string line;
+        CPPUNIT_ASSERT(std::getline(in, line));
+
+        CPPUNIT_ASSERT_PREFIX(
+                std::string(_prefix + "/disks/d1 3 5678 IoException: NO PERMISSION: "
+                            "open(./vdsroot/disks/d1/chunkinfo, 0x1): Failed, "
+                            "errno(13): Permission denied"),
+                line);
+        CPPUNIT_ASSERT(std::getline(in, line));
+        CPPUNIT_ASSERT_PREFIX(
+                std::string(_prefix +"/disks/d2 1 5678 Disk not found during scanning of "
+                            "disks directory"),
+                line);
+        CPPUNIT_ASSERT(std::getline(in, line));
+        CPPUNIT_ASSERT_PREFIX(
+                std::string(_prefix + "/disks/d3 4 5678 Foreign data in mountpoint. New "
+                            "mountpoints added should be empty."),
+                line);
+        CPPUNIT_ASSERT(std::getline(in, line));
+        CPPUNIT_ASSERT_PREFIX(
+                std::string(_prefix + "/disks/d4 2 5678 File d4 in disks directory is not "
+                            "a directory."),
+                line);
+        CPPUNIT_ASSERT(std::getline(in, line));
+        CPPUNIT_ASSERT_PREFIX(std::string(_prefix + "/disks/d5 5 1234 Argh"),
+                              line);
+        CPPUNIT_ASSERT(std::getline(in, line));
+        CPPUNIT_ASSERT_PREFIX(
+                std::string(_prefix + "/disks/d6 2 5678 The path exist, but is not a "
+                            "directory."),
+                line);
+        CPPUNIT_ASSERT(std::getline(in, line));
+        CPPUNIT_ASSERT_EQUAL(std::string("EOF"), line);
+    }
+
+    // Starting over to get new device instances.
+    // Scan disk, read file, and check that erronious disks are not used.
+    {
+        MountPointList list(_prefix,
+                            std::vector<vespalib::string>(),
+                            vespalib::LinkedPtr<DeviceManager>(
+                                    new DeviceManager(
+                                            DeviceMapper::UP(new SimpleDeviceMapper),
+                                            _clock)));
+        list.scanForDisks();
+        list.readFromFile();
+            // Check that we got the expected entries.
+        CPPUNIT_ASSERT_EQUAL(7u, list.getSize());
+
+        // Note.. scanForDisks() should not under any circumstance access the
+        // disks. Thus it should not know that d1 is inaccessible.
+        CPPUNIT_ASSERT_EQUAL(Device::OK,            list[0].getState());
+        CPPUNIT_ASSERT_EQUAL(Device::NO_PERMISSION, list[1].getState());
+        CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND,     list[2].getState());
+        CPPUNIT_ASSERT_EQUAL(Device::INTERNAL_FAILURE, list[3].getState());
+        CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE,  list[4].getState());
+        CPPUNIT_ASSERT_EQUAL(Device::IO_FAILURE,    list[5].getState());
+        CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE,  list[6].getState());
+    }
+}
+
+void MountPointList_Test::testInitDisks()
+{
+    vespalib::string d3target = "d3target";
+    vespalib::string foodev = _prefix + "/foodev";
+    vespalib::string bardev = _prefix + "/bardev";
+
+    tearDown();
+    run("rm -rf " + _prefix);
+    run("mkdir -p " + _prefix + "/disks/d2");
+    run("ln -s " + d3target + " " + _prefix + "/disks/d3");
+
+    std::vector<vespalib::string> diskPaths {
+        // disks/d0 should become a regular directory
+        _prefix + "/disks/d0",
+        // disks/d1 should be a symlink to /foo
+        foodev,
+        // disks/d2 should already be a directory
+        "/ignored",
+        // disks/d3 should already be a symlink
+        "/ignored2"
+    };
+
+    MountPointList list(_prefix, diskPaths, newDeviceManager());
+    list.initDisks();
+
+    CPPUNIT_ASSERT(isDirectory(_prefix + "/disks"));
+    CPPUNIT_ASSERT(isDirectory(_prefix + "/disks/d0"));
+    CPPUNIT_ASSERT(isSymLink(_prefix + "/disks/d1"));
+    CPPUNIT_ASSERT_EQUAL(foodev, readLink(_prefix + "/disks/d1"));
+    CPPUNIT_ASSERT(isDirectory(_prefix + "/disks/d2"));
+    CPPUNIT_ASSERT(isSymLink(_prefix + "/disks/d3"));
+    CPPUNIT_ASSERT_EQUAL(d3target, readLink(_prefix + "/disks/d3"));
+}
+
+} // memfile
+
+} // storage
diff --git a/memfilepersistence/src/tests/device/partitionmonitortest.cpp b/memfilepersistence/src/tests/device/partitionmonitortest.cpp
new file mode 100644
index 00000000000..1a016edcc83
--- /dev/null
+++ b/memfilepersistence/src/tests/device/partitionmonitortest.cpp
@@ -0,0 +1,204 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/device/partitionmonitor.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+namespace storage {
+
+namespace memfile {
+
+struct PartitionMonitorTest : public CppUnit::TestFixture
+{
+    void testNormalUsage();
+    void testHighInodeFillrate();
+    void testAlwaysStatPolicy();
+    void testPeriodPolicy();
+    void testStatOncePolicy();
+    void testDynamicPolicy();
+    void testIsFull();
+
+    CPPUNIT_TEST_SUITE(PartitionMonitorTest);
+    CPPUNIT_TEST(testNormalUsage);
+    CPPUNIT_TEST(testHighInodeFillrate);
+    CPPUNIT_TEST(testAlwaysStatPolicy);
+    CPPUNIT_TEST(testPeriodPolicy);
+    CPPUNIT_TEST(testStatOncePolicy);
+    CPPUNIT_TEST(testDynamicPolicy);
+    CPPUNIT_TEST(testIsFull);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(PartitionMonitorTest);
+
+struct FakeStatter : public PartitionMonitor::Statter {
+    struct statvfs _info;
+
+    FakeStatter() {
+        _info.f_bsize = 4096;
+        _info.f_frsize = 4096;
+        _info.f_blocks = 1000;
+        _info.f_bfree = 500;
+        _info.f_bavail = 400;
+        _info.f_files = 64;
+        _info.f_ffree = 32;
+        _info.f_favail = 30;
+        _info.f_fsid = 13;
+        _info.f_namemax = 256;
+    }
+    void removeData(uint32_t size) {
+        _info.f_bavail += (size / _info.f_bsize);
+        _info.f_bfree += (size / _info.f_bsize);
+    }
+    void addData(uint32_t size) {
+        _info.f_bavail -= (size / _info.f_bsize);
+        _info.f_bfree -= (size / _info.f_bsize);
+    }
+
+    virtual void statFileSystem(const std::string&, struct statvfs& info) {
+        info = _info;
+    }
+};
+
+void PartitionMonitorTest::testNormalUsage()
+{
+    PartitionMonitor monitor("testrunner.cpp");
+    FakeStatter* statter = new FakeStatter();
+    monitor.setStatter(std::unique_ptr<PartitionMonitor::Statter>(statter));
+    std::string expected(
+            "PartitionMonitor(testrunner.cpp, STAT_PERIOD(100), "
+            "2048000/3686400 used - 55.5556 % full)");
+    CPPUNIT_ASSERT_EQUAL(expected, monitor.toString(false));
+    expected =
+            "PartitionMonitor(testrunner.cpp) {\n"
+            "  Fill rate: 55.5556 %\n"
+            "  Inode fill rate: 51.6129 %\n"
+            "  Detected block size: 4096\n"
+            "  File system id: 13\n"
+            "  Total size: 3686400 (3600 kB)\n"
+            "  Used size: 2048000 (2000 kB)\n"
+            "  Queries since last stat: 0\n"
+            "  Monitor policy: STAT_PERIOD(100)\n"
+            "  Root only ratio 0\n"
+            "  Max fill rate 98 %\n"
+            "}";
+    CPPUNIT_ASSERT_EQUAL(expected, monitor.toString(true));
+    CPPUNIT_ASSERT(monitor.getFillRate() > 0.55);
+}
+
+void PartitionMonitorTest::testHighInodeFillrate()
+{
+    PartitionMonitor monitor("testrunner.cpp");
+    FakeStatter* statter = new FakeStatter();
+    statter->_info.f_favail = 2;
+    monitor.setStatter(std::unique_ptr<PartitionMonitor::Statter>(statter));
+    std::string expected(
+            "PartitionMonitor(testrunner.cpp, STAT_PERIOD(100), "
+            "2048000/3686400 used - 94.1176 % full (inodes))");
+    CPPUNIT_ASSERT_EQUAL(expected, monitor.toString(false));
+    expected =
+            "PartitionMonitor(testrunner.cpp) {\n"
+            "  Fill rate: 55.5556 %\n"
+            "  Inode fill rate: 94.1176 %\n"
+            "  Detected block size: 4096\n"
+            "  File system id: 13\n"
+            "  Total size: 3686400 (3600 kB)\n"
+            "  Used size: 2048000 (2000 kB)\n"
+            "  Queries since last stat: 0\n"
+            "  Monitor policy: STAT_PERIOD(100)\n"
+            "  Root only ratio 0\n"
+            "  Max fill rate 98 %\n"
+            "}";
+    CPPUNIT_ASSERT_EQUAL(expected, monitor.toString(true));
+    CPPUNIT_ASSERT(monitor.getFillRate() > 0.94);
+}
+
+void PartitionMonitorTest::testAlwaysStatPolicy()
+{
+    PartitionMonitor monitor("testrunner.cpp");
+    FakeStatter* statter = new FakeStatter();
+    monitor.setStatter(std::unique_ptr<PartitionMonitor::Statter>(statter));
+    monitor.setAlwaysStatPolicy();
+    for (uint32_t i=0; i<10; ++i) {
+        monitor.getFillRate();
+        CPPUNIT_ASSERT_EQUAL(0u, monitor._queriesSinceStat);
+    }
+}
+
+void PartitionMonitorTest::testPeriodPolicy()
+{
+    PartitionMonitor monitor("testrunner.cpp");
+    FakeStatter* statter = new FakeStatter();
+    monitor.setStatter(std::unique_ptr<PartitionMonitor::Statter>(statter));
+    monitor.setStatPeriodPolicy(4);
+    for (uint32_t i=1; i<16; ++i) {
+        monitor.getFillRate();
+        CPPUNIT_ASSERT_EQUAL(i % 4, monitor._queriesSinceStat);
+    }
+}
+
+void PartitionMonitorTest::testStatOncePolicy()
+{
+    PartitionMonitor monitor("testrunner.cpp");
+    FakeStatter* statter = new FakeStatter();
+    monitor.setStatter(std::unique_ptr<PartitionMonitor::Statter>(statter));
+    monitor.setStatOncePolicy();
+    for (uint32_t i=1; i<16; ++i) {
+        monitor.getFillRate();
+        CPPUNIT_ASSERT_EQUAL(i, monitor._queriesSinceStat);
+    }
+}
+
+void PartitionMonitorTest::testDynamicPolicy()
+{
+    PartitionMonitor monitor("testrunner.cpp");
+    FakeStatter* statter = new FakeStatter();
+    monitor.setStatter(std::unique_ptr<PartitionMonitor::Statter>(statter));
+    monitor.setStatDynamicPolicy(2);
+        // Add some data, such that we see that period goes down
+    CPPUNIT_ASSERT_EQUAL(uint64_t(3698), monitor.calcDynamicPeriod());
+    CPPUNIT_ASSERT_EQUAL(55, (int) (100 * monitor.getFillRate()));
+    monitor.addingData(256 * 1024);
+    CPPUNIT_ASSERT_EQUAL(uint64_t(2592), monitor.calcDynamicPeriod());
+    CPPUNIT_ASSERT_EQUAL(62, (int) (100 * monitor.getFillRate()));
+    monitor.addingData(512 * 1024);
+    CPPUNIT_ASSERT_EQUAL(uint64_t(968), monitor.calcDynamicPeriod());
+    CPPUNIT_ASSERT_EQUAL(76, (int) (100 * monitor.getFillRate()));
+        // Add such that we hint that we have more data than possible on disk
+    monitor.addingData(1024 * 1024);
+        // Let fake stat just have a bit more data than before
+    statter->addData(256 * 1024);
+        // With high fill rate, we should check stat each time
+    CPPUNIT_ASSERT_EQUAL(uint64_t(1), monitor.calcDynamicPeriod());
+        // As period is 1, we will now do a new stat, it should find we
+        // actually have less fill rate
+    CPPUNIT_ASSERT_EQUAL(62, (int) (100 * monitor.getFillRate()));
+}
+
+void PartitionMonitorTest::testIsFull()
+{
+    PartitionMonitor monitor("testrunner.cpp");
+    monitor.setMaxFillness(0.85);
+    FakeStatter* statter = new FakeStatter();
+    monitor.setStatOncePolicy();
+    monitor.setStatter(std::unique_ptr<PartitionMonitor::Statter>(statter));
+
+    CPPUNIT_ASSERT_EQUAL(55, (int) (100 * monitor.getFillRate()));
+    CPPUNIT_ASSERT(!monitor.isFull());
+    monitor.addingData(512 * 1024);
+    CPPUNIT_ASSERT_EQUAL(69, (int) (100 * monitor.getFillRate()));
+    CPPUNIT_ASSERT(!monitor.isFull());
+    monitor.addingData(600 * 1024);
+    CPPUNIT_ASSERT_EQUAL(86, (int) (100 * monitor.getFillRate()));
+    CPPUNIT_ASSERT(monitor.isFull());
+    monitor.removingData(32 * 1024);
+    CPPUNIT_ASSERT_EQUAL(85, (int) (100 * monitor.getFillRate()));
+    CPPUNIT_ASSERT(monitor.isFull());
+    monitor.removingData(32 * 1024);
+    CPPUNIT_ASSERT_EQUAL(84, (int) (100 * monitor.getFillRate()));
+    CPPUNIT_ASSERT(!monitor.isFull());
+}
+
+}
+
+} // storage
diff --git a/memfilepersistence/src/tests/init/.gitignore b/memfilepersistence/src/tests/init/.gitignore
new file mode 100644
index 00000000000..7e7c0fe7fae
--- /dev/null
+++ b/memfilepersistence/src/tests/init/.gitignore
@@ -0,0 +1,2 @@
+/.depend
+/Makefile
diff --git a/memfilepersistence/src/tests/init/CMakeLists.txt b/memfilepersistence/src/tests/init/CMakeLists.txt
new file mode 100644
index 00000000000..ebc4738a8c4
--- /dev/null
+++ b/memfilepersistence/src/tests/init/CMakeLists.txt
@@ -0,0 +1,6 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(memfilepersistence_testinit
+    SOURCES
+    filescannertest.cpp
+    DEPENDS
+)
diff --git a/memfilepersistence/src/tests/init/filescannertest.cpp b/memfilepersistence/src/tests/init/filescannertest.cpp
new file mode 100644
index 00000000000..8b49a21dad0
--- /dev/null
+++ b/memfilepersistence/src/tests/init/filescannertest.cpp
@@ -0,0 +1,492 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <iomanip>
+#include <vespa/memfilepersistence/device/devicemanager.h>
+#include <vespa/memfilepersistence/init/filescanner.h>
+#include <vespa/memfilepersistence/mapper/bucketdirectorymapper.h>
+#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h>
+#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
+#include <vespa/vdslib/state/nodestate.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/random.h>
+#include <sys/errno.h>
+
+namespace storage {
+namespace memfile {
+
+struct FileScannerTest : public CppUnit::TestFixture {
+    struct TestParameters {
+        uint32_t filesPerDisk;
+        uint32_t diskCount;
+        uint32_t bucketSplitBits;
+        uint32_t dirLevels;
+        uint32_t dirSpread;
+        uint32_t parts;
+        std::set<uint32_t> disksDown;
+        bool diskDownWithBrokenSymlink;
+        bool bucketWrongDir;
+        bool bucketMultipleDirs;
+        bool bucketMultipleDisks;
+        bool addTemporaryFiles;
+        bool addAlienFiles;
+        bool dirWithNoListPermission;
+        bool dirWithNoWritePermission;
+        bool dirWithNoExecutePermission;
+        bool fileWithNoReadPermission;
+        bool fileWithNoWritePermission;
+
+        TestParameters()
+            : filesPerDisk(10), diskCount(5), bucketSplitBits(20),
+              dirLevels(1), dirSpread(16), parts(1), disksDown(),
+              diskDownWithBrokenSymlink(false),
+              bucketWrongDir(false), bucketMultipleDirs(false),
+              bucketMultipleDisks(false),
+              addTemporaryFiles(false), addAlienFiles(false),
+              dirWithNoListPermission(false),
+              dirWithNoWritePermission(false),
+              dirWithNoExecutePermission(false),
+              fileWithNoReadPermission(false),
+              fileWithNoWritePermission(false) {}
+        void addAllComplexities() {
+            disksDown.insert(0);
+            disksDown.insert(2);
+            disksDown.insert(4);
+            bucketWrongDir = true;
+            bucketMultipleDirs = true;
+            bucketMultipleDisks = true;
+            parts = 7;
+            addTemporaryFiles = true;
+            addAlienFiles = true;
+            dirWithNoWritePermission = true;
+            fileWithNoWritePermission = true;
+            fileWithNoReadPermission = true;
+        }
+    };
+
+    void testNormalUsage() {
+        TestParameters params;
+        runTest(params);
+    }
+    void testMultipleParts() {
+        TestParameters params;
+        params.parts = 3;
+        runTest(params);
+    }
+    void testBucketInWrongDirectory() {
+        TestParameters params;
+        params.bucketWrongDir = true;
+        runTest(params);
+    }
+    void testBucketInMultipleDirectories() {
+        TestParameters params;
+        params.bucketMultipleDirs = true;
+        runTest(params);
+    }
+    void testZeroDirLevel() {
+        TestParameters params;
+        params.dirLevels = 0;
+        runTest(params);
+    }
+    void testSeveralDirLevels() {
+        TestParameters params;
+        params.dirLevels = 3;
+        runTest(params);
+    }
+    void testNonStandardDirSpread() {
+        TestParameters params;
+        params.dirSpread = 63;
+        runTest(params);
+    }
+    void testDiskDown() {
+        TestParameters params;
+        params.disksDown.insert(1);
+        runTest(params);
+    }
+    void testDiskDownBrokenSymlink() {
+        TestParameters params;
+        params.disksDown.insert(1);
+        params.disksDown.insert(3);
+        params.diskDownWithBrokenSymlink = true;
+        runTest(params);
+    }
+    void testRemoveTemporaryFile() {
+        TestParameters params;
+        params.addTemporaryFiles = true;
+        runTest(params);
+    }
+    void testAlienFile() {
+        TestParameters params;
+        params.addAlienFiles = true;
+        runTest(params);
+    }
+    void testUnlistableDirectory() {
+        TestParameters params;
+        params.dirWithNoListPermission = true;
+        runTest(params);
+    }
+    void testDirWithNoWritePermission() {
+        TestParameters params;
+        params.dirWithNoWritePermission = true;
+        runTest(params);
+    }
+    void testDirWithNoExecutePermission() {
+        TestParameters params;
+        params.dirWithNoWritePermission = true;
+        runTest(params);
+    }
+    void testFileWithNoReadPermission() {
+        TestParameters params;
+        params.bucketWrongDir = true;
+        params.fileWithNoReadPermission = true;
+        runTest(params);
+    }
+    void testFileWithNoWritePermission() {
+        TestParameters params;
+        params.bucketWrongDir = true;
+        params.fileWithNoWritePermission = true;
+        runTest(params);
+    }
+    void testAllFailuresCombined() {
+        TestParameters params;
+        params.addAllComplexities();
+        runTest(params);
+    }
+
+    CPPUNIT_TEST_SUITE(FileScannerTest);
+    CPPUNIT_TEST(testNormalUsage);
+    CPPUNIT_TEST(testMultipleParts);
+    CPPUNIT_TEST(testBucketInWrongDirectory);
+    CPPUNIT_TEST(testBucketInMultipleDirectories);
+    CPPUNIT_TEST(testZeroDirLevel);
+    CPPUNIT_TEST(testSeveralDirLevels);
+    CPPUNIT_TEST(testNonStandardDirSpread);
+    CPPUNIT_TEST(testDiskDown);
+    CPPUNIT_TEST(testDiskDownBrokenSymlink);
+    CPPUNIT_TEST(testRemoveTemporaryFile);
+    CPPUNIT_TEST(testAlienFile);
+    CPPUNIT_TEST(testUnlistableDirectory);
+    CPPUNIT_TEST(testDirWithNoWritePermission);
+    CPPUNIT_TEST(testDirWithNoExecutePermission);
+    CPPUNIT_TEST(testFileWithNoReadPermission);
+    CPPUNIT_TEST(testFileWithNoWritePermission);
+    CPPUNIT_TEST(testAllFailuresCombined);
+    CPPUNIT_TEST_SUITE_END();
+
+    // Actual implementation of the tests.
+
+    /** Run a console command and fail test if it fails. */
+    void run(std::string cmd);
+
+    /** Struct containing metadata for a single bucket. */
+    struct BucketData {
+        document::BucketId bucket;
+        uint32_t disk;
+        std::vector<uint32_t> directory;
+        bool shouldExist; // Set to false for buckets that won't exist due to
+                          // some failure.
+
+        BucketData() : shouldExist(true) {}
+
+        bool sameDir(BucketData& other) const {
+            return (disk == other.disk && directory == other.directory);
+        }
+    };
+
+    /**
+     * Create an overview of the buckets we're gonna use in the test.
+     * (Without any failures introduced)
+     */
+    std::vector<BucketData> createBuckets(const TestParameters& params);
+
+    /**
+     * Create the data in the bucket map and introduce the failures specified
+     * in the test. Mark buckets in bucket list that won't exist due to the
+     * failures so we know how to verify result of test.
+     */
+    void createData(const TestParameters&, std::vector<BucketData>& buckets,
+                    std::vector<std::string>& tempFiles,
+                    std::vector<std::string>& alienFiles);
+
+    /**
+     * Run a test with a given set of parameters, calling createData to set up
+     * the data, and then using a file scanner to actually list the files.
+     */
+    void runTest(const TestParameters&);
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(FileScannerTest);
+
+void
+FileScannerTest::run(std::string cmd)
+{
+    int result = system(cmd.c_str());
+    if (result != 0) {
+        CPPUNIT_FAIL("Failed to run command '" + cmd + "'.");
+    }
+}
+
+std::vector<FileScannerTest::BucketData>
+FileScannerTest::createBuckets(const TestParameters& params)
+{
+    std::vector<BucketData> buckets;
+    BucketDirectoryMapper dirMapper(params.dirLevels, params.dirSpread);
+    for (uint32_t i=0; i<params.diskCount; ++i) {
+        if (params.disksDown.find(i) != params.disksDown.end()) {
+            continue;
+        }
+        for (uint32_t j=0; j<params.filesPerDisk; ++j) {
+            BucketData data;
+            data.bucket = document::BucketId(params.bucketSplitBits,
+                                             params.filesPerDisk * i + j);
+            data.disk = i;
+            data.directory = dirMapper.getPath(data.bucket);
+            buckets.push_back(data);
+        }
+    }
+    return buckets;
+}
+
+void
+FileScannerTest::createData(const TestParameters& params,
+                            std::vector<BucketData>& buckets,
+                            std::vector<std::string>& tempFiles,
+                            std::vector<std::string>& alienFiles)
+{
+    if (params.bucketWrongDir) {
+        CPPUNIT_ASSERT(params.dirLevels > 0);
+        buckets[0].directory[0] = (buckets[0].directory[0] + 1)
+                                % params.dirSpread;
+    }
+    if (params.bucketMultipleDirs) {
+        CPPUNIT_ASSERT(params.dirLevels > 0);
+        BucketData copy(buckets[1]);
+        copy.directory[0] = (buckets[1].directory[0] + 1) % params.dirSpread;
+        buckets.push_back(copy);
+    }
+    if (params.bucketMultipleDisks && params.dirLevels > 0) {
+        BucketData copy(buckets[2]);
+        uint32_t disk = 0;
+        for (; disk<params.diskCount; ++disk) {
+            if (disk == copy.disk) continue;
+            if (params.disksDown.find(disk) == params.disksDown.end()) break;
+        }
+        CPPUNIT_ASSERT(disk < params.diskCount);
+        copy.disk = disk;
+        buckets.push_back(copy);
+    }
+
+    run("mkdir -p vdsroot");
+    run("chmod -R a+rwx vdsroot");
+    run("rm -rf vdsroot");
+    run("mkdir -p vdsroot/disks");
+    vespalib::RandomGen randomizer;
+    uint32_t diskToHaveBrokenSymlink = (params.disksDown.empty()
+            ? 0 : randomizer.nextUint32(0, params.disksDown.size()));
+    uint32_t downIndex = 0;
+    for (uint32_t i=0; i<params.diskCount; ++i) {
+        if (params.disksDown.find(i) != params.disksDown.end()) {
+            if (downIndex++ == diskToHaveBrokenSymlink
+                && params.diskDownWithBrokenSymlink)
+            {
+                std::ostringstream path;
+                path << "vdsroot/disks/d" << i;
+                run("ln -s /non-existing-dir " + path.str());
+            }
+        } else {
+            std::ostringstream path;
+            path << "vdsroot/disks/d" << i;
+            run("mkdir -p " + path.str());
+            std::ofstream of((path.str() + "/chunkinfo").c_str());
+            of << "#chunkinfo\n" << i << "\n" << params.diskCount << "\n";
+        }
+    }
+    for (uint32_t i=0; i<buckets.size(); ++i) {
+        if (!buckets[i].shouldExist) continue;
+        std::ostringstream path;
+        path << "vdsroot/disks/d" << buckets[i].disk << std::hex;
+        for (uint32_t j=0; j<buckets[i].directory.size(); ++j) {
+            path << '/' << std::setw(4) << std::setfill('0')
+                 << buckets[i].directory[j];
+        }
+        run("mkdir -p " + path.str());
+        if (params.dirWithNoListPermission && i == 8) {
+            run("chmod a-r " + path.str());
+                // Scanner will abort with exception, so we don't really know
+                // how many docs will not be found due to this.
+            continue;
+        }
+        if (params.dirWithNoExecutePermission && i == 9) {
+            run("chmod a-x " + path.str());
+                // Scanner will abort with exception, so we don't really know
+                // how many docs will not be found due to this.
+            continue;
+        }
+        path << '/' << std::setw(16) << std::setfill('0')
+             << buckets[i].bucket.getId() << ".0";
+        run("touch " + path.str());
+        if (params.addTemporaryFiles && i == 4) {
+            run("touch " + path.str() + ".tmp");
+            tempFiles.push_back(path.str() + ".tmp");
+        }
+        if (params.addAlienFiles && i == 6) {
+            run("touch " + path.str() + ".alien");
+            alienFiles.push_back(path.str() + ".alien");
+        }
+        if (params.fileWithNoWritePermission && i == 0) {
+            // Overlapping with wrong dir so it would want to move file
+            run("chmod a-w " + path.str());
+        }
+        if (params.fileWithNoReadPermission && i == 0) {
+            // Overlapping with wrong dir so it would want to move file
+            run("chmod a-r " + path.str());
+        }
+        if (params.dirWithNoWritePermission && i == 9) {
+            run("chmod a-w " + path.str());
+        }
+    }
+}
+
+namespace {
+    struct BucketDataFound {
+        uint16_t _disk;
+        bool _checked;
+
+        BucketDataFound() : _disk(65535), _checked(false) {}
+        BucketDataFound(uint32_t disk) : _disk(disk), _checked(false) {}
+    };
+}
+
+void
+FileScannerTest::runTest(const TestParameters& params)
+{
+    std::vector<BucketData> buckets(createBuckets(params));
+    std::vector<std::string> tempFiles;
+    std::vector<std::string> alienFiles;
+    createData(params, buckets, tempFiles, alienFiles);
+
+    framework::defaultimplementation::RealClock clock;
+    framework::defaultimplementation::ComponentRegisterImpl compReg;
+    compReg.setClock(clock);
+
+    MountPointList mountPoints("./vdsroot",
+                               std::vector<vespalib::string>(),
+                               vespalib::LinkedPtr<DeviceManager>(
+                                       new DeviceManager(
+                                               DeviceMapper::UP(new SimpleDeviceMapper),
+                                               clock)));
+    mountPoints.init(params.diskCount);
+
+    FileScanner scanner(compReg, mountPoints,
+                        params.dirLevels, params.dirSpread);
+    std::map<document::BucketId, BucketDataFound> foundBuckets;
+    uint32_t extraBucketsSameDisk = 0;
+    uint32_t extraBucketsOtherDisk = 0;
+    for (uint32_t j=0; j<params.diskCount; ++j) {
+        // std::cerr << "Disk " << j << "\n";
+        if (params.disksDown.find(j) != params.disksDown.end()) continue;
+        for (uint32_t i=0; i<params.parts; ++i) {
+           document::BucketId::List bucketList;
+            try{
+                scanner.buildBucketList(bucketList, j, i, params.parts);
+                for (uint32_t k=0; k<bucketList.size(); ++k) {
+                    if (foundBuckets.find(bucketList[k]) != foundBuckets.end())
+                    {
+                        if (j == foundBuckets[bucketList[k]]._disk) {
+                            ++extraBucketsSameDisk;
+                        } else {
+                            ++extraBucketsOtherDisk;
+                        }
+//                        std::cerr << "Bucket " << bucketList[k]
+//                                  << " on disk " << j << " is already found on disk "
+//                                  << foundBuckets[bucketList[k]]._disk << ".\n";
+                    }
+                    foundBuckets[bucketList[k]] = BucketDataFound(j);
+                }
+            } catch (vespalib::IoException& e) {
+                if (!(params.dirWithNoListPermission
+                        && e.getType() == vespalib::IoException::NO_PERMISSION))
+                {
+                    throw;
+                }
+            }
+        }
+    }
+    std::vector<BucketData> notFound;
+    std::vector<BucketData> wasFound;
+    std::vector<BucketDataFound> foundNonExisting;
+        // Verify that found buckets match buckets expected.
+    for (uint32_t i=0; i<buckets.size(); ++i) {
+        std::map<document::BucketId, BucketDataFound>::iterator found(
+                foundBuckets.find(buckets[i].bucket));
+        if (buckets[i].shouldExist && found == foundBuckets.end()) {
+            notFound.push_back(buckets[i]);
+        } else if (!buckets[i].shouldExist && found != foundBuckets.end()) {
+            wasFound.push_back(buckets[i]);
+        }
+        if (found != foundBuckets.end()) { found->second._checked = true; }
+    }
+    for (std::map<document::BucketId, BucketDataFound>::iterator it
+            = foundBuckets.begin(); it != foundBuckets.end(); ++it)
+    {
+        if (!it->second._checked) {
+            foundNonExisting.push_back(it->second);
+        }
+    }
+    if (params.dirWithNoListPermission) {
+        CPPUNIT_ASSERT(!notFound.empty());
+    } else if (!notFound.empty()) {
+        std::ostringstream ost;
+        ost << "Failed to find " << notFound.size() << " of "
+            << buckets.size() << " buckets. Including buckets:";
+        for (uint32_t i=0; i<5 && i<notFound.size(); ++i) {
+            ost << " " << notFound[i].bucket;
+        }
+        CPPUNIT_FAIL(ost.str());
+    }
+    CPPUNIT_ASSERT(wasFound.empty());
+    CPPUNIT_ASSERT(foundNonExisting.empty());
+    if (params.bucketMultipleDirs) {
+        // TODO: Test something else here? This is not correct test, as when
+        // there are two buckets on the same disk, one of them will be ignored by
+        // the bucket lister.
+        // CPPUNIT_ASSERT_EQUAL(1u, extraBucketsSameDisk);
+    } else {
+        CPPUNIT_ASSERT_EQUAL(0u, extraBucketsSameDisk);
+    }
+    if (params.bucketMultipleDisks) {
+        CPPUNIT_ASSERT_EQUAL(1u, extraBucketsOtherDisk);
+    } else {
+        CPPUNIT_ASSERT_EQUAL(0u, extraBucketsOtherDisk);
+    }
+    if (params.addTemporaryFiles) {
+        CPPUNIT_ASSERT_EQUAL(
+                1, int(scanner.getMetrics()._temporaryFilesDeleted.getValue()));
+    } else {
+        CPPUNIT_ASSERT_EQUAL(
+                0, int(scanner.getMetrics()._temporaryFilesDeleted.getValue()));
+    }
+    if (params.addAlienFiles) {
+        CPPUNIT_ASSERT_EQUAL(
+                1, int(scanner.getMetrics()._alienFileCounter.getValue()));
+    } else {
+        CPPUNIT_ASSERT_EQUAL(
+                0, int(scanner.getMetrics()._alienFileCounter.getValue()));
+    }
+        // We automatically delete temporary files (created by VDS, indicating
+        // an operation that only half finished.
+    for (uint32_t i=0; i<tempFiles.size(); ++i) {
+        CPPUNIT_ASSERT_MSG(tempFiles[i], !vespalib::fileExists(tempFiles[i]));
+    }
+        // We don't automatically delete alien files
+    for (uint32_t i=0; i<alienFiles.size(); ++i) {
+        CPPUNIT_ASSERT_MSG(alienFiles[i], vespalib::fileExists(alienFiles[i]));
+    }
+}
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/mapper/.gitignore b/memfilepersistence/src/tests/mapper/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/memfilepersistence/src/tests/mapper/.gitignore
diff --git a/memfilepersistence/src/tests/spi/.gitignore b/memfilepersistence/src/tests/spi/.gitignore
new file mode 100644
index 00000000000..7e7c0fe7fae
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/.gitignore
@@ -0,0 +1,2 @@
+/.depend
+/Makefile
diff --git a/memfilepersistence/src/tests/spi/CMakeLists.txt b/memfilepersistence/src/tests/spi/CMakeLists.txt
new file mode 100644
index 00000000000..d5dade96f57
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/CMakeLists.txt
@@ -0,0 +1,20 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(memfilepersistence_testspi
+    SOURCES
+    memfiletestutils.cpp
+    providerconformancetest.cpp
+    memfilev1serializertest.cpp
+    memfilev1verifiertest.cpp
+    basicoperationhandlertest.cpp
+    splitoperationhandlertest.cpp
+    joinoperationhandlertest.cpp
+    iteratorhandlertest.cpp
+    memfiletest.cpp
+    memcachetest.cpp
+    simplememfileiobuffertest.cpp
+    memfileautorepairtest.cpp
+    shared_data_location_tracker_test.cpp
+    buffered_file_writer_test.cpp
+    buffer_test.cpp
+    DEPENDS
+)
diff --git a/memfilepersistence/src/tests/spi/basicoperationhandlertest.cpp b/memfilepersistence/src/tests/spi/basicoperationhandlertest.cpp
new file mode 100644
index 00000000000..2f7913b0e1f
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/basicoperationhandlertest.cpp
@@ -0,0 +1,735 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <tests/spi/memfiletestutils.h>
+#include <tests/spi/simulatedfailurefile.h>
+#include <tests/spi/options_builder.h>
+#include <vespa/document/fieldset/fieldsetrepo.h>
+#include <vespa/document/fieldset/fieldsets.h>
+
+namespace storage {
+namespace memfile {
+namespace {
+    spi::LoadType defaultLoadType(0, "default");
+}
+
+class BasicOperationHandlerTest : public SingleDiskMemFileTestUtils
+{
+    CPPUNIT_TEST_SUITE(BasicOperationHandlerTest);
+    CPPUNIT_TEST(testGetHeaderOnly);
+    CPPUNIT_TEST(testGetFieldFiltering);
+    CPPUNIT_TEST(testRemove);
+    CPPUNIT_TEST(testRemoveWithNonMatchingTimestamp);
+    CPPUNIT_TEST(testRemoveWithNonMatchingTimestampAlwaysPersist);
+    CPPUNIT_TEST(testRemoveForExistingRemoveSameTimestamp);
+    CPPUNIT_TEST(testRemoveForExistingRemoveNewTimestamp);
+    CPPUNIT_TEST(testRemoveForExistingRemoveNewTimestampAlwaysPersist);
+    CPPUNIT_TEST(testRemoveDocumentNotFound);
+    CPPUNIT_TEST(testRemoveDocumentNotFoundAlwaysPersist);
+    CPPUNIT_TEST(testRemoveExistingOlderDocumentVersion);
+    CPPUNIT_TEST(testPutSameTimestampAsRemove);
+    CPPUNIT_TEST(testUpdateBody);
+    CPPUNIT_TEST(testUpdateHeaderOnly);
+    CPPUNIT_TEST(testUpdateTimestampExists);
+    CPPUNIT_TEST(testUpdateForNonExistentDocWillFail);
+    CPPUNIT_TEST(testUpdateMayCreateDoc);
+    CPPUNIT_TEST(testRemoveEntry);
+    CPPUNIT_TEST(testEraseFromCacheOnFlushException);
+    CPPUNIT_TEST(testEraseFromCacheOnMaintainException);
+    CPPUNIT_TEST(testEraseFromCacheOnDeleteBucketException);
+    CPPUNIT_TEST_SUITE_END();
+
+    void doTestRemoveDocumentNotFound(
+            OperationHandler::RemoveType persistRemove);
+    void doTestRemoveWithNonMatchingTimestamp(
+            OperationHandler::RemoveType persistRemove);
+    void doTestRemoveForExistingRemoveNewTimestamp(
+            OperationHandler::RemoveType persistRemove);
+public:
+    void setupTestConfig();
+    void testPutHeadersOnly();
+    void testPutHeadersOnlyDocumentNotFound();
+    void testPutHeadersOnlyTimestampNotFound();
+    void testGetHeaderOnly();
+    void testGetFieldFiltering();
+    void testRemove();
+    void testRemoveWithNonMatchingTimestamp();
+    void testRemoveWithNonMatchingTimestampAlwaysPersist();
+    void testRemoveForExistingRemoveSameTimestamp();
+    void testRemoveForExistingRemoveNewTimestamp();
+    void testRemoveForExistingRemoveNewTimestampAlwaysPersist();
+    void testRemoveDocumentNotFound();
+    void testRemoveDocumentNotFoundAlwaysPersist();
+    void testRemoveExistingOlderDocumentVersion();
+    void testPutSameTimestampAsRemove();
+    void testUpdateBody();
+    void testUpdateHeaderOnly();
+    void testUpdateTimestampExists();
+    void testUpdateForNonExistentDocWillFail();
+    void testUpdateMayCreateDoc();
+    void testRemoveEntry();
+    void testEraseFromCacheOnFlushException();
+    void testEraseFromCacheOnMaintainException();
+    void testEraseFromCacheOnDeleteBucketException();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(BasicOperationHandlerTest);
+
+/**
+ * Test that doing a header-only get gives back a document containing
+ * only the document header
+ */
+void
+BasicOperationHandlerTest::testGetHeaderOnly()
+{
+    document::BucketId bucketId(16, 4);
+
+    Document::SP doc(createRandomDocumentAtLocation(4));
+    doc->setValue(doc->getField("hstringval"), document::StringFieldValue("hypnotoad"));
+    doc->setValue(doc->getField("headerval"), document::IntFieldValue(42));
+
+    doPut(doc, bucketId, Timestamp(4567), 0);
+    flush(bucketId);
+
+    spi::GetResult reply = doGet(bucketId, doc->getId(), document::HeaderFields());
+
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, reply.getErrorCode());
+    CPPUNIT_ASSERT(reply.hasDocument());
+    CPPUNIT_ASSERT_EQUAL(std::string("headerval: 42\nhstringval: hypnotoad\n"),
+                         stringifyFields(reply.getDocument()));
+    CPPUNIT_ASSERT_EQUAL(
+            size_t(1),
+            getPersistenceProvider().getMetrics().headerOnlyGets.getValue());
+}
+
+void
+BasicOperationHandlerTest::testGetFieldFiltering()
+{
+    document::BucketId bucketId(16, 4);
+    Document::SP doc(createRandomDocumentAtLocation(4));
+    doc->setValue(doc->getField("headerval"), document::IntFieldValue(42));
+    doc->setValue(doc->getField("hstringval"),
+                  document::StringFieldValue("groovy"));
+
+    document::FieldSetRepo repo;
+
+    doPut(doc, bucketId, Timestamp(4567), 0);
+    flush(bucketId);
+    spi::GetResult reply(doGet(bucketId,
+                               doc->getId(),
+                               *repo.parse(*getTypeRepo(), "testdoctype1:hstringval")));
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, reply.getErrorCode());
+    CPPUNIT_ASSERT(reply.hasDocument());
+    CPPUNIT_ASSERT_EQUAL(std::string("hstringval: groovy\n"),
+                         stringifyFields(reply.getDocument()));
+    CPPUNIT_ASSERT_EQUAL(
+            size_t(1),
+            getPersistenceProvider().getMetrics().headerOnlyGets.getValue());
+}
+
+void
+BasicOperationHandlerTest::testRemove()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+
+    document::Document::SP doc = doPut(4, Timestamp(1));
+
+    CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId,
+                                        doc->getId(),
+                                        Timestamp(2),
+                                        OperationHandler::PERSIST_REMOVE_IF_FOUND));
+
+    getPersistenceProvider().flush(
+            spi::Bucket(bucketId, spi::PartitionId(0)), context);
+
+    env()._cache.clear();
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL));
+
+    CPPUNIT_ASSERT_EQUAL(Timestamp(2), (*file)[1].getTimestamp());
+    CPPUNIT_ASSERT((*file)[1].deleted());
+    CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), (*file)[1].getLocation(BODY));
+    CPPUNIT_ASSERT_EQUAL((*file)[0].getLocation(HEADER),
+                         (*file)[1].getLocation(HEADER));
+}
+
+/**
+ * Test that removing a document with a max timestamp for which there
+ * is no matching document does not add a remove slot to the memfile
+ */
+void
+BasicOperationHandlerTest::doTestRemoveWithNonMatchingTimestamp(
+        OperationHandler::RemoveType persistRemove)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+    document::Document::SP doc = doPut(4, Timestamp(1234));
+
+    CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId,
+                                         doc->getId(),
+                                         Timestamp(1233),
+                                         persistRemove));
+
+    getPersistenceProvider().flush(
+            spi::Bucket(bucketId, spi::PartitionId(0)), context);
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(
+            uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE
+                     ? 2 : 1),
+            file->getSlotCount());
+
+    int i = 0;
+    if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) {
+        CPPUNIT_ASSERT_EQUAL(Timestamp(1233), (*file)[0].getTimestamp());
+        CPPUNIT_ASSERT((*file)[0].deleted());
+        CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), (*file)[0].getLocation(BODY));
+        CPPUNIT_ASSERT((*file)[0].getLocation(HEADER)
+                       != (*file)[1].getLocation(HEADER));
+        CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[0]));
+        ++i;
+    }
+
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[i].getTimestamp());
+    CPPUNIT_ASSERT(!(*file)[i].deleted());
+    CPPUNIT_ASSERT(file->getDocument((*file)[i], ALL)->getValue("content").get());
+}
+
+/**
+ * Test that removing a document with a max timestamp for which there
+ * is no matching document does not add a remove slot to the memfile
+ */
+void
+BasicOperationHandlerTest::testRemoveWithNonMatchingTimestamp()
+{
+    doTestRemoveWithNonMatchingTimestamp(
+            OperationHandler::PERSIST_REMOVE_IF_FOUND);
+}
+
+void
+BasicOperationHandlerTest::testRemoveWithNonMatchingTimestampAlwaysPersist()
+{
+    doTestRemoveWithNonMatchingTimestamp(
+            OperationHandler::ALWAYS_PERSIST_REMOVE);
+}
+
+/**
+ * Test that doing a remove with a timestamp for which there already
+ * exists a remove does not add another remove slot
+ */
+void
+BasicOperationHandlerTest::testRemoveForExistingRemoveSameTimestamp()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+    document::Document::SP doc = doPut(4, Timestamp(1234));
+
+    CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId,
+                                        doc->getId(),
+                                        Timestamp(1235),
+                                        OperationHandler::PERSIST_REMOVE_IF_FOUND));
+    CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId,
+                                         doc->getId(),
+                                         Timestamp(1235),
+                                         OperationHandler::PERSIST_REMOVE_IF_FOUND));
+
+    getPersistenceProvider().flush(
+            spi::Bucket(bucketId, spi::PartitionId(0)), context);
+
+    // Should only be one remove entry still
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get());
+
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp());
+    CPPUNIT_ASSERT((*file)[1].deleted());
+}
+
+void
+BasicOperationHandlerTest::doTestRemoveForExistingRemoveNewTimestamp(
+        OperationHandler::RemoveType persistRemove)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+    document::Document::SP doc = doPut(4, Timestamp(1234));
+
+    CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId,
+                                        doc->getId(),
+                                        Timestamp(1235),
+                                        OperationHandler::PERSIST_REMOVE_IF_FOUND));
+    CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId,
+                                        doc->getId(),
+                                        Timestamp(1236),
+                                        persistRemove));
+
+    getPersistenceProvider().flush(
+            spi::Bucket(bucketId, spi::PartitionId(0)), context);
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(
+            uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE
+                     ? 3 : 2),
+            file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get());
+
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp());
+    CPPUNIT_ASSERT((*file)[1].deleted());
+
+    if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) {
+        CPPUNIT_ASSERT_EQUAL(Timestamp(1236), (*file)[2].getTimestamp());
+        CPPUNIT_ASSERT((*file)[2].deleted());
+    }
+}
+
+/**
+ * Test that doing a second remove with a newer timestamp does not add
+ * another remove slot when PERSIST_REMOVE_IF_FOUND is specified
+ */
+void
+BasicOperationHandlerTest::testRemoveForExistingRemoveNewTimestamp()
+{
+    doTestRemoveForExistingRemoveNewTimestamp(
+            OperationHandler::PERSIST_REMOVE_IF_FOUND);
+}
+
+void
+BasicOperationHandlerTest::testRemoveForExistingRemoveNewTimestampAlwaysPersist()
+{
+    doTestRemoveForExistingRemoveNewTimestamp(
+            OperationHandler::ALWAYS_PERSIST_REMOVE);
+}
+
+/**
+ * Test removing an older version of a document. Older version should be removed
+ * in-place without attempting to add a new slot (which would fail).
+ */
+void
+BasicOperationHandlerTest::testRemoveExistingOlderDocumentVersion()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+    document::Document::SP doc = doPut(4, Timestamp(1234));
+
+    CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId,
+                                        doc->getId(),
+                                        Timestamp(1235),
+                                        OperationHandler::ALWAYS_PERSIST_REMOVE));
+
+    getPersistenceProvider().flush(
+            spi::Bucket(bucketId, spi::PartitionId(0)), context);
+
+    CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId,
+                                        doc->getId(),
+                                        Timestamp(1234),
+                                        OperationHandler::ALWAYS_PERSIST_REMOVE));
+
+    getPersistenceProvider().flush(
+            spi::Bucket(bucketId, spi::PartitionId(0)), context);
+
+    // Should now be two remove entries.
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[0]));
+    CPPUNIT_ASSERT((*file)[0].deleted());
+
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp());
+    CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[1]));
+    CPPUNIT_ASSERT((*file)[1].deleted());
+}
+
+void
+BasicOperationHandlerTest::doTestRemoveDocumentNotFound(
+        OperationHandler::RemoveType persistRemove)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+    document::DocumentId docId("userdoc:test:4:0");
+    doPut(4, Timestamp(1234));
+
+    CPPUNIT_ASSERT_EQUAL(false,
+                         doRemove(bucketId,
+                                  docId,
+                                  Timestamp(1235),
+                                  persistRemove));
+
+    getPersistenceProvider().flush(
+            spi::Bucket(bucketId, spi::PartitionId(0)), context);
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(
+            uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE
+                     ? 2 : 1),
+            file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp());
+    if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) {
+        CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp());
+        CPPUNIT_ASSERT((*file)[1].deleted());
+        CPPUNIT_ASSERT_EQUAL(docId, file->getDocumentId((*file)[1]));
+    }
+/* TODO: Test this in service layer tests.
+    CPPUNIT_ASSERT_EQUAL(
+            uint64_t(1),
+            env()._metrics.remove[documentapi::LoadType::DEFAULT].notFound.getValue());
+*/
+}
+
+/**
+ * Test that removing a non-existing document when PERSIST_EXISTING_ONLY is
+ * specified does not add a remove entry
+ */
+void
+BasicOperationHandlerTest::testRemoveDocumentNotFound()
+{
+    doTestRemoveDocumentNotFound(
+            OperationHandler::PERSIST_REMOVE_IF_FOUND);
+}
+
+void
+BasicOperationHandlerTest::testRemoveDocumentNotFoundAlwaysPersist()
+{
+    doTestRemoveDocumentNotFound(
+            OperationHandler::ALWAYS_PERSIST_REMOVE);
+}
+
+void
+BasicOperationHandlerTest::testPutSameTimestampAsRemove()
+{
+    document::BucketId bucketId(16, 4);
+
+    document::Document::SP doc = doPut(4, Timestamp(1234));
+
+    CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId,
+                                        doc->getId(),
+                                        Timestamp(1235),
+                                        OperationHandler::PERSIST_REMOVE_IF_FOUND));
+
+    // Flush here to avoid put+remove being thrown away by duplicate timestamp
+    // exception evicting the cache and unpersisted changes.
+    flush(bucketId);
+
+    doPut(4, Timestamp(1235));
+    flush(bucketId);
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get());
+
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp());
+    CPPUNIT_ASSERT((*file)[1].deleted());
+}
+
+/**
+ * Test that updating body results in a new memfile slot containing
+ * an updated document
+ */
+void
+BasicOperationHandlerTest::testUpdateBody()
+{
+    document::BucketId bucketId(16, 4);
+    document::StringFieldValue updateValue("foo");
+    document::Document::SP doc = doPut(4, Timestamp(1234));
+    document::Document originalDoc(*doc);
+
+    document::DocumentUpdate::SP update = createBodyUpdate(
+            doc->getId(), updateValue);
+
+    spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(5678));
+    flush(bucketId);
+    CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp());
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get());
+    CPPUNIT_ASSERT_EQUAL(*(originalDoc.getValue("content")),
+                         *file->getDocument((*file)[0], ALL)->getValue("content"));
+
+    CPPUNIT_ASSERT_EQUAL(Timestamp(5678), (*file)[1].getTimestamp());
+    CPPUNIT_ASSERT(file->getDocument((*file)[1], ALL)->getValue("content").get());
+    CPPUNIT_ASSERT_EQUAL(updateValue,
+                         dynamic_cast<document::StringFieldValue&>(
+                                 *file->getDocument((*file)[1], ALL)->getValue(
+                                         "content")));
+    CPPUNIT_ASSERT_EQUAL(
+            size_t(0),
+            getPersistenceProvider().getMetrics().headerOnlyUpdates.getValue());
+}
+
+void
+BasicOperationHandlerTest::testUpdateHeaderOnly()
+{
+    document::BucketId bucketId(16, 4);
+    document::IntFieldValue updateValue(42);
+    document::Document::SP doc = doPut(4, Timestamp(1234));
+
+    document::DocumentUpdate::SP update = createHeaderUpdate(
+            doc->getId(), updateValue);
+
+    spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(5678));
+    flush(bucketId);
+    CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp());
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("headerval").get() ==
+                   NULL);
+
+    CPPUNIT_ASSERT_EQUAL(Timestamp(5678), (*file)[1].getTimestamp());
+    CPPUNIT_ASSERT(file->getDocument((*file)[1], ALL)->getValue("headerval").get());
+    CPPUNIT_ASSERT_EQUAL(updateValue,
+                         dynamic_cast<document::IntFieldValue&>(
+                                 *file->getDocument((*file)[1], ALL)->getValue(
+                                         "headerval")));
+    CPPUNIT_ASSERT_EQUAL(
+            size_t(1),
+            getPersistenceProvider().getMetrics().headerOnlyUpdates.getValue());
+}
+
+void
+BasicOperationHandlerTest::testUpdateTimestampExists()
+{
+    document::BucketId bucketId(16, 4);
+    document::IntFieldValue updateValue(42);
+    document::Document::SP doc = doPut(4, Timestamp(1234));
+
+    document::DocumentUpdate::SP update = createHeaderUpdate(
+            doc->getId(), updateValue);
+
+    spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(1234));
+    flush(bucketId);
+    CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, result.getErrorCode());
+}
+
+void
+BasicOperationHandlerTest::testUpdateForNonExistentDocWillFail()
+{
+    document::BucketId bucketId(16, 4);
+    document::IntFieldValue updateValue(42);
+    Timestamp timestamp(5678);
+
+    // Is there an easier way to get a DocumentId?
+    document::Document::UP doc(
+            createRandomDocumentAtLocation(4, timestamp.getTime()));
+    const DocumentId& documentId = doc->getId();
+
+    document::DocumentUpdate::SP update = createHeaderUpdate(
+            documentId, updateValue);
+
+    spi::UpdateResult result = doUpdate(bucketId, update, timestamp);
+    flush(bucketId);
+    CPPUNIT_ASSERT_EQUAL(0, (int)result.getExistingTimestamp());
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(0), file->getSlotCount());
+}
+
+void
+BasicOperationHandlerTest::testUpdateMayCreateDoc()
+{
+    document::BucketId bucketId(16, 4);
+    document::IntFieldValue updateValue(42);
+    Timestamp timestamp(5678);
+
+    // Is there an easier way to get a DocumentId?
+    document::Document::UP doc(
+            createRandomDocumentAtLocation(4, timestamp.getTime()));
+    const DocumentId& documentId = doc->getId();
+
+    document::DocumentUpdate::SP update = createHeaderUpdate(
+            documentId, updateValue);
+    update->setCreateIfNonExistent(true);
+
+    spi::UpdateResult result = doUpdate(bucketId, update, timestamp);
+    flush(bucketId);
+    CPPUNIT_ASSERT_EQUAL(timestamp.getTime(),
+                         (uint64_t)result.getExistingTimestamp());
+
+    MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(timestamp, (*file)[0].getTimestamp());
+
+    auto headerval = file->getDocument((*file)[0], ALL)->getValue("headerval");
+    CPPUNIT_ASSERT(headerval.get() != nullptr);
+    CPPUNIT_ASSERT_EQUAL(updateValue,
+                         dynamic_cast<document::IntFieldValue&>(*headerval));
+}
+
+void
+BasicOperationHandlerTest::testRemoveEntry()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+
+    doPut(4, Timestamp(1234));
+    Document::SP doc = doPut(4, Timestamp(2345));
+    doPut(4, Timestamp(3456));
+
+    getPersistenceProvider().removeEntry(
+            spi::Bucket(bucketId, spi::PartitionId(0)),
+            spi::Timestamp(1234), context);
+    getPersistenceProvider().removeEntry(
+            spi::Bucket(bucketId, spi::PartitionId(0)),
+            spi::Timestamp(3456), context);
+    flush(bucketId);
+
+    memfile::MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(2345), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL));
+}
+
+void
+BasicOperationHandlerTest::setupTestConfig()
+{
+    using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig; 
+    using MemFileConfigBuilder
+        = vespa::config::storage::StorMemfilepersistenceConfigBuilder;
+    MemFileConfigBuilder builder(
+            *env().acquireConfigReadLock().memFilePersistenceConfig());
+    builder.minimumFileMetaSlots = 2;
+    builder.minimumFileHeaderBlockSize = 3000;
+    auto newConfig = std::unique_ptr<MemFileConfig>(new MemFileConfig(builder));
+    env().acquireConfigWriteLock().setMemFilePersistenceConfig(
+            std::move(newConfig));
+}
+
+void
+BasicOperationHandlerTest::testEraseFromCacheOnFlushException()
+{
+    document::BucketId bucketId(16, 4);
+
+    setupTestConfig();
+
+    document::Document::SP doc(
+            createRandomDocumentAtLocation(4, 2345, 1024, 1024));
+    doPut(doc, bucketId, Timestamp(2345));
+    flush(bucketId);
+    // Must throw out cache to re-create lazyfile
+    env()._cache.clear();
+
+    env()._lazyFileFactory =
+        std::unique_ptr<Environment::LazyFileFactory>(
+                new SimulatedFailureLazyFile::Factory);
+
+    // Try partial write, followed by full rewrite
+    for (int i = 0; i < 2; ++i) {
+        for (int j = 0; j < i+1; ++j) {
+            document::Document::SP doc2(
+                    createRandomDocumentAtLocation(4, 4000 + j, 1500, 1500));
+            doPut(doc2, bucketId, Timestamp(4000 + j));
+        }
+        spi::Result result = flush(bucketId);
+        CPPUNIT_ASSERT(result.hasError());
+        CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write")
+                       != vespalib::string::npos);
+
+        CPPUNIT_ASSERT(!env()._cache.contains(bucketId));
+
+        // Check that we still have first persisted put
+        memfile::MemFilePtr file(getMemFile(bucketId));
+        CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount());
+        CPPUNIT_ASSERT_EQUAL(Timestamp(2345), (*file)[0].getTimestamp());
+        CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL));
+    }
+}
+
+void
+BasicOperationHandlerTest::testEraseFromCacheOnMaintainException()
+{
+    document::BucketId bucketId(16, 4);
+
+    setupTestConfig();
+
+    getFakeClock()._absoluteTime = framework::MicroSecTime(2000 * 1000000);
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options)
+                .revertTimePeriod(framework::MicroSecTime(100000ULL * 1000000))
+                .build());
+    // Put a doc twice to allow for revert time compaction to be done
+    document::Document::SP doc1(
+            createRandomDocumentAtLocation(4, 2345, 1024, 1024));
+    document::Document::SP doc2(
+            createRandomDocumentAtLocation(4, 2345, 1024, 1024));
+    doPut(doc1, bucketId, Timestamp(1000 * 1000000));
+    doPut(doc2, bucketId, Timestamp(1500 * 1000000));
+    flush(bucketId);
+    env()._cache.clear();
+
+    options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options)
+                .revertTimePeriod(framework::MicroSecTime(100ULL * 1000000))
+                .build());
+
+    env()._lazyFileFactory =
+        std::unique_ptr<Environment::LazyFileFactory>(
+                new SimulatedFailureLazyFile::Factory);
+
+        spi::Result result = getPersistenceProvider().maintain(
+                spi::Bucket(bucketId, spi::PartitionId(0)),
+                spi::HIGH);
+        CPPUNIT_ASSERT(result.hasError());
+        CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write")
+                       != vespalib::string::npos);
+
+    CPPUNIT_ASSERT(!env()._cache.contains(bucketId));
+
+    // Check that we still have both persisted puts
+    memfile::MemFilePtr file(getMemFile(bucketId));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1000 * 1000000), (*file)[0].getTimestamp());
+    CPPUNIT_ASSERT_EQUAL(*doc1, *file->getDocument((*file)[0], ALL));
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1500 * 1000000), (*file)[1].getTimestamp());
+    CPPUNIT_ASSERT_EQUAL(*doc2, *file->getDocument((*file)[1], ALL));
+}
+
+void
+BasicOperationHandlerTest::testEraseFromCacheOnDeleteBucketException()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+    document::Document::SP doc(
+            createRandomDocumentAtLocation(4, 2345, 1024, 1024));
+    doPut(doc, bucketId, Timestamp(2345));
+    flush(bucketId);
+    env()._cache.clear();
+
+    SimulatedFailureLazyFile::Factory* factory(
+            new SimulatedFailureLazyFile::Factory);
+    factory->setReadOpsBeforeFailure(0);
+    env()._lazyFileFactory =
+        std::unique_ptr<Environment::LazyFileFactory>(factory);
+
+    // loadFile will fail
+    spi::Result result = getPersistenceProvider().deleteBucket(
+            spi::Bucket(bucketId, spi::PartitionId(0)), context);
+    CPPUNIT_ASSERT(result.hasError());
+    CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O read")
+                   != vespalib::string::npos);
+
+    CPPUNIT_ASSERT(!env()._cache.contains(bucketId));
+
+}
+
+}
+
+}
diff --git a/memfilepersistence/src/tests/spi/buffer_test.cpp b/memfilepersistence/src/tests/spi/buffer_test.cpp
new file mode 100644
index 00000000000..a2d917301fc
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/buffer_test.cpp
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/memfilepersistence/mapper/buffer.h>
+
+namespace storage {
+namespace memfile {
+
+class BufferTest : public CppUnit::TestFixture
+{
+public:
+    void getSizeReturnsInitiallyAllocatedSize();
+    void getSizeReturnsUnAlignedSizeForMMappedAllocs();
+    void resizeRetainsExistingDataWhenSizingUp();
+    void resizeRetainsExistingDataWhenSizingDown();
+    void bufferAddressIs512ByteAligned();
+
+    CPPUNIT_TEST_SUITE(BufferTest);
+    CPPUNIT_TEST(getSizeReturnsInitiallyAllocatedSize);
+    CPPUNIT_TEST(getSizeReturnsUnAlignedSizeForMMappedAllocs);
+    CPPUNIT_TEST(resizeRetainsExistingDataWhenSizingUp);
+    CPPUNIT_TEST(resizeRetainsExistingDataWhenSizingDown);
+    CPPUNIT_TEST(bufferAddressIs512ByteAligned);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(BufferTest);
+
+void
+BufferTest::getSizeReturnsInitiallyAllocatedSize()
+{
+    Buffer buf(1234);
+    CPPUNIT_ASSERT_EQUAL(size_t(1234), buf.getSize());
+}
+
+void
+BufferTest::getSizeReturnsUnAlignedSizeForMMappedAllocs()
+{
+    Buffer buf(vespalib::MMapAlloc::HUGEPAGE_SIZE + 1);
+    CPPUNIT_ASSERT_EQUAL(size_t(vespalib::MMapAlloc::HUGEPAGE_SIZE + 1),
+                         buf.getSize());
+}
+
+void
+BufferTest::resizeRetainsExistingDataWhenSizingUp()
+{
+    std::string src = "hello world";
+    Buffer buf(src.size());
+    memcpy(buf.getBuffer(), src.data(), src.size());
+    buf.resize(src.size() * 2);
+    CPPUNIT_ASSERT_EQUAL(src.size() * 2, buf.getSize());
+    CPPUNIT_ASSERT_EQUAL(0, memcmp(buf.getBuffer(), src.data(), src.size()));
+}
+
+void
+BufferTest::resizeRetainsExistingDataWhenSizingDown()
+{
+    std::string src = "hello world";
+    Buffer buf(src.size());
+    memcpy(buf.getBuffer(), src.data(), src.size());
+    buf.resize(src.size() / 2);
+    CPPUNIT_ASSERT_EQUAL(src.size() / 2, buf.getSize());
+    CPPUNIT_ASSERT_EQUAL(0, memcmp(buf.getBuffer(), src.data(), src.size() / 2));
+}
+
+void
+BufferTest::bufferAddressIs512ByteAligned()
+{
+    Buffer buf(32);
+    CPPUNIT_ASSERT(reinterpret_cast<size_t>(buf.getBuffer()) % 512 == 0);
+}
+
+} // memfile
+} // storage
+
diff --git a/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp b/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp
new file mode 100644
index 00000000000..b59e8a32258
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/memfilepersistence/mapper/bufferedfilewriter.h>
+#include <vespa/memfilepersistence/mapper/buffer.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+namespace storage {
+namespace memfile {
+
+class BufferedFileWriterTest : public CppUnit::TestFixture
+{
+public:
+    void noImplicitFlushingWhenDestructing();
+
+    CPPUNIT_TEST_SUITE(BufferedFileWriterTest);
+    CPPUNIT_TEST(noImplicitFlushingWhenDestructing);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(BufferedFileWriterTest);
+
+namespace {
+
+// Partial mock of vespalib::File. Unfortunately, there's currently no
+// base interface to implement so have to override a class that already has
+// implementation code present.
+class MockFile : public vespalib::File
+{
+public:
+    bool _didWrite;
+
+    MockFile(const std::string& filename)
+        : File(filename),
+          _didWrite(false)
+    {
+    }
+
+    void open(int flags, bool autoCreateDirectories) override {
+        (void) flags;
+        (void) autoCreateDirectories;
+        // Don't do anything here to prevent us from actually opening a file
+        // on disk.
+    }
+
+    off_t write(const void *buf, size_t bufsize, off_t offset) override {
+        (void) buf;
+        (void) bufsize;
+        (void) offset;
+        _didWrite = true;
+        return 0;
+    }
+};
+
+}
+
+void
+BufferedFileWriterTest::noImplicitFlushingWhenDestructing()
+{
+    MockFile file("foo");
+    {
+        Buffer buffer(1024);
+        BufferedFileWriter writer(file, buffer, buffer.getSize());
+        // Do a buffered write. This fits well within the buffer and should
+        // consequently not be immediately written out to the backing file.
+        writer.write("blarg", 5);
+        // Escape scope without having flushed anything.
+    }
+    // Since BufferedFileWriter is meant to be used with O_DIRECT files,
+    // flushing just implies writing rather than syncing (this is a half truth
+    // since you still sync directories etc to ensure metadata is written, but
+    // this constrained assumption works fine in the context of this test).
+    CPPUNIT_ASSERT(!file._didWrite);
+}
+
+} // memfile
+} // storage
+
diff --git a/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp b/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp
new file mode 100644
index 00000000000..6fea98e3c8e
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp
@@ -0,0 +1,940 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <set>
+#include <vector>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h>
+#include <tests/spi/memfiletestutils.h>
+#include <tests/spi/simulatedfailurefile.h>
+#include <tests/spi/options_builder.h>
+#include <vespa/document/fieldset/fieldsets.h>
+
+namespace storage {
+namespace memfile {
+namespace {
+    spi::LoadType defaultLoadType(0, "default");
+}
+
+class IteratorHandlerTest : public SingleDiskMemFileTestUtils
+{
+    CPPUNIT_TEST_SUITE(IteratorHandlerTest);
+    CPPUNIT_TEST(testCreateIterator);
+    CPPUNIT_TEST(testSomeSlotsRemovedBetweenInvocations);
+    CPPUNIT_TEST(testAllSlotsRemovedBetweenInvocations);
+    CPPUNIT_TEST(testIterateMetadataOnly);
+    CPPUNIT_TEST(testIterateHeadersOnly);
+    CPPUNIT_TEST(testIterateLargeDocument);
+    CPPUNIT_TEST(testDocumentsRemovedBetweenInvocations);
+    CPPUNIT_TEST(testUnrevertableRemoveBetweenInvocations);
+    CPPUNIT_TEST(testUnrevertableRemoveBetweenInvocationsIncludeRemoves);
+    CPPUNIT_TEST(testMatchTimestampRangeDocAltered);
+    CPPUNIT_TEST(testIterateAllVersions);
+    CPPUNIT_TEST(testFieldSetFiltering);
+    CPPUNIT_TEST(testIteratorInactiveOnException);
+    CPPUNIT_TEST(testDocsCachedBeforeDocumentSelection);
+    CPPUNIT_TEST(testTimestampRangeLimitedPrefetch);
+    CPPUNIT_TEST(testCachePrefetchRequirements);
+    CPPUNIT_TEST(testBucketEvictedFromCacheOnIterateException);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testCreateIterator();
+    void testSomeSlotsRemovedBetweenInvocations();
+    void testAllSlotsRemovedBetweenInvocations();
+    void testIterateMetadataOnly();
+    void testIterateHeadersOnly();
+    void testIterateLargeDocument();
+    void testDocumentsRemovedBetweenInvocations();
+    void testUnrevertableRemoveBetweenInvocations();
+    void testUnrevertableRemoveBetweenInvocationsIncludeRemoves();
+    void testMatchTimestampRangeDocAltered();
+    void testIterateAllVersions();
+    void testFieldSetFiltering();
+    void testIteratorInactiveOnException();
+    void testDocsCachedBeforeDocumentSelection();
+    void testTimestampRangeLimitedPrefetch();
+    void testCachePrefetchRequirements();
+    void testBucketEvictedFromCacheOnIterateException();
+
+    void setUp();
+    void tearDown();
+
+    struct Chunk
+    {
+        std::vector<spi::DocEntry::LP> _entries;
+    };
+
+private:
+    spi::Selection createSelection(const std::string& docSel) const;
+
+
+    spi::CreateIteratorResult create(
+            const spi::Bucket& b,
+            const spi::Selection& sel,
+            spi::IncludedVersions versions = spi::NEWEST_DOCUMENT_ONLY,
+            const document::FieldSet& fieldSet = document::AllFields())
+    {
+        spi::Context context(defaultLoadType, spi::Priority(0),
+                             spi::Trace::TraceLevel(0));
+        return getPersistenceProvider().createIterator(b, fieldSet, sel,
+                                                       versions, context);
+    }
+
+    typedef std::pair<Document::SP, spi::Timestamp> DocAndTimestamp;
+
+    std::vector<DocAndTimestamp> feedDocs(size_t numDocs,
+                                          uint32_t minSize = 110,
+                                          uint32_t maxSize = 110);
+
+    std::vector<Chunk> doIterate(spi::IteratorId id,
+                                 uint64_t maxByteSize,
+                                 size_t maxChunks = 0,
+                                 bool allowEmptyResult = false);
+
+    void verifyDocs(const std::vector<DocAndTimestamp>& wanted,
+                    const std::vector<IteratorHandlerTest::Chunk>& chunks,
+                    const std::set<vespalib::string>& removes
+                    = std::set<vespalib::string>()) const;
+
+    void doTestUnrevertableRemoveBetweenInvocations(bool includeRemoves);
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(IteratorHandlerTest);
+
+void
+IteratorHandlerTest::setUp()
+{
+    SingleDiskMemFileTestUtils::setUp();
+}
+
+void
+IteratorHandlerTest::tearDown()
+{
+    SingleDiskMemFileTestUtils::tearDown();
+}
+
+spi::Selection
+IteratorHandlerTest::createSelection(const std::string& docSel) const
+{
+    return spi::Selection(spi::DocumentSelection(docSel));
+}
+
+void
+IteratorHandlerTest::testCreateIterator()
+{
+    spi::Bucket b(BucketId(16, 1234), spi::PartitionId(0));
+
+    spi::CreateIteratorResult iter1(create(b, createSelection("true")));
+    CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1), iter1.getIteratorId());
+
+    spi::CreateIteratorResult iter2(create(b, createSelection("true")));
+    CPPUNIT_ASSERT_EQUAL(spi::IteratorId(2), iter2.getIteratorId());
+}
+
+std::vector<IteratorHandlerTest::Chunk>
+IteratorHandlerTest::doIterate(spi::IteratorId id,
+                               uint64_t maxByteSize,
+                               size_t maxChunks,
+                               bool allowEmptyResult)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    std::vector<Chunk> chunks;
+
+    while (true) {
+        std::vector<spi::DocEntry::LP> entries;
+
+        spi::IterateResult result(getPersistenceProvider().iterate(
+                id, maxByteSize, context));
+        CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode());
+        CPPUNIT_ASSERT(result.getEntries().size() > 0 || allowEmptyResult);
+
+        for (size_t i = 0; i < result.getEntries().size(); ++i) {
+            entries.push_back(result.getEntries()[i]);
+        }
+        chunks.push_back(Chunk());
+        chunks.back()._entries.swap(entries);
+        if (result.isCompleted()
+            || (maxChunks != 0 && chunks.size() >= maxChunks))
+        {
+            break;
+        }
+    }
+    return chunks;
+}
+
+namespace {
+
+size_t
+getDocCount(const std::vector<IteratorHandlerTest::Chunk>& chunks)
+{
+    size_t count = 0;
+    for (size_t i=0; i<chunks.size(); ++i) {
+        count += chunks[i]._entries.size();
+    }
+    return count;
+}
+
+size_t
+getRemoveEntryCount(const std::vector<spi::DocEntry::LP>& entries)
+{
+    size_t ret = 0;
+    for (size_t i = 0; i < entries.size(); ++i) {
+        if (entries[i]->isRemove()) {
+            ++ret;
+        }
+    }
+    return ret;
+}
+
+struct DocEntryIndirectTimestampComparator
+{
+    bool operator()(const spi::DocEntry::LP& e1,
+                    const spi::DocEntry::LP& e2) const
+    {
+        return e1->getTimestamp() < e2->getTimestamp();
+    }
+};
+
+std::vector<spi::DocEntry::LP>
+getEntriesFromChunks(const std::vector<IteratorHandlerTest::Chunk>& chunks)
+{
+    std::vector<spi::DocEntry::LP> ret;
+    for (size_t chunk = 0; chunk < chunks.size(); ++chunk) {
+        for (size_t i = 0; i < chunks[chunk]._entries.size(); ++i) {
+            ret.push_back(chunks[chunk]._entries[i]);
+        }
+    }
+    std::sort(ret.begin(),
+              ret.end(),
+              DocEntryIndirectTimestampComparator());
+    return ret;
+}
+
+const vespalib::LazyFile&
+getFileHandle(const MemFile& mf1)
+{
+    return static_cast<const SimpleMemFileIOBuffer&>(
+            mf1.getMemFileIO()).getFileHandle();
+}
+
+const LoggingLazyFile&
+getLoggerFile(const MemFile& file)
+{
+    return dynamic_cast<const LoggingLazyFile&>(getFileHandle(file));
+}
+
+}
+
+void
+IteratorHandlerTest::verifyDocs(const std::vector<DocAndTimestamp>& wanted,
+                                const std::vector<IteratorHandlerTest::Chunk>& chunks,
+                                const std::set<vespalib::string>& removes) const
+{
+    std::vector<spi::DocEntry::LP> retrieved(
+            getEntriesFromChunks(chunks));
+    size_t removeCount = getRemoveEntryCount(retrieved);
+    // Ensure that we've got the correct number of puts and removes
+    CPPUNIT_ASSERT_EQUAL(removes.size(), removeCount);
+    CPPUNIT_ASSERT_EQUAL(wanted.size(), retrieved.size() - removeCount);
+
+    size_t wantedIdx = 0;
+    for (size_t i = 0; i < retrieved.size(); ++i) {
+        spi::DocEntry& entry(*retrieved[i]);
+        if (entry.getDocument() != 0) {
+            if (!(*wanted[wantedIdx].first == *entry.getDocument())) {
+                std::ostringstream ss;
+                ss << "Documents differ! Wanted:\n"
+                   << wanted[wantedIdx].first->toString(true)
+                   << "\n\nGot:\n"
+                   << entry.getDocument()->toString(true);
+                CPPUNIT_FAIL(ss.str());
+            }
+            CPPUNIT_ASSERT_EQUAL(wanted[wantedIdx].second, entry.getTimestamp());
+            CPPUNIT_ASSERT_EQUAL(wanted[wantedIdx].first->serialize()->getLength()
+                                 + sizeof(spi::DocEntry),
+                                 size_t(entry.getSize()));
+            ++wantedIdx;
+        } else {
+            // Remove-entry
+            CPPUNIT_ASSERT(entry.getDocumentId() != 0);
+            CPPUNIT_ASSERT_EQUAL(entry.getDocumentId()->getSerializedSize()
+                                 + sizeof(spi::DocEntry),
+                                 size_t(entry.getSize()));
+            if (removes.find(entry.getDocumentId()->toString()) == removes.end()) {
+                std::ostringstream ss;
+                ss << "Got unexpected remove entry for document id "
+                   << *entry.getDocumentId();
+                CPPUNIT_FAIL(ss.str());
+            }
+        }
+    }
+}
+
+// Feed numDocs documents, starting from timestamp 1000
+std::vector<IteratorHandlerTest::DocAndTimestamp>
+IteratorHandlerTest::feedDocs(size_t numDocs,
+                              uint32_t minSize,
+                              uint32_t maxSize)
+{
+    std::vector<DocAndTimestamp> docs;
+    for (uint32_t i = 0; i < numDocs; ++i) {
+        docs.push_back(
+                DocAndTimestamp(
+                        doPut(4,
+                              framework::MicroSecTime(1000 + i),
+                              minSize,
+                              maxSize),
+                        spi::Timestamp(1000 + i)));
+    }
+    flush(document::BucketId(16, 4));
+    return docs;
+}
+
+void
+IteratorHandlerTest::testSomeSlotsRemovedBetweenInvocations()
+{
+    std::vector<DocAndTimestamp> docs = feedDocs(100, 4096, 4096);
+
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    spi::Selection sel(createSelection("true"));
+
+    spi::CreateIteratorResult iter(create(b, sel));
+    CPPUNIT_ASSERT(env()._cache.contains(b.getBucketId()));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 10000, 25);
+    CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size());
+
+    {
+        MemFilePtr file(getMemFile(b.getBucketId()));
+
+        for (int i = 0 ; i < 2; ++i) {
+            const MemSlot* slot = file->getSlotWithId(docs.front().first->getId());
+            CPPUNIT_ASSERT(slot != 0);
+            file->removeSlot(*slot);
+            docs.erase(docs.begin());
+        }
+        file->flushToDisk();
+    }
+
+    std::vector<Chunk> chunks2 = doIterate(iter.getIteratorId(), 10000);
+    CPPUNIT_ASSERT_EQUAL(size_t(24), chunks2.size());
+    std::copy(chunks2.begin(),
+              chunks2.end(),
+              std::back_insert_iterator<std::vector<Chunk> >(chunks));
+
+    verifyDocs(docs, chunks);
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+
+    // Bucket should not be evicted from cache during normal operation.
+    CPPUNIT_ASSERT(env()._cache.contains(b.getBucketId()));
+}
+
+void
+IteratorHandlerTest::testAllSlotsRemovedBetweenInvocations()
+{
+    std::vector<DocAndTimestamp> docs = feedDocs(100, 4096, 4096);
+
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    spi::Selection sel(createSelection("true"));
+
+    spi::CreateIteratorResult iter(create(b, sel));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 1, 25);
+    CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size());
+
+    {
+        MemFilePtr file(getMemFile(b.getBucketId()));
+
+        for (int i = 0 ; i < 75; ++i) {
+            const MemSlot* slot = file->getSlotWithId(docs[i].first->getId());
+            CPPUNIT_ASSERT(slot != 0);
+            file->removeSlot(*slot);
+        }
+        file->flushToDisk();
+        docs.erase(docs.begin(), docs.begin() + 75);
+    }
+
+    std::vector<Chunk> chunks2 = doIterate(iter.getIteratorId(), 1, 0, true);
+    CPPUNIT_ASSERT_EQUAL(size_t(0), getDocCount(chunks2));
+    verifyDocs(docs, chunks);
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+}
+
+void
+IteratorHandlerTest::testIterateMetadataOnly()
+{
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    std::vector<DocAndTimestamp> docs = feedDocs(10);
+
+    CPPUNIT_ASSERT(
+        doUnrevertableRemove(b.getBucketId(),
+                             docs[docs.size() - 2].first->getId(),
+                             Timestamp(1008)));
+
+    CPPUNIT_ASSERT(
+            doRemove(b.getBucketId(),
+                     docs[docs.size() - 1].first->getId(),
+                     framework::MicroSecTime(3001),
+                     OperationHandler::PERSIST_REMOVE_IF_FOUND));
+
+    flush(b.getBucketId());
+
+    spi::Selection sel(createSelection("true"));
+    spi::CreateIteratorResult iter(
+            create(b, sel, spi::NEWEST_DOCUMENT_OR_REMOVE, document::NoFields()));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 4096);
+    std::vector<spi::DocEntry::LP> entries = getEntriesFromChunks(chunks);
+    CPPUNIT_ASSERT_EQUAL(docs.size(), entries.size());
+    std::vector<DocAndTimestamp>::const_iterator docIter(
+            docs.begin());
+    for (size_t i = 0; i < entries.size(); ++i, ++docIter) {
+        const spi::DocEntry& entry = *entries[i];
+
+        CPPUNIT_ASSERT(entry.getDocument() == 0);
+        CPPUNIT_ASSERT(entry.getDocumentId() == 0);
+        if (i == 9) {
+            CPPUNIT_ASSERT(entry.isRemove());
+            CPPUNIT_ASSERT_EQUAL(spi::Timestamp(3001), entry.getTimestamp());
+        } else if (i == 8) {
+            CPPUNIT_ASSERT(entry.isRemove());
+            CPPUNIT_ASSERT_EQUAL(spi::Timestamp(1008), entry.getTimestamp());
+        } else {
+            CPPUNIT_ASSERT(!entry.isRemove());
+            CPPUNIT_ASSERT_EQUAL(docIter->second, entry.getTimestamp());
+        }
+    }
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+}
+
+void
+IteratorHandlerTest::testIterateHeadersOnly()
+{
+    std::vector<DocAndTimestamp> docs = feedDocs(20);
+    // Remove all bodies.
+    for (size_t i = 0; i < docs.size(); ++i) {
+        clearBody(*docs[i].first);
+    }
+
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    spi::Selection sel(createSelection("true"));
+
+    spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY,
+                                          document::HeaderFields()));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 1024);
+    verifyDocs(docs, chunks);
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+}
+
+void
+IteratorHandlerTest::testIterateLargeDocument()
+{
+    std::vector<DocAndTimestamp> docs = feedDocs(10, 10000, 10000);
+    std::vector<DocAndTimestamp> largedoc;
+    largedoc.push_back(docs.back());
+
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    spi::Selection sel(createSelection("true"));
+
+    spi::CreateIteratorResult iter(create(b, sel));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 100, 1);
+    verifyDocs(largedoc, chunks);
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+}
+
+void
+IteratorHandlerTest::testDocumentsRemovedBetweenInvocations()
+{
+    int docCount = 100;
+    std::vector<DocAndTimestamp> docs = feedDocs(docCount);
+
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    spi::Selection sel(createSelection("true"));
+
+    spi::CreateIteratorResult iter(create(b, sel));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 1, 25);
+    CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size());
+
+    // Remove a subset of the documents. We should still get all the
+    // original documents from the iterator, assuming no compactions.
+    std::vector<DocumentId> removedDocs;
+    std::vector<DocAndTimestamp> nonRemovedDocs;
+    for (int i = 0; i < docCount; ++i) {
+        if (i % 3 == 0) {
+            removedDocs.push_back(docs[i].first->getId());
+            CPPUNIT_ASSERT(doRemove(b.getBucketId(),
+                                    removedDocs.back(),
+                                    framework::MicroSecTime(2000 + i),
+                                    OperationHandler::PERSIST_REMOVE_IF_FOUND));
+        } else {
+            nonRemovedDocs.push_back(docs[i]);
+        }
+    }
+    flush(b.getBucketId());
+
+    std::vector<Chunk> chunks2 = doIterate(iter.getIteratorId(), 1);
+    CPPUNIT_ASSERT_EQUAL(size_t(75), chunks2.size());
+    std::copy(chunks2.begin(),
+              chunks2.end(),
+              std::back_insert_iterator<std::vector<Chunk> >(chunks));
+
+    verifyDocs(docs, chunks);
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+}
+
+void
+IteratorHandlerTest::doTestUnrevertableRemoveBetweenInvocations(bool includeRemoves)
+{
+    int docCount = 100;
+    std::vector<DocAndTimestamp> docs = feedDocs(docCount);
+
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    spi::Selection sel(createSelection("true"));
+    spi::CreateIteratorResult iter(
+            create(b, sel,
+                   includeRemoves ?
+                   spi::NEWEST_DOCUMENT_OR_REMOVE : spi::NEWEST_DOCUMENT_ONLY));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 1, 25);
+    CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size());
+
+    // Remove a subset of the documents unrevertably.
+    std::vector<DocumentId> removedDocs;
+    std::vector<DocAndTimestamp> nonRemovedDocs;
+    for (int i = 0; i < docCount - 25; ++i) {
+        if (i < 10) {
+            removedDocs.push_back(docs[i].first->getId());
+            CPPUNIT_ASSERT(
+                    doUnrevertableRemove(b.getBucketId(),
+                                         removedDocs.back(),
+                                         Timestamp(1000+i)));
+        } else {
+            nonRemovedDocs.push_back(docs[i]);
+        }
+    }
+    flush(b.getBucketId());
+
+    std::vector<Chunk> chunks2 = doIterate(iter.getIteratorId(), 1);
+    std::vector<spi::DocEntry::LP> entries = getEntriesFromChunks(chunks2);
+    if (!includeRemoves) {
+        CPPUNIT_ASSERT_EQUAL(nonRemovedDocs.size(), chunks2.size());
+        verifyDocs(nonRemovedDocs, chunks2);
+    } else {
+        CPPUNIT_ASSERT_EQUAL(size_t(75), entries.size());
+        for (int i = 0; i < docCount - 25; ++i) {
+            spi::DocEntry& entry(*entries[i]);
+            if (i < 10) {
+                CPPUNIT_ASSERT(entry.isRemove());
+            } else {
+                CPPUNIT_ASSERT(!entry.isRemove());
+            }
+        }
+    }
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+}
+
+void
+IteratorHandlerTest::testUnrevertableRemoveBetweenInvocations()
+{
+    doTestUnrevertableRemoveBetweenInvocations(false);
+}
+
+void
+IteratorHandlerTest::testUnrevertableRemoveBetweenInvocationsIncludeRemoves()
+{
+    doTestUnrevertableRemoveBetweenInvocations(true);
+}
+
+void
+IteratorHandlerTest::testMatchTimestampRangeDocAltered()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucketId(16, 4);
+    document::StringFieldValue updateValue1("update1");
+    document::StringFieldValue updateValue2("update2");
+
+    Document::SP originalDoc = doPut(4, Timestamp(1234));
+
+    {
+        document::DocumentUpdate::SP update = createBodyUpdate(
+                originalDoc->getId(), updateValue1);
+
+        spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(2345));
+        CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp());
+    }
+
+    {
+        document::DocumentUpdate::SP update = createBodyUpdate(
+                originalDoc->getId(), updateValue2);
+
+        spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(3456));
+        CPPUNIT_ASSERT_EQUAL(2345, (int)result.getExistingTimestamp());
+    }
+
+    CPPUNIT_ASSERT(
+            doRemove(bucketId,
+                     originalDoc->getId(),
+                     Timestamp(4567),
+                     OperationHandler::PERSIST_REMOVE_IF_FOUND));
+    flush(bucketId);
+
+    spi::Bucket b(bucketId, spi::PartitionId(0));
+
+    {
+        spi::Selection sel(createSelection("true"));
+        sel.setFromTimestamp(spi::Timestamp(0));
+        sel.setToTimestamp(spi::Timestamp(10));
+        spi::CreateIteratorResult iter(create(b, sel));
+
+        spi::IterateResult result(getPersistenceProvider().iterate(
+                iter.getIteratorId(), 4096, context));
+        CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode());
+        CPPUNIT_ASSERT_EQUAL(size_t(0), result.getEntries().size());
+        CPPUNIT_ASSERT(result.isCompleted());
+
+        getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+    }
+
+    {
+        spi::Selection sel(createSelection("true"));
+        sel.setFromTimestamp(spi::Timestamp(10000));
+        sel.setToTimestamp(spi::Timestamp(20000));
+        spi::CreateIteratorResult iter(create(b, sel));
+
+        spi::IterateResult result(getPersistenceProvider().iterate(
+                iter.getIteratorId(), 4096, context));
+        CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode());
+        CPPUNIT_ASSERT_EQUAL(size_t(0), result.getEntries().size());
+        CPPUNIT_ASSERT(result.isCompleted());
+
+        getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+    }
+
+    {
+        spi::Selection sel(createSelection("true"));
+        sel.setFromTimestamp(spi::Timestamp(0));
+        sel.setToTimestamp(spi::Timestamp(1234));
+        spi::CreateIteratorResult iter(create(b, sel));
+
+        spi::IterateResult result(getPersistenceProvider().iterate(
+                iter.getIteratorId(), 4096, context));
+        CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode());
+        CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size());
+        CPPUNIT_ASSERT(result.isCompleted());
+
+        const Document& receivedDoc(*result.getEntries()[0]->getDocument());
+        if (!(*originalDoc == receivedDoc)) {
+            std::ostringstream ss;
+            ss << "Documents differ! Wanted:\n"
+               << originalDoc->toString(true)
+               << "\n\nGot:\n"
+               << receivedDoc.toString(true);
+            CPPUNIT_FAIL(ss.str());
+        }
+
+        getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+    }
+
+    {
+        spi::Selection sel(createSelection("true"));
+        sel.setFromTimestamp(spi::Timestamp(0));
+        sel.setToTimestamp(spi::Timestamp(2345));
+        spi::CreateIteratorResult iter(create(b, sel));
+
+        spi::IterateResult result(getPersistenceProvider().iterate(
+                iter.getIteratorId(), 4096, context));
+        CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode());
+        CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size());
+        CPPUNIT_ASSERT(result.isCompleted());
+
+        const Document& receivedDoc(*result.getEntries()[0]->getDocument());
+        CPPUNIT_ASSERT(receivedDoc.getValue("content").get());
+        CPPUNIT_ASSERT_EQUAL(updateValue1,
+                             dynamic_cast<document::StringFieldValue&>(
+                                     *receivedDoc.getValue(
+                                             "content")));
+
+        getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+    }
+
+    {
+        spi::Selection sel(createSelection("true"));
+        sel.setFromTimestamp(spi::Timestamp(0));
+        sel.setToTimestamp(spi::Timestamp(3456));
+        spi::CreateIteratorResult iter(create(b, sel));
+
+        spi::IterateResult result(getPersistenceProvider().iterate(
+                iter.getIteratorId(), 4096, context));
+        CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode());
+        CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size());
+        CPPUNIT_ASSERT(result.isCompleted());
+
+        const Document& receivedDoc(*result.getEntries()[0]->getDocument());
+        CPPUNIT_ASSERT(receivedDoc.getValue("content").get());
+        CPPUNIT_ASSERT_EQUAL(updateValue2,
+                             dynamic_cast<document::StringFieldValue&>(
+                                     *receivedDoc.getValue(
+                                             "content")));
+
+        getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+    }
+}
+
+void
+IteratorHandlerTest::testIterateAllVersions()
+{
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    std::vector<DocAndTimestamp> docs;
+
+    Document::SP originalDoc(createRandomDocumentAtLocation(
+                                     4, 1001, 110, 110));
+
+    doPut(originalDoc, framework::MicroSecTime(1001), 0);
+
+    document::StringFieldValue updateValue1("update1");
+    {
+        document::DocumentUpdate::SP update = createBodyUpdate(
+                originalDoc->getId(), updateValue1);
+
+        spi::UpdateResult result = doUpdate(b.getBucketId(), update, Timestamp(2345));
+        CPPUNIT_ASSERT_EQUAL(1001, (int)result.getExistingTimestamp());
+    }
+    flush(b.getBucketId());
+
+    Document::SP updatedDoc(new Document(*originalDoc));
+    updatedDoc->setValue("content", document::StringFieldValue("update1"));
+    docs.push_back(DocAndTimestamp(originalDoc, spi::Timestamp(1001)));
+    docs.push_back(DocAndTimestamp(updatedDoc, spi::Timestamp(2345)));
+
+    spi::Selection sel(createSelection("true"));
+    spi::CreateIteratorResult iter(create(b, sel, spi::ALL_VERSIONS));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 4096);
+    verifyDocs(docs, chunks);
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+}
+
+void
+IteratorHandlerTest::testFieldSetFiltering()
+{
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    Document::SP doc(createRandomDocumentAtLocation(
+                             4, 1001, 110, 110));
+    doc->setValue(doc->getField("headerval"), document::IntFieldValue(42));
+    doc->setValue(doc->getField("hstringval"),
+                  document::StringFieldValue("groovy, baby!"));
+    doc->setValue(doc->getField("content"),
+                  document::StringFieldValue("fancy content"));
+    doPut(doc, framework::MicroSecTime(1001), 0);
+    flush(b.getBucketId());
+
+    document::FieldSetRepo repo;
+    spi::Selection sel(createSelection("true"));
+    spi::CreateIteratorResult iter(
+            create(b, sel, spi::NEWEST_DOCUMENT_ONLY,
+                   *repo.parse(*getTypeRepo(), "testdoctype1:hstringval,content")));
+    std::vector<spi::DocEntry::LP> entries(
+            getEntriesFromChunks(doIterate(iter.getIteratorId(), 4096)));
+    CPPUNIT_ASSERT_EQUAL(size_t(1), entries.size());
+    CPPUNIT_ASSERT_EQUAL(std::string("content: fancy content\n"
+                                     "hstringval: groovy, baby!\n"),
+                         stringifyFields(*entries[0]->getDocument()));
+}
+
+void
+IteratorHandlerTest::testIteratorInactiveOnException()
+{
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    feedDocs(10);
+
+    env()._cache.clear();
+
+    simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1));
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    spi::CreateIteratorResult iter(create(b, createSelection("true")));
+    spi::IterateResult result(getPersistenceProvider().iterate(
+                                      iter.getIteratorId(), 100000, context));
+    CPPUNIT_ASSERT(result.hasError());
+    // Check that iterator is marked as inactive
+    const SharedIteratorHandlerState& state(
+            getPersistenceProvider().getIteratorHandler().getState());
+    CPPUNIT_ASSERT(state._iterators.find(iter.getIteratorId().getValue())
+                   != state._iterators.end());
+    CPPUNIT_ASSERT(state._iterators.find(iter.getIteratorId().getValue())
+                   ->second.isActive() == false);
+
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+}
+
+void
+IteratorHandlerTest::testDocsCachedBeforeDocumentSelection()
+{
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    std::vector<DocAndTimestamp> docs = feedDocs(100, 4096, 4096);
+
+    env()._cache.clear();
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options).maximumReadThroughGap(1024*1024).build());
+    env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>(
+            new LoggingLazyFile::Factory());
+
+    spi::Selection sel(createSelection("id.user=4"));
+    spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY,
+                                          document::BodyFields()));
+
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 4096);
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+    {
+        MemFilePtr file(getMemFile(b.getBucketId()));
+        // Should have 3 read ops; metadata, (precached) headers and bodies
+        CPPUNIT_ASSERT_EQUAL(size_t(3),
+                             getLoggerFile(*file).operations.size());
+    }
+}
+
+void
+IteratorHandlerTest::testTimestampRangeLimitedPrefetch()
+{
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    // Feed docs with timestamp range [1000, 1100)
+    feedDocs(100, 4096, 4096);
+
+    env()._cache.clear();
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options).maximumReadThroughGap(512).build());
+    env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>(
+            new LoggingLazyFile::Factory());
+
+    spi::Selection sel(createSelection("id.user=4"));
+    sel.setFromTimestamp(spi::Timestamp(1050));
+    sel.setToTimestamp(spi::Timestamp(1059));
+    spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY,
+                                          document::BodyFields()));
+    std::vector<Chunk> chunks = doIterate(iter.getIteratorId(), 4096);
+    CPPUNIT_ASSERT_EQUAL(size_t(10), getDocCount(chunks));
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().destroyIterator(iter.getIteratorId(), context);
+    // Iterate over all slots, ensuring that only those that fall within the
+    // timestamp range have actually been cached.
+    {
+        MemFilePtr file(getMemFile(b.getBucketId()));
+        // Should have 3 read ops; metadata, (precached) headers and bodies
+        CPPUNIT_ASSERT_EQUAL(size_t(3),
+                             getLoggerFile(*file).operations.size());
+        for (size_t i = 0; i < file->getSlotCount(); ++i) {
+            const MemSlot& slot((*file)[i]);
+            if (slot.getTimestamp() >= Timestamp(1050)
+                && slot.getTimestamp() <= Timestamp(1059))
+            {
+                CPPUNIT_ASSERT(file->partAvailable(slot, HEADER));
+                CPPUNIT_ASSERT(file->partAvailable(slot, BODY));
+            } else {
+                CPPUNIT_ASSERT(!file->partAvailable(slot, HEADER));
+                CPPUNIT_ASSERT(!file->partAvailable(slot, BODY));
+            }
+        }
+    }
+}
+
+void
+IteratorHandlerTest::testCachePrefetchRequirements()
+{
+    document::select::Parser parser(
+            env().repo(), env()._bucketFactory);
+    {
+        // No prefetch required.
+        // NOTE: since stuff like id.user=1234 won't work, we have to handle
+        // that explicitly in createIterator based on the assumption that a
+        // non-empty document selection at _least_ requires header to be read.
+        std::unique_ptr<document::select::Node> sel(
+                parser.parse("true"));
+        CachePrefetchRequirements req(
+                CachePrefetchRequirements::createFromSelection(env().repo(),
+                *sel));
+        CPPUNIT_ASSERT(!req.isHeaderPrefetchRequired());
+        CPPUNIT_ASSERT(!req.isBodyPrefetchRequired());
+    }
+
+    {
+        // Header prefetch required.
+        std::unique_ptr<document::select::Node> sel(
+                parser.parse("testdoctype1.hstringval='blarg'"));
+        CachePrefetchRequirements req(
+                CachePrefetchRequirements::createFromSelection(env().repo(),
+                *sel));
+        CPPUNIT_ASSERT(req.isHeaderPrefetchRequired());
+        CPPUNIT_ASSERT(!req.isBodyPrefetchRequired());
+    }
+
+    {
+        // Body prefetch required.
+        std::unique_ptr<document::select::Node> sel(
+                parser.parse("testdoctype1.content='foobar'"));
+        CachePrefetchRequirements req(
+                CachePrefetchRequirements::createFromSelection(env().repo(),
+                *sel));
+        CPPUNIT_ASSERT(!req.isHeaderPrefetchRequired());
+        CPPUNIT_ASSERT(req.isBodyPrefetchRequired());
+    }
+}
+
+void
+IteratorHandlerTest::testBucketEvictedFromCacheOnIterateException()
+{
+    spi::Bucket b(BucketId(16, 4), spi::PartitionId(0));
+    feedDocs(10);
+    env()._cache.clear();
+
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    spi::CreateIteratorResult iter(create(b, createSelection("true")));
+    simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1));
+    spi::IterateResult result(getPersistenceProvider().iterate(
+                                      iter.getIteratorId(), 100000, context));
+    CPPUNIT_ASSERT(result.hasError());
+
+    // This test is actually a bit disingenuous since calling iterate will
+    // implicitly invoke maintain() on an IO exception, which will subsequently
+    // evict the bucket due to the exception happening again in its context.
+    CPPUNIT_ASSERT(!env()._cache.contains(b.getBucketId()));
+}
+
+}
+}
diff --git a/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp b/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp
new file mode 100644
index 00000000000..78601b461ab
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp
@@ -0,0 +1,504 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/datatype/documenttype.h>
+#include <tests/spi/memfiletestutils.h>
+#include <tests/spi/simulatedfailurefile.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+using document::DocumentType;
+
+namespace storage {
+namespace memfile {
+namespace {
+    spi::LoadType defaultLoadType(0, "default");
+}
+
+class JoinOperationHandlerTest : public MemFileTestUtils
+{
+    CPPUNIT_TEST_SUITE(JoinOperationHandlerTest);
+    CPPUNIT_TEST(testSimple);
+    CPPUNIT_TEST(testTargetExists);
+    CPPUNIT_TEST(testTargetWithOverlap);
+    CPPUNIT_TEST(testMultiDisk);
+    CPPUNIT_TEST(testMultiDiskFlushed);
+    CPPUNIT_TEST(testInternalJoin);
+    CPPUNIT_TEST(testInternalJoinDiskFull);
+    CPPUNIT_TEST(testTargetIoWriteExceptionEvictsTargetFromCache);
+    CPPUNIT_TEST(test1stSourceIoReadExceptionEvictsSourceFromCache);
+    CPPUNIT_TEST(test2ndSourceExceptionEvictsExistingTargetFromCache);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testSimple();
+    void testTargetExists();
+    void testTargetWithOverlap();
+    void testMultiDisk();
+    void testMultiDiskFlushed();
+    void testInternalJoin();
+    void testInternalJoinDiskFull();
+    void testTargetIoWriteExceptionEvictsTargetFromCache();
+    void test1stSourceIoReadExceptionEvictsSourceFromCache();
+    void test2ndSourceExceptionEvictsExistingTargetFromCache();
+
+    void insertDocumentInBucket(uint64_t location,
+                                Timestamp timestamp,
+                                document::BucketId bucket);
+
+private:
+    void feedSingleDisk();
+    void feedMultiDisk();
+    std::string getStandardMemFileStatus(uint32_t disk = 0);
+
+    spi::Result doJoin(const document::BucketId to,
+                       const document::BucketId from1,
+                       const document::BucketId from2);
+};
+
+namespace {
+
+document::BucketId TARGET = document::BucketId(15, 4);
+document::BucketId SOURCE1 = document::BucketId(16, 4);
+document::BucketId SOURCE2 = document::BucketId(16, (uint64_t)4 | ((uint64_t)1 << 15));
+}
+
+CPPUNIT_TEST_SUITE_REGISTRATION(JoinOperationHandlerTest);
+
+void
+JoinOperationHandlerTest::feedSingleDisk()
+{
+    for (uint32_t i = 0; i < 100; i++) {
+        std::ostringstream ost;
+        ost << "userdoc:storage_test:1234:" << i;
+        const DocumentType& type(
+                *getTypeRepo()->getDocumentType("testdoctype1"));
+        document::Document::SP doc(
+                new document::Document(type, document::DocumentId(ost.str())));
+
+        document::BucketId bucket(
+                getBucketIdFactory().getBucketId(doc->getId()));
+        bucket.setUsedBits(33);
+        doPut(doc, Timestamp(1000 + i), 0, 33);
+        flush(bucket);
+    }
+}
+
+void
+JoinOperationHandlerTest::feedMultiDisk()
+{
+    for (uint32_t i = 0; i < 100; i += 2) {
+        doPutOnDisk(7, 4 | (1 << 15), Timestamp(1000 + i));
+    }
+    flush(SOURCE2);
+
+    for (uint32_t i = 1; i < 100; i += 2) {
+        doPutOnDisk(4, 4, Timestamp(1000 + i));
+    }
+    flush(SOURCE1);
+
+    {
+        MemFilePtr file(getMemFile(SOURCE1, 4));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+        CPPUNIT_ASSERT_EQUAL(4, (int)file->getDisk());
+    }
+
+    {
+        MemFilePtr file(getMemFile(SOURCE2, 7));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+        CPPUNIT_ASSERT_EQUAL(7, (int)file->getDisk());
+    }
+}
+
+std::string
+JoinOperationHandlerTest::getStandardMemFileStatus(uint32_t disk)
+{
+    std::ostringstream ost;
+
+    ost << getMemFileStatus(TARGET, disk) << "\n"
+        << getMemFileStatus(SOURCE1, disk ) << "\n"
+        << getMemFileStatus(SOURCE2, disk) << "\n";
+
+    return ost.str();
+}
+
+void
+JoinOperationHandlerTest::insertDocumentInBucket(
+        uint64_t location,
+        Timestamp timestamp,
+        document::BucketId bucket)
+{
+    Document::SP doc(
+            createRandomDocumentAtLocation(
+                    location, timestamp.getTime(), 100, 100));
+    doPut(doc, bucket, timestamp);
+}
+
+spi::Result
+JoinOperationHandlerTest::doJoin(const document::BucketId to,
+                                 const document::BucketId from1,
+                                 const document::BucketId from2)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    return getPersistenceProvider().join(
+            spi::Bucket(from1, spi::PartitionId(0)),
+            spi::Bucket(from2, spi::PartitionId(0)),
+            spi::Bucket(to, spi::PartitionId(0)),
+            context);
+}
+
+void
+JoinOperationHandlerTest::testSimple()
+{
+    setupDisks(1);
+    feedSingleDisk();
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(33, 1234)));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+    }
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(33, (uint64_t)1234 | ((uint64_t)1 << 32))));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+    }
+
+    spi::Result result =
+        doJoin(document::BucketId(32, 1234),
+               document::BucketId(33, 1234),
+               document::BucketId(33, (uint64_t)1234 | ((uint64_t)1 << 32)));
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(32, (uint64_t)1234)));
+        CPPUNIT_ASSERT_EQUAL(100, (int)file->getSlotCount());
+        CPPUNIT_ASSERT(!file->slotsAltered());
+    }
+}
+
+void
+JoinOperationHandlerTest::testTargetExists()
+{
+    setupDisks(1);
+
+    for (uint32_t i = 0; i < 100; i += 2) {
+        doPut(4 | (1 << 15), Timestamp(1000 + i));
+    }
+    flush(SOURCE2);
+
+    for (uint32_t i = 1; i < 100; i += 2) {
+        doPut(4, Timestamp(1000 + i));
+    }
+    flush(SOURCE1);
+
+    for (uint32_t i = 0; i < 100; i++) {
+        uint32_t location = 4;
+        if (i % 2 == 0) {
+            location |= (1 << 15);
+        }
+
+        insertDocumentInBucket(location, Timestamp(500 + i), TARGET);
+    }
+    flush(TARGET);
+
+    doJoin(TARGET, SOURCE1, SOURCE2);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x3c00000000000004): 200,0\n"
+                    "BucketId(0x4000000000000004): 0,0\n"
+                    "BucketId(0x4000000000008004): 0,0\n"),
+            getStandardMemFileStatus());
+}
+
+void
+JoinOperationHandlerTest::testTargetWithOverlap()
+{
+    setupDisks(1);
+
+    for (uint32_t i = 0; i < 100; i += 2) {
+        doPut(4 | (1 << 15), Timestamp(1000 + i));
+    }
+    flush(SOURCE2);
+
+    for (uint32_t i = 1; i < 100; i += 2) {
+        doPut(4, Timestamp(1000 + i));
+    }
+    flush(SOURCE1);
+
+    for (uint32_t i = 0; i < 100; i++) {
+        uint32_t location = 4;
+        if (i % 2 == 0) {
+            location |= (1 << 15);
+        }
+
+        insertDocumentInBucket(location, Timestamp(950 + i), TARGET);
+    }
+    flush(TARGET);
+
+    doJoin(TARGET, SOURCE1, SOURCE2);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x3c00000000000004): 150,0\n"
+                    "BucketId(0x4000000000000004): 0,0\n"
+                    "BucketId(0x4000000000008004): 0,0\n"),
+            getStandardMemFileStatus());
+}
+
+void
+JoinOperationHandlerTest::testMultiDisk()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    setupDisks(10);
+    feedMultiDisk();
+
+    getPersistenceProvider().join(spi::Bucket(SOURCE2, spi::PartitionId(7)),
+                                  spi::Bucket(SOURCE1, spi::PartitionId(4)),
+                                  spi::Bucket(TARGET, spi::PartitionId(3)),
+                                  context);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x3c00000000000004): 100,3\n"
+                    "BucketId(0x4000000000000004): 0,0\n"
+                    "BucketId(0x4000000000008004): 0,0\n"),
+            getStandardMemFileStatus());
+}
+
+void
+JoinOperationHandlerTest::testMultiDiskFlushed()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    setupDisks(10);
+    feedMultiDisk();
+
+    // Flush everything to disk, to check that we can join even
+    // if it's not in cache before.
+    env()._cache.flushDirtyEntries();
+    env()._cache.clear();
+
+    getPersistenceProvider().join(spi::Bucket(SOURCE2, spi::PartitionId(7)),
+                                  spi::Bucket(SOURCE1, spi::PartitionId(4)),
+                                  spi::Bucket(TARGET, spi::PartitionId(3)),
+                                  context);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x3c00000000000004): 100,3\n"
+                    "BucketId(0x4000000000000004): 0,3\n"
+                    "BucketId(0x4000000000008004): 0,3\n"),
+            getStandardMemFileStatus(3));
+}
+
+void
+JoinOperationHandlerTest::testInternalJoin()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    setupDisks(10);
+
+    for (uint32_t i = 4; i < 6; i++) {
+        for (uint32_t j = 0; j < 10; j++) {
+            uint32_t location = 4;
+            doPutOnDisk(i, location, Timestamp(i * 1000 + j));
+        }
+        flush(document::BucketId(16, 4), i);
+        env()._cache.clear();
+    }
+
+    std::string fileName1 =
+        env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[4]);
+    std::string fileName2 =
+        env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[5]);
+
+    CPPUNIT_ASSERT(vespalib::stat(fileName1).get());
+    vespalib::FileInfo::UP file2(vespalib::stat(fileName2));
+
+    CPPUNIT_ASSERT(file2.get());
+    CPPUNIT_ASSERT(file2->_size > 0);
+
+    PartitionMonitor* mon = env().getDirectory(5).getPartition().getMonitor();
+    // Set disk under 80% full. Over 80%, we shouldn't move buckets to the target.
+    mon->setStatOncePolicy();
+    mon->overrideRealStat(512, 100000, 50000);
+    CPPUNIT_ASSERT(!mon->isFull(0, .80f));
+
+    getPersistenceProvider().join(spi::Bucket(SOURCE1, spi::PartitionId(4)),
+                                  spi::Bucket(SOURCE1, spi::PartitionId(4)),
+                                  spi::Bucket(SOURCE1, spi::PartitionId(5)),
+                                  context);
+
+    env()._cache.clear();
+
+    CPPUNIT_ASSERT(!vespalib::stat(fileName1).get());
+    CPPUNIT_ASSERT(vespalib::stat(fileName2).get());
+}
+
+void
+JoinOperationHandlerTest::testInternalJoinDiskFull()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    setupDisks(10);
+
+    for (uint32_t i = 4; i < 6; i++) {
+        for (uint32_t j = 0; j < 10; j++) {
+            uint32_t location = 4;
+            doPutOnDisk(i, location, Timestamp(i * 1000 + j));
+        }
+        flush(document::BucketId(16, 4), i);
+        env()._cache.clear();
+    }
+
+    std::string fileName1 =
+        env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[4]);
+    std::string fileName2 =
+        env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[5]);
+
+    CPPUNIT_ASSERT(vespalib::stat(fileName1).get());
+    vespalib::FileInfo::UP file2(vespalib::stat(fileName2));
+
+    CPPUNIT_ASSERT(file2.get());
+    CPPUNIT_ASSERT(file2->_size > 0);
+
+    PartitionMonitor* mon = env().getDirectory(5).getPartition().getMonitor();
+    // Set disk to 81% full. Over 80%, we shouldn't move buckets to the target.
+    mon->setStatOncePolicy();
+    mon->overrideRealStat(512, 100000, 81000);
+    CPPUNIT_ASSERT(!mon->isFull());
+    CPPUNIT_ASSERT(mon->isFull(0, .08f));
+
+    spi::Result result =
+        getPersistenceProvider().join(spi::Bucket(SOURCE1, spi::PartitionId(4)),
+                                      spi::Bucket(SOURCE1, spi::PartitionId(4)),
+                                      spi::Bucket(SOURCE1, spi::PartitionId(5)),
+                                      context);
+
+    CPPUNIT_ASSERT(result.hasError());
+}
+
+void
+JoinOperationHandlerTest::testTargetIoWriteExceptionEvictsTargetFromCache()
+{
+    setupDisks(1);
+    feedSingleDisk();
+
+    document::BucketId src1(33, 1234);
+    document::BucketId src2(33, 1234ULL | (1ULL << 32));
+    document::BucketId target(32, 1234);
+
+    CPPUNIT_ASSERT(env()._cache.contains(src1));
+    CPPUNIT_ASSERT(env()._cache.contains(src2));
+    CPPUNIT_ASSERT(!env()._cache.contains(target));
+
+    // Reading existing (fully cached) files will go fine, but writing
+    // new file will not.
+    simulateIoErrorsForSubsequentlyOpenedFiles();
+
+    spi::Result result = doJoin(target, src1, src2);
+    CPPUNIT_ASSERT(result.hasError());
+    CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write")
+                   != vespalib::string::npos);
+
+    CPPUNIT_ASSERT(!env()._cache.contains(target));
+    // NOTE: since we end up renaming src1 -> target during the first
+    // iteration of join, src1 will actually be empty. This should not
+    // matter since the service layer will query the bucket info for
+    // all these afterwards and will thus pick up on this automatically.
+    unSimulateIoErrorsForSubsequentlyOpenedFiles();
+    {
+        MemFilePtr file(getMemFile(src1));
+        CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount());
+        CPPUNIT_ASSERT(!file->slotsAltered());
+    }
+    {
+        MemFilePtr file(getMemFile(src2));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+        CPPUNIT_ASSERT(!file->slotsAltered());
+    }
+    {
+        MemFilePtr file(getMemFile(target));
+        // Renamed from src1
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+        CPPUNIT_ASSERT(!file->slotsAltered());
+    }
+}
+
+void
+JoinOperationHandlerTest::test1stSourceIoReadExceptionEvictsSourceFromCache()
+{
+    setupDisks(1);
+    feedSingleDisk();
+
+    document::BucketId src1(33, 1234);
+    document::BucketId src2(33, 1234ULL | (1ULL << 32));
+    document::BucketId target(32, 1234);
+
+    env()._cache.clear();
+    // Allow for reading in initial metadata so that loadFile itself doesn't
+    // fail. This could otherwise cause a false negative since that happens
+    // during initial cache lookup on a cache miss, at which point any
+    // exception will always stop a file from being added to the cache. Here
+    // we want to test the case where a file has been successfully hoisted
+    // out of the cache initially.
+    simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1));
+
+    spi::Result result = doJoin(target, src1, src2);
+    CPPUNIT_ASSERT(result.hasError());
+    CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O read")
+                   != vespalib::string::npos);
+
+    CPPUNIT_ASSERT(!env()._cache.contains(src1));
+    CPPUNIT_ASSERT(!env()._cache.contains(src2));
+    CPPUNIT_ASSERT(!env()._cache.contains(target));
+}
+
+/**
+ * It must be exception safe for any source bucket to throw an exception during
+ * processing. Otherwise the node will core due to cache sanity checks.
+ *
+ * See VESPA-674 for context. In this scenario, it was not possible to write
+ * to the target file when attempting to join in the 2nd source bucket due to
+ * the disk fill ratio exceeding configured limits.
+ */
+void
+JoinOperationHandlerTest::test2ndSourceExceptionEvictsExistingTargetFromCache()
+{
+    setupDisks(1);
+    feedSingleDisk();
+
+    constexpr uint64_t location = 1234;
+
+    document::BucketId src1(33, location);
+    document::BucketId src2(33, location | (1ULL << 32));
+    document::BucketId target(32, location);
+
+    // Ensure target file is _not_ empty so that copySlots is triggered for
+    // each source bucket (rather than just renaming the file, which does not
+    // invoke the file read/write paths).
+    insertDocumentInBucket(location, Timestamp(100000), target);
+    flush(target);
+
+    env()._cache.clear();
+    // File rewrites are buffered before ever reaching the failure simulation
+    // layer, so only 1 actual write is used to flush the target file after
+    // the first source file has been processed. Attempting to flush the writes
+    // for the second source file should fail with an exception.
+    simulateIoErrorsForSubsequentlyOpenedFiles(
+            IoErrors().afterReads(INT_MAX).afterWrites(1));
+
+    spi::Result result = doJoin(target, src1, src2);
+    CPPUNIT_ASSERT(result.hasError());
+    CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write")
+                   != vespalib::string::npos);
+
+    CPPUNIT_ASSERT(!env()._cache.contains(src1));
+    CPPUNIT_ASSERT(!env()._cache.contains(src2));
+    CPPUNIT_ASSERT(!env()._cache.contains(target));
+}
+
+}
+
+}
diff --git a/memfilepersistence/src/tests/spi/logginglazyfile.h b/memfilepersistence/src/tests/spi/logginglazyfile.h
new file mode 100644
index 00000000000..e54753f7c3e
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/logginglazyfile.h
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/io/fileutil.h>
+#include <iostream>
+
+namespace storage {
+
+namespace memfile {
+
+class LoggingLazyFile : public vespalib::LazyFile {
+public:
+    class Factory : public Environment::LazyFileFactory {
+    public:
+        vespalib::LazyFile::UP createFile(const std::string& fileName) const {
+            return vespalib::LazyFile::UP(
+                    new LoggingLazyFile(fileName, vespalib::File::DIRECTIO));
+        }
+    };
+
+    enum OpType {
+        READ = 0,
+        WRITE
+    };
+
+    struct Entry {
+        OpType opType;
+        size_t bufsize;
+        off_t offset;
+
+        std::string toString() const {
+            std::ostringstream ost;
+            ost << (opType == READ ? "Reading " : "Writing ")
+                << bufsize
+                << " bytes at "
+                << offset;
+            return ost.str();
+        }
+    };
+
+    mutable std::vector<Entry> operations;
+
+    LoggingLazyFile(const std::string& filename, int flags)
+        : LazyFile(filename, flags) {};
+
+    size_t getOperationCount() const {
+        return operations.size();
+    }
+
+    virtual off_t write(const void *buf, size_t bufsize, off_t offset) {
+        Entry e;
+        e.opType = WRITE;
+        e.bufsize = bufsize;
+        e.offset = offset;
+
+        operations.push_back(e);
+
+        return vespalib::LazyFile::write(buf, bufsize, offset);
+    }
+
+    virtual size_t read(void *buf, size_t bufsize, off_t offset) const {
+        Entry e;
+        e.opType = READ;
+        e.bufsize = bufsize;
+        e.offset = offset;
+
+        operations.push_back(e);
+
+        return vespalib::LazyFile::read(buf, bufsize, offset);
+    }
+
+    std::string toString() const {
+        std::ostringstream ost;
+        for (uint32_t i = 0; i < operations.size(); i++) {
+            ost << operations[i].toString() << "\n";
+        }
+
+        return ost.str();
+    }
+
+
+
+};
+
+}
+
+}
+
diff --git a/memfilepersistence/src/tests/spi/memcachetest.cpp b/memfilepersistence/src/tests/spi/memcachetest.cpp
new file mode 100644
index 00000000000..d34159ce3f4
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/memcachetest.cpp
@@ -0,0 +1,412 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/memfile/memfilecache.h>
+#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h>
+#include <tests/spi/memfiletestutils.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+
+namespace storage {
+namespace memfile {
+
+class MemCacheTest : public SingleDiskMemFileTestUtils
+{
+    CPPUNIT_TEST_SUITE(MemCacheTest);
+    CPPUNIT_TEST(testSimpleLRU);
+    CPPUNIT_TEST(testCacheSize);
+    CPPUNIT_TEST(testEvictBody);
+    CPPUNIT_TEST(testEvictHeader);
+    CPPUNIT_TEST(testKeepBodyWhenLessThanOneFourth);
+    CPPUNIT_TEST(testComplexEviction);
+    CPPUNIT_TEST(testEraseEmptyOnReturn);
+    CPPUNIT_TEST(testDeleteDoesNotReAddMemoryUsage);
+    CPPUNIT_TEST(testEraseDoesNotReAddMemoryUsage);
+    CPPUNIT_TEST(testGetWithNoCreation);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testSimpleLRU();
+    void testCacheSize();
+    void testReduceCacheSizeCallback();
+    void testReduceCacheSizeCallbackWhileActive();
+    void testEvictBody();
+    void testEvictHeader();
+    void testKeepBodyWhenLessThanOneFourth();
+    void testComplexEviction();
+    void testEraseEmptyOnReturn();
+    void testDeleteDoesNotReAddMemoryUsage();
+    void testEraseDoesNotReAddMemoryUsage();
+    void testGetWithNoCreation();
+
+private:
+    framework::defaultimplementation::ComponentRegisterImpl::UP _register;
+    framework::Component::UP _component;
+    FakeClock::UP _clock;
+    framework::defaultimplementation::MemoryManager::UP _memoryManager;
+    std::vector<framework::MemoryToken::LP> _stolenMemory;
+    std::unique_ptr<MemFilePersistenceMetrics> _metrics;
+
+    std::unique_ptr<MemFileCache> _cache;
+
+    void setSize(const document::BucketId& id,
+                 uint64_t metaSize,
+                 uint64_t headerSz = 0,
+                 uint64_t bodySz = 0,
+                 bool createIfNotInCache = true)
+    {
+        MemFilePtr file(_cache->get(id, env(), env().getDirectory(),
+                                    createIfNotInCache));
+        CPPUNIT_ASSERT(file.get());
+
+        file->_cacheSizeOverride.metaSize = metaSize;
+        file->_cacheSizeOverride.headerSize = headerSz;
+        file->_cacheSizeOverride.bodySize = bodySz;
+    }
+
+    std::string
+    getBucketStatus(uint32_t buckets)
+    {
+        std::ostringstream ost;
+        for (uint32_t i = 1; i < buckets + 1; i++) {
+            document::BucketId id(16, i);
+            ost << id << " ";
+            if (!_cache->contains(id)) {
+                ost << "<nil>\n";
+            } else {
+                MemFilePtr file(_cache->get(id, env(), env().getDirectory()));
+                if (file->_cacheSizeOverride.bodySize > 0) {
+                    ost << "body,";
+                }
+                if (file->_cacheSizeOverride.headerSize > 0) {
+                    ost << "header\n";
+                } else {
+                    ost << "meta only\n";
+                }
+            }
+        }
+
+        return ost.str();
+    }
+
+    uint64_t cacheSize() {
+        return _cache->size();
+    }
+
+    document::BucketId getLRU() {
+        return _cache->getLeastRecentlyUsedBucket()->_bid;
+    }
+
+    void setCacheSize(uint64_t sz) {
+        MemFileCache::MemoryUsage usage;
+        usage.metaSize = sz / 3;
+        usage.headerSize = sz / 3;
+        usage.bodySize = sz - usage.metaSize - usage.headerSize;
+
+        _cache->setCacheSize(usage);
+    }
+
+    void stealMemory(uint64_t memToSteal) {
+        setCacheSize(_cache->getCacheSize() - memToSteal);
+    }
+
+    void setup(uint64_t maxMemory) {
+        tearDown();
+        _register.reset(
+                new framework::defaultimplementation::ComponentRegisterImpl);
+        _clock.reset(new FakeClock);
+        _register->setClock(*_clock);
+        _memoryManager.reset(
+                new framework::defaultimplementation::MemoryManager(
+                    framework::defaultimplementation::AllocationLogic::UP(
+                        new framework::defaultimplementation::SimpleMemoryLogic(
+                            *_clock, maxMemory * 2))));
+        _register->setMemoryManager(*_memoryManager);
+        _component.reset(new framework::Component(*_register, "testcomponent"));
+        _metrics.reset(new MemFilePersistenceMetrics(*_component));
+        _cache.reset(new MemFileCache(*_register, _metrics->_cache));
+        setCacheSize(maxMemory);
+        _memoryManager->registerAllocationType(framework::MemoryAllocationType(
+                "steal", framework::MemoryAllocationType::FORCE_ALLOCATE));
+    }
+
+public:
+    void tearDown() {
+        _stolenMemory.clear();
+        _cache.reset(0);
+        _metrics.reset(0);
+        _component.reset(0);
+        _register.reset(0);
+        _memoryManager.reset(0);
+        _clock.reset(0);
+    }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MemCacheTest);
+
+namespace {
+    FakeClock clock;
+}
+
+void
+MemCacheTest::testSimpleLRU()
+{
+    setup(2000);
+
+    for (uint32_t i = 1; i < 4; i++) {
+        setSize(document::BucketId(16, i), 100);
+    }
+
+    CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), getLRU());
+
+    setSize(document::BucketId(16, 1), 100);
+
+    CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().hits.getValue());
+    CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 2), getLRU());
+}
+
+void
+MemCacheTest::testCacheSize()
+{
+    setup(400);
+
+    setSize(document::BucketId(16, 2), 100);
+    setSize(document::BucketId(16, 1), 150);
+
+    CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().hits.getValue());
+    CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue());
+
+    CPPUNIT_ASSERT_EQUAL(250ul, cacheSize());
+
+    setSize(document::BucketId(16, 1), 200);
+
+    CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().hits.getValue());
+    CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue());
+
+    CPPUNIT_ASSERT_EQUAL(300ul, cacheSize());
+
+    CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 2)));
+    CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 1)));
+
+    setSize(document::BucketId(16, 1), 301);
+
+    CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().hits.getValue());
+    CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue());
+
+    CPPUNIT_ASSERT(!_cache->contains(document::BucketId(16, 2)));
+    CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 1)));
+
+    _cache->clear();
+    CPPUNIT_ASSERT_EQUAL(0ul, cacheSize());
+}
+
+void
+MemCacheTest::testEvictBody()
+{
+    setup(1400);
+
+    CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().body_evictions.getValue());
+
+    setSize(BucketId(16, 1), 150, 100, 0);
+    setSize(BucketId(16, 2), 100, 100, 900);
+
+    CPPUNIT_ASSERT_EQUAL(1350ul, cacheSize());
+
+    stealMemory(150);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x4000000000000001) header\n"
+                    "BucketId(0x4000000000000002) header\n"),
+            getBucketStatus(2));
+    CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().body_evictions.getValue());
+}
+
+void
+MemCacheTest::testKeepBodyWhenLessThanOneFourth()
+{
+    setup(450);
+
+    setSize(BucketId(16, 1), 150, 0, 0);
+    setSize(BucketId(16, 2), 100, 50, 50);
+
+    stealMemory(150);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x4000000000000001) <nil>\n"
+                    "BucketId(0x4000000000000002) body,header\n"),
+            getBucketStatus(2));
+}
+
+void
+MemCacheTest::testEvictHeader()
+{
+    setup(550);
+
+    CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().header_evictions.getValue());
+
+    setSize(BucketId(16, 1), 150, 0, 0);
+    setSize(BucketId(16, 2), 100, 200, 100);
+
+    stealMemory(150);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x4000000000000001) meta only\n"
+                    "BucketId(0x4000000000000002) meta only\n"),
+            getBucketStatus(2));
+    CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().header_evictions.getValue());
+}
+
+#define ASSERT_CACHE_EVICTIONS(meta, header, body) \
+    CPPUNIT_ASSERT_EQUAL(size_t(meta), _cache->getMetrics().body_evictions.getValue()); \
+    CPPUNIT_ASSERT_EQUAL(size_t(header), _cache->getMetrics().header_evictions.getValue()); \
+    CPPUNIT_ASSERT_EQUAL(size_t(body), _cache->getMetrics().meta_evictions.getValue());
+
+void
+MemCacheTest::testComplexEviction()
+{
+    setup(4200);
+
+    setSize(BucketId(16, 1), 150, 0, 0);
+    setSize(BucketId(16, 2), 100, 200, 200);
+    setSize(BucketId(16, 3), 100, 200, 0);
+    setSize(BucketId(16, 4), 100, 400, 0);
+    setSize(BucketId(16, 5), 100, 200, 400);
+    setSize(BucketId(16, 6), 100, 200, 300);
+    setSize(BucketId(16, 7), 100, 0, 0);
+    setSize(BucketId(16, 8), 100, 200, 400);
+    setSize(BucketId(16, 9), 100, 200, 250);
+
+    CPPUNIT_ASSERT_EQUAL(4100ul, cacheSize());
+
+    ASSERT_CACHE_EVICTIONS(0, 0, 0);
+
+    stealMemory(600);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x4000000000000001) meta only\n"
+                    "BucketId(0x4000000000000002) header\n"
+                    "BucketId(0x4000000000000003) header\n"
+                    "BucketId(0x4000000000000004) header\n"
+                    "BucketId(0x4000000000000005) header\n"
+                    "BucketId(0x4000000000000006) body,header\n"
+                    "BucketId(0x4000000000000007) meta only\n"
+                    "BucketId(0x4000000000000008) body,header\n"
+                    "BucketId(0x4000000000000009) body,header\n"),
+            getBucketStatus(9));
+
+    CPPUNIT_ASSERT_EQUAL(3500ul, cacheSize());
+
+    ASSERT_CACHE_EVICTIONS(2, 0, 0);
+
+    stealMemory(500);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x4000000000000001) meta only\n"
+                    "BucketId(0x4000000000000002) meta only\n"
+                    "BucketId(0x4000000000000003) meta only\n"
+                    "BucketId(0x4000000000000004) header\n"
+                    "BucketId(0x4000000000000005) header\n"
+                    "BucketId(0x4000000000000006) body,header\n"
+                    "BucketId(0x4000000000000007) meta only\n"
+                    "BucketId(0x4000000000000008) body,header\n"
+                    "BucketId(0x4000000000000009) body,header\n"),
+            getBucketStatus(9));
+
+    CPPUNIT_ASSERT_EQUAL(3100ul, cacheSize());
+
+    ASSERT_CACHE_EVICTIONS(2, 2, 0);
+
+    stealMemory(1000);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x4000000000000001) <nil>\n"
+                    "BucketId(0x4000000000000002) meta only\n"
+                    "BucketId(0x4000000000000003) meta only\n"
+                    "BucketId(0x4000000000000004) meta only\n"
+                    "BucketId(0x4000000000000005) meta only\n"
+                    "BucketId(0x4000000000000006) header\n"
+                    "BucketId(0x4000000000000007) meta only\n"
+                    "BucketId(0x4000000000000008) body,header\n"
+                    "BucketId(0x4000000000000009) body,header\n"),
+            getBucketStatus(9));
+
+    CPPUNIT_ASSERT_EQUAL(2050ul, cacheSize());
+
+    ASSERT_CACHE_EVICTIONS(3, 4, 1);
+
+    stealMemory(1100);
+
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                    "BucketId(0x4000000000000001) <nil>\n"
+                    "BucketId(0x4000000000000002) <nil>\n"
+                    "BucketId(0x4000000000000003) <nil>\n"
+                    "BucketId(0x4000000000000004) <nil>\n"
+                    "BucketId(0x4000000000000005) <nil>\n"
+                    "BucketId(0x4000000000000006) <nil>\n"
+                    "BucketId(0x4000000000000007) meta only\n"
+                    "BucketId(0x4000000000000008) header\n"
+                    "BucketId(0x4000000000000009) body,header\n"),
+            getBucketStatus(9));
+
+    CPPUNIT_ASSERT_EQUAL(950ul, cacheSize());
+}
+
+#undef ASSERT_CACHE_EVICTIONS
+
+void
+MemCacheTest::testEraseEmptyOnReturn()
+{
+    setup(4200);
+    setSize(BucketId(16, 1), 0, 0, 0);
+    CPPUNIT_ASSERT(!_cache->contains(document::BucketId(16, 1)));
+}
+
+void
+MemCacheTest::testDeleteDoesNotReAddMemoryUsage()
+{
+    BucketId id(16, 1);
+    setup(1000);
+    setSize(id, 100, 200, 300);
+    CPPUNIT_ASSERT_EQUAL(600ul, cacheSize());
+    {
+        MemFilePtr file(_cache->get(id, env(), env().getDirectory()));
+        file.deleteFile();
+    }
+    CPPUNIT_ASSERT_EQUAL(0ul, cacheSize());
+
+}
+
+void
+MemCacheTest::testGetWithNoCreation()
+{
+    BucketId id(16, 1);
+    setup(1000);
+    setSize(id, 100, 200, 300, false);
+    CPPUNIT_ASSERT_EQUAL(0ul, cacheSize());
+}
+
+
+void
+MemCacheTest::testEraseDoesNotReAddMemoryUsage()
+{
+    BucketId id(16, 1);
+    setup(1000);
+    setSize(id, 100, 200, 300);
+    CPPUNIT_ASSERT_EQUAL(600ul, cacheSize());
+    {
+        MemFilePtr file(_cache->get(id, env(), env().getDirectory()));
+        file.eraseFromCache();
+    }
+    CPPUNIT_ASSERT_EQUAL(0ul, cacheSize());
+
+}
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp b/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp
new file mode 100644
index 00000000000..04d82741e67
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp
@@ -0,0 +1,411 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/mapper/memfilemapper.h>
+#include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h>
+#include <vespa/memfilepersistence/mapper/memfile_v1_verifier.h>
+#include <vespa/memfilepersistence/mapper/fileinfo.h>
+#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h>
+#include <tests/spi/memfiletestutils.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+namespace storage {
+namespace memfile {
+
+class MemFileAutoRepairTest : public SingleDiskMemFileTestUtils
+{
+public:
+    void setUp();
+    void tearDown();
+
+    void testFileMetadataCorruptionIsAutoRepaired();
+    void testDocumentContentCorruptionIsAutoRepaired();
+    void testCorruptionEvictsBucketFromCache();
+    void testRepairFailureInMaintainEvictsBucketFromCache();
+    void testZeroLengthFileIsDeleted();
+    void testTruncatedBodyLocationIsAutoRepaired();
+    void testTruncatedHeaderLocationIsAutoRepaired();
+    void testTruncatedHeaderBlockIsAutoRepaired();
+
+    void corruptBodyBlock();
+
+    CPPUNIT_TEST_SUITE(MemFileAutoRepairTest);
+    CPPUNIT_TEST(testFileMetadataCorruptionIsAutoRepaired);
+    CPPUNIT_TEST(testDocumentContentCorruptionIsAutoRepaired);
+    CPPUNIT_TEST(testCorruptionEvictsBucketFromCache);
+    CPPUNIT_TEST(testRepairFailureInMaintainEvictsBucketFromCache);
+    CPPUNIT_TEST(testZeroLengthFileIsDeleted);
+    CPPUNIT_TEST(testTruncatedBodyLocationIsAutoRepaired);
+    CPPUNIT_TEST(testTruncatedHeaderLocationIsAutoRepaired);
+    CPPUNIT_TEST(testTruncatedHeaderBlockIsAutoRepaired);
+    CPPUNIT_TEST_SUITE_END();
+
+private:
+    void assertDocumentIsSilentlyRemoved(
+            const document::BucketId& bucket,
+            const document::DocumentId& docId);
+
+    void reconfigureMinimumHeaderBlockSize(uint32_t newMinSize);
+
+    document::BucketId _bucket;
+    std::unique_ptr<FileSpecification> _file;
+    std::vector<document::DocumentId> _slotIds;
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MemFileAutoRepairTest);
+
+namespace {
+    // A totall uncached memfile with content to use for verify testing
+    std::unique_ptr<MemFile> _memFile;
+
+    // Clear old content. Create new file. Make sure nothing is cached.
+    void prepareBucket(SingleDiskMemFileTestUtils& util,
+                       const FileSpecification& file) {
+        _memFile.reset();
+        util.env()._cache.clear();
+        vespalib::unlink(file.getPath());
+        util.createTestBucket(file.getBucketId(), 0);
+        util.env()._cache.clear();
+        _memFile.reset(new MemFile(file, util.env()));
+        _memFile->getMemFileIO().close();
+
+    }
+
+    MetaSlot getSlot(uint32_t index) {
+        assert(_memFile.get());
+        vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
+        MetaSlot result;
+        file.read(&result, sizeof(MetaSlot),
+                  sizeof(Header) + sizeof(MetaSlot) * index);
+        return result;
+    }
+
+    void setSlot(uint32_t index, MetaSlot slot,
+                 bool updateFileChecksum = true)
+    {
+        (void)updateFileChecksum;
+        assert(_memFile.get());
+        //if (updateFileChecksum) slot.updateFileChecksum();
+        vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
+        file.write(&slot, sizeof(MetaSlot),
+                   sizeof(Header) + sizeof(MetaSlot) * index);
+    }
+}
+
+void
+MemFileAutoRepairTest::setUp()
+{
+    SingleDiskMemFileTestUtils::setUp();
+    _bucket = BucketId(16, 0xa);
+    createTestBucket(_bucket, 0);
+
+    {
+        MemFilePtr memFilePtr(env()._cache.get(_bucket, env(), env().getDirectory()));
+        _file.reset(new FileSpecification(memFilePtr->getFile()));
+        CPPUNIT_ASSERT(memFilePtr->getSlotCount() >= 2);
+        for (size_t i = 0; i < memFilePtr->getSlotCount(); ++i) {
+            _slotIds.push_back(memFilePtr->getDocumentId((*memFilePtr)[i]));
+        }
+    }
+    env()._cache.clear();
+}
+
+void
+MemFileAutoRepairTest::tearDown()
+{
+    _file.reset(0);
+    _memFile.reset(0);
+    SingleDiskMemFileTestUtils::tearDown();
+};
+
+void
+MemFileAutoRepairTest::testFileMetadataCorruptionIsAutoRepaired()
+{
+    // Test corruption detected in initial metadata load
+    prepareBucket(*this, *_file);
+    document::DocumentId id(_slotIds[1]);
+    MetaSlot slot(getSlot(1));
+    CPPUNIT_ASSERT_EQUAL(slot._gid,
+                         id.getGlobalId()); // Sanity checking...
+    {
+        MetaSlot s(slot);
+        s.setTimestamp(Timestamp(40));
+        setSlot(1, s);
+    }
+
+    CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets());
+    
+    // File not in cache; should be detected in initial load
+    spi::GetResult res(doGet(_bucket, id, document::AllFields()));
+    // FIXME: currently loadFile is silently fixing corruptions!
+    //CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res.getErrorCode());
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode());
+    CPPUNIT_ASSERT(!res.hasDocument());
+
+    CPPUNIT_ASSERT_EQUAL(std::string("400000000000000a"), getModifiedBuckets());
+    CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets());
+
+    // File should now have been repaired, so a subsequent get for
+    // the same document should just return an empty (but OK) result.
+    spi::GetResult res2(doGet(_bucket, id, document::AllFields()));
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res2.getErrorCode());
+    CPPUNIT_ASSERT(!res2.hasDocument());
+
+    CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets());
+}
+
+void
+MemFileAutoRepairTest::corruptBodyBlock()
+{
+    CPPUNIT_ASSERT(!env()._cache.contains(_bucket));
+    // Corrupt body block of slot 1
+    MetaSlot slot(getSlot(1));
+    {
+        MetaSlot s(slot);
+        s.setBodyPos(52);
+        s.setBodySize(18);
+        s.updateChecksum();
+        setSlot(1, s);
+    }
+}
+
+void
+MemFileAutoRepairTest::testDocumentContentCorruptionIsAutoRepaired()
+{
+    // Corrupt body block
+    prepareBucket(*this, *_file);
+    document::DocumentId id(_slotIds[1]);
+    corruptBodyBlock();
+
+    CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets());
+    
+    spi::GetResult res(doGet(_bucket, id, document::AllFields()));
+    CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res.getErrorCode());
+    CPPUNIT_ASSERT(!res.hasDocument());
+
+    CPPUNIT_ASSERT(!env()._cache.contains(_bucket));
+
+    CPPUNIT_ASSERT_EQUAL(std::string("400000000000000a"), getModifiedBuckets());
+    CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets());
+
+    // File should now have been repaired, so a subsequent get for
+    // the same document should just return an empty (but OK) result.
+    spi::GetResult res2(doGet(_bucket, id, document::AllFields()));
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res2.getErrorCode());
+    CPPUNIT_ASSERT(!res2.hasDocument());
+
+    // File should now be in cache OK
+    CPPUNIT_ASSERT(env()._cache.contains(_bucket));
+    CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets());
+}
+
+// Ideally we'd test this for each spi operation that accesses MemFiles, but
+// they all use the same eviction+auto-repair logic...
+void
+MemFileAutoRepairTest::testCorruptionEvictsBucketFromCache()
+{
+    prepareBucket(*this, *_file);
+    corruptBodyBlock();
+
+    // Read slot 0 and shove file into cache
+    spi::GetResult res(doGet(_bucket, _slotIds[0], document::AllFields()));
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode());
+    CPPUNIT_ASSERT(res.hasDocument());
+    CPPUNIT_ASSERT(env()._cache.contains(_bucket));
+
+    spi::GetResult res2(doGet(_bucket, _slotIds[1], document::AllFields()));
+    CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res2.getErrorCode());
+    CPPUNIT_ASSERT(!res2.hasDocument());
+
+    // Out of the cache! Begone! Shoo!
+    CPPUNIT_ASSERT(!env()._cache.contains(_bucket));
+
+}
+
+void
+MemFileAutoRepairTest::testRepairFailureInMaintainEvictsBucketFromCache()
+{
+    prepareBucket(*this, *_file);
+    corruptBodyBlock();
+    spi::Result result(getPersistenceProvider().maintain(
+            spi::Bucket(_bucket, spi::PartitionId(0)), spi::HIGH));
+    // File being successfully repaired does not constitute a failure of
+    // the maintain() call.
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode());
+    // It should, however, shove it out of the cache.
+    CPPUNIT_ASSERT(!env()._cache.contains(_bucket));
+}
+
+void
+MemFileAutoRepairTest::testZeroLengthFileIsDeleted()
+{
+    // Completely truncate auto-created file
+    vespalib::LazyFile file(_file->getPath(), 0);
+    file.resize(0);
+
+    // No way to deal with zero-length files aside from deleting them.
+    spi::Result result(getPersistenceProvider().maintain(
+            spi::Bucket(_bucket, spi::PartitionId(0)), spi::HIGH));
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode());
+    CPPUNIT_ASSERT(!env()._cache.contains(_bucket));
+    CPPUNIT_ASSERT(!vespalib::fileExists(_file->getPath()));
+}
+
+namespace {
+
+uint32_t
+alignDown(uint32_t value)
+{
+    uint32_t blocks = value / 512;
+    return blocks * 512;
+};
+
+FileInfo
+fileInfoFromMemFile(const MemFilePtr& mf)
+{
+    auto& ioBuf(dynamic_cast<const SimpleMemFileIOBuffer&>(
+                mf->getMemFileIO()));
+    return ioBuf.getFileInfo();
+}
+
+}
+
+void
+MemFileAutoRepairTest::assertDocumentIsSilentlyRemoved(
+        const document::BucketId& bucket,
+        const document::DocumentId& docId)
+{
+    // Corrupted (truncated) slot should be transparently removed during
+    // loadFile and it should be as if it was never there!
+    spi::Bucket spiBucket(bucket, spi::PartitionId(0));
+    spi::GetResult res(doGet(spiBucket, docId, document::AllFields()));
+    CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode());
+    CPPUNIT_ASSERT(!res.hasDocument());
+}
+
+void
+MemFileAutoRepairTest::testTruncatedBodyLocationIsAutoRepaired()
+{
+    document::BucketId bucket(16, 4);
+    document::Document::SP doc(
+            createRandomDocumentAtLocation(4, 1234, 1024, 1024));
+
+    doPut(doc, bucket, framework::MicroSecTime(1000));
+    flush(bucket);
+    FileInfo fileInfo;
+    {
+        MemFilePtr mf(getMemFile(bucket));
+        CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount());
+        fileInfo = fileInfoFromMemFile(mf);
+
+        const uint32_t bodyBlockStart(
+                sizeof(Header)
+                + fileInfo._metaDataListSize * sizeof(MetaSlot)
+                + fileInfo._headerBlockSize);
+
+        vespalib::LazyFile file(mf->getFile().getPath(), 0);
+        uint32_t slotBodySize = (*mf)[0].getLocation(BODY)._size;
+        CPPUNIT_ASSERT(slotBodySize > 0);
+        // Align down to nearest sector alignment to avoid unrelated DirectIO
+        // checks to kick in. Since the body block is always aligned on a
+        // sector boundary, we know this cannot truncate into the header block.
+        file.resize(alignDown(bodyBlockStart + slotBodySize - 1));
+    }
+    env()._cache.clear();
+    assertDocumentIsSilentlyRemoved(bucket, doc->getId());
+}
+
+void
+MemFileAutoRepairTest::testTruncatedHeaderLocationIsAutoRepaired()
+{
+    document::BucketId bucket(16, 4);
+    document::Document::SP doc(
+            createRandomDocumentAtLocation(4, 1234, 1024, 1024));
+    // Ensure header has a bunch of data (see alignment comments below).
+    doc->setValue(doc->getField("hstringval"),
+                  document::StringFieldValue(std::string(1024, 'A')));
+
+    doPut(doc, bucket, framework::MicroSecTime(1000));
+    flush(bucket);
+    FileInfo fileInfo;
+    {
+        MemFilePtr mf(getMemFile(bucket));
+        CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount());
+        fileInfo = fileInfoFromMemFile(mf);
+
+        const uint32_t headerBlockStart(
+                sizeof(Header)
+                + fileInfo._metaDataListSize * sizeof(MetaSlot));
+
+        vespalib::LazyFile file(mf->getFile().getPath(), 0);
+        uint32_t slotHeaderSize = (*mf)[0].getLocation(HEADER)._size;
+        CPPUNIT_ASSERT(slotHeaderSize > 0);
+        // Align down to nearest sector alignment to avoid unrelated DirectIO
+        // checks to kick in. The header block is not guaranteed to start on
+        // sector boundary, but we assume there is enough slack in the header
+        // section for the metadata slots themselves to be untouched since we
+        // have a minimum header size of 1024 for the doc in question.
+        file.resize(alignDown(headerBlockStart + slotHeaderSize - 1));
+    }
+    env()._cache.clear();
+    assertDocumentIsSilentlyRemoved(bucket, doc->getId());
+}
+
+void
+MemFileAutoRepairTest::reconfigureMinimumHeaderBlockSize(uint32_t newMinSize)
+{
+    using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig;
+    using MemFileConfigBuilder
+        = vespa::config::storage::StorMemfilepersistenceConfigBuilder;
+    MemFileConfigBuilder builder(
+            *env().acquireConfigReadLock().memFilePersistenceConfig());
+    builder.minimumFileMetaSlots = 2;
+    builder.minimumFileHeaderBlockSize = newMinSize;
+    auto newConfig = std::unique_ptr<MemFileConfig>(new MemFileConfig(builder));
+    env().acquireConfigWriteLock().setMemFilePersistenceConfig(
+            std::move(newConfig));
+}
+
+void
+MemFileAutoRepairTest::testTruncatedHeaderBlockIsAutoRepaired()
+{
+    document::BucketId bucket(16, 4);
+    document::Document::SP doc(
+            createRandomDocumentAtLocation(4, 1234, 1, 1));
+    // Ensure header block is large enough that free space is added to the end.
+    reconfigureMinimumHeaderBlockSize(8192);
+    // Add header field and remove randomly generated body field, ensuring
+    // we have no data to add to body field. This will prevent slot body
+    // location checking from detecting a header truncation.
+    doc->setValue(doc->getField("hstringval"),
+                  document::StringFieldValue("foo"));
+    doc->remove(doc->getField("content"));
+
+    doPut(doc, bucket, framework::MicroSecTime(1000));
+    flush(bucket);
+    FileInfo fileInfo;
+    {
+        MemFilePtr mf(getMemFile(bucket));
+        CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount());
+        fileInfo = fileInfoFromMemFile(mf);
+
+        const uint32_t headerBlockEnd(
+                sizeof(Header)
+                + fileInfo._metaDataListSize * sizeof(MetaSlot)
+                + fileInfo._headerBlockSize);
+
+        vespalib::LazyFile file(mf->getFile().getPath(), 0);
+        CPPUNIT_ASSERT_EQUAL(uint32_t(0),
+                             (*mf)[0].getLocation(BODY)._size); // No body.
+        const auto headerLoc((*mf)[0].getLocation(HEADER));
+        const uint32_t extent(headerLoc._pos + headerLoc._size);
+        // Make sure we don't intersect an existing slot range.
+        CPPUNIT_ASSERT(extent < alignDown(headerBlockEnd - 1));
+        file.resize(alignDown(headerBlockEnd - 1));
+    }
+    env()._cache.clear();
+    assertDocumentIsSilentlyRemoved(bucket, doc->getId());
+}
+
+}
+}
diff --git a/memfilepersistence/src/tests/spi/memfiletest.cpp b/memfilepersistence/src/tests/spi/memfiletest.cpp
new file mode 100644
index 00000000000..70b03271da9
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/memfiletest.cpp
@@ -0,0 +1,987 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/memfile/memfile.h>
+#include <tests/spi/memfiletestutils.h>
+#include <tests/spi/logginglazyfile.h>
+#include <tests/spi/options_builder.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/memfilepersistence/memfile/memfilecompactor.h>
+#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h>
+#include <limits>
+
+namespace storage {
+namespace memfile {
+
+struct MemFileTest : public SingleDiskMemFileTestUtils
+{
+    typedef MemFileCompactor::SlotList SlotList;
+
+    /**
+     * Feed a document whose ID is deterministically generated from `seed` to
+     * bucket (16, 4) at time `timestamp`.
+     */
+    document::DocumentId feedDocument(
+            uint64_t seed,
+            uint64_t timestamp,
+            uint32_t headerSize = 0,
+            uint32_t minBodySize = 10,
+            uint32_t maxBodySize = 100);
+
+    /**
+     * Feed n instances of documents with the same ID to bucket (16, 4) using
+     * a timestamp range of [1000, 1000+n).
+     */
+    void feedSameDocNTimes(uint32_t n);
+
+    void setMaxDocumentVersionsOption(uint32_t n);
+
+    std::vector<Types::Timestamp> compactWithVersionLimit(uint32_t maxVersions);
+
+    void testCompactRemoveDoublePut();
+    void testCompactPutRemove();
+    void testCompactGidCollision();
+    void testCompactGidCollisionAndNot();
+    void testCompactWithMemFile();
+    void testCompactCombined();
+    void testCompactDifferentPuts();
+    void testNoCompactionWhenDocumentVersionsWithinLimit();
+    void testCompactWhenDocumentVersionsExceedLimit();
+    void testCompactLimit1KeepsNewestVersionOnly();
+    void testCompactionOptionsArePropagatedFromConfig();
+    void testZeroDocumentVersionConfigIsCorrected();
+    void testResizeToFreeSpace();
+    void testNoFileWriteOnNoOpCompaction();
+    void testCacheSize();
+    void testClearCache();
+    void testGetSlotsByTimestamp();
+    void testCacheInconsistentSlot();
+    void testEnsureCached();
+    void testAddSlotWhenDiskFull();
+    void testGetSerializedSize();
+    void testGetBucketInfo();
+    void testCopySlotsPreservesLocationSharing();
+    void testFlushingToNonExistingFileAlwaysRunsCompaction();
+    void testOrderDocSchemeDocumentsCanBeAddedToFile();
+
+    CPPUNIT_TEST_SUITE(MemFileTest);
+    CPPUNIT_TEST(testCompactRemoveDoublePut);
+    CPPUNIT_TEST(testCompactPutRemove);
+    CPPUNIT_TEST(testCompactGidCollision);
+    CPPUNIT_TEST(testCompactGidCollisionAndNot);
+    CPPUNIT_TEST(testCompactWithMemFile);
+    CPPUNIT_TEST(testCompactCombined);
+    CPPUNIT_TEST(testCompactDifferentPuts);
+    CPPUNIT_TEST(testNoCompactionWhenDocumentVersionsWithinLimit);
+    CPPUNIT_TEST(testCompactWhenDocumentVersionsExceedLimit);
+    CPPUNIT_TEST(testCompactLimit1KeepsNewestVersionOnly);
+    CPPUNIT_TEST(testCompactionOptionsArePropagatedFromConfig);
+    CPPUNIT_TEST(testZeroDocumentVersionConfigIsCorrected);
+    CPPUNIT_TEST(testNoFileWriteOnNoOpCompaction);
+    CPPUNIT_TEST(testCacheSize);
+    CPPUNIT_TEST(testClearCache);
+    CPPUNIT_TEST(testGetSlotsByTimestamp);
+    CPPUNIT_TEST(testEnsureCached);
+    CPPUNIT_TEST(testResizeToFreeSpace);
+    CPPUNIT_TEST(testAddSlotWhenDiskFull);
+    CPPUNIT_TEST(testGetSerializedSize);
+    CPPUNIT_TEST(testGetBucketInfo);
+    CPPUNIT_TEST(testCopySlotsPreservesLocationSharing);
+    CPPUNIT_TEST(testFlushingToNonExistingFileAlwaysRunsCompaction);
+    CPPUNIT_TEST(testOrderDocSchemeDocumentsCanBeAddedToFile);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MemFileTest);
+
+/**
+ * Slots should actually be the same pointer. Use this assert to do correct
+ * check, and still print content of slots on failure.
+ */
+#define ASSERT_SLOT_EQUAL(slotptra, slotptrb) \
+{ \
+    CPPUNIT_ASSERT(slotptra != 0); \
+    CPPUNIT_ASSERT(slotptrb != 0); \
+    std::ostringstream slotdiff; \
+    slotdiff << "Expected: " << *slotptra << ", but got " << *slotptrb; \
+    CPPUNIT_ASSERT_EQUAL_MSG(slotdiff.str(), slotptra, slotptrb); \
+}
+
+namespace {
+
+framework::MicroSecTime sec(uint64_t n) {
+    return framework::MicroSecTime(n * 1000000ULL);
+}
+
+/**
+ * Utility functions for tests to call to do compacting, such that the
+ * tests themselves are not bound to the current interface.
+ *
+ * Also, this function translates second time to microsecond time.
+ */
+MemFileTest::SlotList getSlotsToRemove(
+        const MemFile& file, uint64_t currentTime,
+        uint64_t revertTime, uint64_t keepRemoveTime)
+{
+    MemFileCompactor compactor(
+            sec(currentTime),
+            CompactionOptions()
+                .maxDocumentVersions(
+                    std::numeric_limits<uint32_t>::max())
+                .revertTimePeriod(sec(revertTime))
+                .keepRemoveTimePeriod(sec(keepRemoveTime)));
+    return compactor.getSlotsToRemove(file);
+}
+
+class AutoFlush
+{
+public:
+    AutoFlush(MemFilePtr& ptr) : _ptr(ptr) {}
+    ~AutoFlush() { _ptr->flushToDisk(); }
+private:
+    MemFilePtr& _ptr;
+};
+
+}
+
+document::DocumentId
+MemFileTest::feedDocument(
+        uint64_t seed,
+        uint64_t timestamp,
+        uint32_t headerSize,
+        uint32_t minDocSize,
+        uint32_t maxDocSize) {
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       4, seed, minDocSize, maxDocSize));
+
+    if (headerSize > 0) {
+        std::string val(headerSize, 'A');
+        doc->setValue(doc->getField("hstringval"),
+                      document::StringFieldValue(val));
+    }
+
+    doPut(doc,
+          document::BucketId(16, 4),
+          Timestamp(timestamp * 1000000));
+
+    return doc->getId();
+}
+
+void
+MemFileTest::feedSameDocNTimes(uint32_t n)
+{
+    for (uint32_t i = 0; i < n; ++i) {
+        feedDocument(1234, 1000 + i);
+    }
+}
+
+void
+MemFileTest::setMaxDocumentVersionsOption(uint32_t n)
+{
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options)
+                .maxDocumentVersions(n)
+                .build());
+}
+
+void
+MemFileTest::testCacheSize()
+{
+    // Feed some puts
+    for (uint32_t i = 0; i < 4; i++) {
+        feedDocument(1234 * (i % 2), 1000 + 200 * i);
+    }
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+
+    CPPUNIT_ASSERT(file->getCacheSize().sum() > 0);
+}
+
+void
+MemFileTest::testClearCache()
+{
+    // Feed some puts
+    for (uint32_t i = 0; i < 4; i++) {
+        feedDocument(1234 * (i % 2), 1000 + 200 * i);
+    }
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    file->flushToDisk();
+
+    CPPUNIT_ASSERT(file->getCacheSize().bodySize > 0);
+    CPPUNIT_ASSERT(file->getCacheSize().headerSize > 0);
+
+    file->clearCache(HEADER);
+
+    CPPUNIT_ASSERT(file->getCacheSize().bodySize > 0);
+    CPPUNIT_ASSERT(file->getMemFileIO().getCachedSize(BODY) > 0);
+    CPPUNIT_ASSERT_EQUAL(0, (int)file->getCacheSize().headerSize);
+    CPPUNIT_ASSERT_EQUAL(uint64_t(0), file->getMemFileIO().getCachedSize(HEADER));
+
+    file->clearCache(BODY);
+
+    CPPUNIT_ASSERT_EQUAL(0, (int)file->getCacheSize().bodySize);
+    CPPUNIT_ASSERT_EQUAL(uint64_t(0), file->getMemFileIO().getCachedSize(BODY));
+}
+
+
+void
+MemFileTest::testCompactGidCollision()
+{
+    // Feed two puts
+    for (uint32_t i = 0; i < 2; i++) {
+        feedDocument(1234 * i, 1000 + 200 * i);
+    }
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    AutoFlush af(file);
+    const_cast<MemSlot&>((*file)[1]).setGlobalId((*file)[0].getGlobalId());
+
+    CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount());
+
+    {
+        SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 86400));
+        CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size());
+        file->removeSlots(toRemove);
+    }
+}
+
+void
+MemFileTest::testCompactGidCollisionAndNot()
+{
+    // Feed some puts
+    for (uint32_t i = 0; i < 4; i++) {
+        feedDocument(1234 * (i % 2), 1000 + 200 * i);
+    }
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    AutoFlush af(file);
+    const_cast<MemSlot&>((*file)[2]).setGlobalId((*file)[0].getGlobalId());
+    const_cast<MemSlot&>((*file)[3]).setGlobalId((*file)[1].getGlobalId());
+
+    CPPUNIT_ASSERT_EQUAL(4, (int)file->getSlotCount());
+
+    {
+        SlotList toRemove(getSlotsToRemove(*file, 2000, 300, 86400));
+
+        CPPUNIT_ASSERT_EQUAL(2, (int)toRemove.size());
+        ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]);
+        ASSERT_SLOT_EQUAL(&(*file)[1], toRemove[1]);
+        file->removeSlots(toRemove);
+    }
+}
+
+
+void
+MemFileTest::testCompactRemoveDoublePut()
+{
+    // Feed two puts at time 1000 and 1200
+    for (uint32_t i = 0; i < 2; i++) {
+        feedDocument(1234, 1000 + 200 * i);
+    }
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    AutoFlush af(file);
+    CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount());
+
+    {
+        // Not time to collect yet, newest is still revertable
+        SlotList toRemove(getSlotsToRemove(*file, 1300, 300, 86400));
+        CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size());
+    }
+
+    {
+        SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 86400));
+
+        CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size());
+        ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]);
+        file->removeSlots(toRemove);
+    }
+}
+
+void
+MemFileTest::testCompactPutRemove()
+{
+    document::DocumentId docId = feedDocument(1234, 1000);
+
+    doRemove(docId, Timestamp(1200*1000000), 0);
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    AutoFlush af(file);
+
+    {
+        // Since remove can still be reverted, we can't revert anything.
+        SlotList toRemove(getSlotsToRemove(*file, 1300, 300, 600));
+
+        CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size());
+    }
+
+    {
+        SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 600));
+
+        CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size());
+        ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]);
+        file->removeSlots(toRemove);
+    }
+
+    {
+        SlotList toRemove(getSlotsToRemove(*file, 1900, 300, 600));
+
+        CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size());
+        ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]);
+        file->removeSlots(toRemove);
+    }
+}
+
+void
+MemFileTest::testCompactCombined()
+{
+    document::DocumentId docId;
+
+    // Feed some puts at time 1000, 1200, 1400, 1600 and 1800 for same doc.
+    for (uint32_t i = 0; i < 5; i++) {
+        docId = feedDocument(1234, 1000 + i * 200);
+    }
+    flush(document::BucketId(16, 4));
+
+    // Now add remove at time 2000.
+    doRemove(docId, Timestamp(2000 * 1000000), 0);
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    AutoFlush af(file);
+    CPPUNIT_ASSERT_EQUAL(6, (int)file->getSlotCount());
+
+    {
+        // Compact all redundant slots that are older than revert period of 300.
+        // This includes 1000, 1200, 1400 and 1600.
+        SlotList toRemove(getSlotsToRemove(*file, 2001, 300, 86400));
+        CPPUNIT_ASSERT_EQUAL(4, (int)toRemove.size());
+        for (int i = 0; i < 4; ++i) {
+            ASSERT_SLOT_EQUAL(&(*file)[i], toRemove[i]);
+        }
+        file->removeSlots(toRemove);
+    }
+}
+
+void
+MemFileTest::testCompactDifferentPuts()
+{
+    document::DocumentId docId;
+
+    // Feed some puts
+    for (uint32_t i = 0; i < 2; i++) {
+        for (uint32_t j = 0; j < 3; j++) {
+            feedDocument(1234 * j, 1000 + (i * 3 + j) * 200);
+        }
+    }
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    AutoFlush af(file);
+    CPPUNIT_ASSERT_EQUAL(6, (int)file->getSlotCount());
+
+    {
+        SlotList toRemove(getSlotsToRemove(*file, 3000, 300, 86400));
+        CPPUNIT_ASSERT_EQUAL(3, (int)toRemove.size());
+
+        for (uint32_t i = 0; i < 3; i++) {
+            bool found = false;
+            for (uint32_t j = 0; j < 3; j++) {
+                if ((*file)[j] == *toRemove[i]) {
+                    found = true;
+                }
+            }
+
+            CPPUNIT_ASSERT(found);
+        }
+        file->removeSlots(toRemove);
+    }
+}
+
+void
+MemFileTest::testCompactWithMemFile()
+{
+    // Feed two puts
+    for (uint32_t i = 0; i < 2; i++) {
+        document::Document::SP doc(createRandomDocumentAtLocation(
+                                           4, 1234, 10, 100));
+
+        doPut(doc, document::BucketId(16, 4), Timestamp((1000 + i * 200)*1000000), 0);
+    }
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    AutoFlush af(file);
+    CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount());
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options)
+                .revertTimePeriod(framework::MicroSecTime(1000))
+                .build());
+
+    getFakeClock()._absoluteTime = framework::MicroSecTime(2000ULL * 1000000);
+
+    CPPUNIT_ASSERT(file->compact());
+    CPPUNIT_ASSERT(!file->compact());
+
+    CPPUNIT_ASSERT_EQUAL(1, (int)file->getSlotCount());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1200 * 1000000), (*file)[0].getTimestamp());
+}
+
+/**
+ * Feed 5 versions of a single document at absolute times 0 through 4 seconds
+ * and run compaction using the provided max document version option.
+ * Revert time/keep remove time options are effectively disabled for this test.
+ * Returns timestamps of all slots that are marked as compactable.
+ */
+std::vector<Types::Timestamp>
+MemFileTest::compactWithVersionLimit(uint32_t maxVersions)
+{
+    document::BucketId bucket(16, 4);
+    std::shared_ptr<Document> doc(
+            createRandomDocumentAtLocation(4, 1234, 10, 100));
+    uint32_t versionLimit = 5;
+    for (uint32_t i = 0; i < versionLimit; ++i) {
+        Timestamp ts(sec(i).getTime());
+        doPut(doc, bucket, ts, 0);
+    }
+    flush(bucket);
+
+    MemFilePtr file(getMemFile(bucket));
+    CPPUNIT_ASSERT_EQUAL(versionLimit, file->getSlotCount());
+
+    framework::MicroSecTime currentTime(sec(versionLimit));
+    MemFileCompactor compactor(
+            currentTime,
+            CompactionOptions()
+                .revertTimePeriod(sec(versionLimit))
+                .keepRemoveTimePeriod(sec(versionLimit))
+                .maxDocumentVersions(maxVersions));
+    auto slots = compactor.getSlotsToRemove(*file);
+    // Convert to timestamps since caller won't have access to actual MemFile.
+    std::vector<Timestamp> timestamps;
+    for (const MemSlot* slot : slots) {
+        timestamps.push_back(slot->getTimestamp());
+    }
+    return timestamps;
+}
+
+void
+MemFileTest::testNoCompactionWhenDocumentVersionsWithinLimit()
+{
+    auto timestamps = compactWithVersionLimit(5);
+    CPPUNIT_ASSERT(timestamps.empty());
+}
+
+void
+MemFileTest::testCompactWhenDocumentVersionsExceedLimit()
+{
+    auto timestamps = compactWithVersionLimit(2);
+    CPPUNIT_ASSERT_EQUAL(size_t(3), timestamps.size());
+    std::vector<Timestamp> expected = {
+        sec(0), sec(1), sec(2)
+    };
+    CPPUNIT_ASSERT_EQUAL(expected, timestamps);
+}
+
+void
+MemFileTest::testCompactLimit1KeepsNewestVersionOnly()
+{
+    auto timestamps = compactWithVersionLimit(1);
+    CPPUNIT_ASSERT_EQUAL(size_t(4), timestamps.size());
+    std::vector<Timestamp> expected = {
+        sec(0), sec(1), sec(2), sec(3)
+    };
+    CPPUNIT_ASSERT_EQUAL(expected, timestamps);
+}
+
+void
+MemFileTest::testCompactionOptionsArePropagatedFromConfig()
+{
+    vespa::config::storage::StorMemfilepersistenceConfigBuilder mfcBuilder;
+    vespa::config::content::PersistenceConfigBuilder pcBuilder;
+
+    pcBuilder.maximumVersionsOfSingleDocumentStored = 12345;
+    pcBuilder.revertTimePeriod = 555;
+    pcBuilder.keepRemoveTimePeriod = 777;
+
+    vespa::config::storage::StorMemfilepersistenceConfig mfc(mfcBuilder);
+    vespa::config::content::PersistenceConfig pc(pcBuilder);
+    Options opts(mfc, pc);
+
+    CPPUNIT_ASSERT_EQUAL(framework::MicroSecTime(555 * 1000000),
+                         opts._revertTimePeriod);
+    CPPUNIT_ASSERT_EQUAL(framework::MicroSecTime(777 * 1000000),
+                         opts._keepRemoveTimePeriod);
+    CPPUNIT_ASSERT_EQUAL(uint32_t(12345), opts._maxDocumentVersions);
+}
+
+void
+MemFileTest::testZeroDocumentVersionConfigIsCorrected()
+{
+    vespa::config::storage::StorMemfilepersistenceConfigBuilder mfcBuilder;
+    vespa::config::content::PersistenceConfigBuilder pcBuilder;
+
+    pcBuilder.maximumVersionsOfSingleDocumentStored = 0;
+
+    vespa::config::storage::StorMemfilepersistenceConfig mfc(mfcBuilder);
+    vespa::config::content::PersistenceConfig pc(pcBuilder);
+    Options opts(mfc, pc);
+
+    CPPUNIT_ASSERT_EQUAL(uint32_t(1), opts._maxDocumentVersions);
+}
+
+void
+MemFileTest::testGetSlotsByTimestamp()
+{
+    for (uint32_t i = 0; i < 10; i++) {
+        feedDocument(i, 1000 + i);
+    }
+    flush(document::BucketId(16, 4));
+
+    std::vector<Timestamp> timestamps;
+    timestamps.push_back(Timestamp(999  * 1000000));
+    timestamps.push_back(Timestamp(1001 * 1000000));
+    timestamps.push_back(Timestamp(1002 * 1000000));
+    timestamps.push_back(Timestamp(1007 * 1000000));
+    timestamps.push_back(Timestamp(1100 * 1000000));
+    std::vector<const MemSlot*> slots;
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    file->getSlotsByTimestamp(timestamps, slots);
+    CPPUNIT_ASSERT_EQUAL(std::size_t(3), slots.size());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1001 * 1000000), slots[0]->getTimestamp());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1002 * 1000000), slots[1]->getTimestamp());
+    CPPUNIT_ASSERT_EQUAL(Timestamp(1007 * 1000000), slots[2]->getTimestamp());
+}
+
+void
+MemFileTest::testEnsureCached()
+{
+    // Feed some puts
+    for (uint32_t i = 0; i < 5; i++) {
+        feedDocument(i, 1000 + i * 200, 600, 600, 600);
+    }
+    flush(document::BucketId(16, 4));
+
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options).maximumReadThroughGap(512).build());
+    env()._cache.clear();
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+        CPPUNIT_ASSERT(file.get());
+        CPPUNIT_ASSERT_EQUAL(5, (int)file->getSlotCount());
+
+        file->ensureDocumentIdCached((*file)[1]);
+
+        for (std::size_t i = 0; i < file->getSlotCount(); ++i) {
+            if (i == 1) {
+                CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i]));
+            } else {
+                CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i]));
+            }
+            CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY));
+        }
+    }
+
+    env()._cache.clear();
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+        file->ensureDocumentCached((*file)[2], true);
+
+        for (std::size_t i = 0; i < file->getSlotCount(); ++i) {
+            if (i == 2) {
+                CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i]));
+                CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER));
+            } else {
+                CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i]));
+                CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER));
+            }
+            CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY));
+        }
+    }
+
+    env()._cache.clear();
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+
+        file->ensureDocumentCached((*file)[3], false);
+
+        for (std::size_t i = 0; i < file->getSlotCount(); ++i) {
+            if (i == 3) {
+                CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i]));
+                CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER));
+                CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY));
+            } else {
+                CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i]));
+                CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER));
+                CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY));
+            }
+        }
+    }
+
+    env()._cache.clear();
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+
+        std::vector<Timestamp> ts;
+        for (int i = 2; i < 5; ++i) {
+            ts.push_back((*file)[i].getTimestamp());
+        }
+
+        file->ensureDocumentCached(ts, false);
+
+        for (std::size_t i = 0; i < file->getSlotCount(); ++i) {
+            if (i > 1 && i < 5) {
+                CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i]));
+                CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER));
+                CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY));
+            } else {
+                CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i]));
+                CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER));
+                CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY));
+            }
+        }
+    }
+
+    env()._cache.clear();
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+
+        file->ensureHeaderBlockCached();
+
+        for (std::size_t i = 0; i < file->getSlotCount(); ++i) {
+            CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i]));
+            CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER));
+            CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY));
+        }
+    }
+
+    env()._cache.clear();
+
+    {
+        MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+
+        file->ensureBodyBlockCached();
+
+        for (std::size_t i = 0; i < file->getSlotCount(); ++i) {
+            CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i]));
+            CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER));
+            CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY));
+        }
+    }
+}
+
+void
+MemFileTest::testResizeToFreeSpace()
+{
+    /**
+     * This test tests that files are resized to a smaller size when they need
+     * to be. This should happen during a call to flushToDisk() in MemFile,
+     * which is either dirty or if passed flag to check even if clean. (Which
+     * the integrity checker cycle uses). A clean file is used for testing to
+     * ensure that no part of the code only works for dirty files. This test
+     * only test for the case where body block is too large. The real
+     * implementation here will be in the flushUpdatesToFile() function for the
+     * given file formats. (VersionSerializer's) If more cases wants to be
+     * tested add those as unit tests for the versionserializers themselves.
+     */
+
+        // Create a test bucket to test with.
+    BucketId bucket(16, 0xa);
+    createTestBucket(bucket, 0);
+
+    off_t file_size =
+        ((SimpleMemFileIOBuffer&)getMemFile(bucket)->getMemFileIO()).
+        getFileHandle().getFileSize();
+
+        // Clear cache so we can manually modify backing file to increase the
+        // size of it.
+    FileSpecification file(getMemFile(bucket)->getFile());
+    env()._cache.clear();
+    {
+            // Extend file to 1 MB, which should create an excessively large
+            // body block such that file should be resized to be smaller
+        vespalib::LazyFile fileHandle(file.getPath(), 0);
+        fileHandle.write("foobar", 6, 2 * 1024 * 1024 - 6);
+    }
+    MemFilePtr memFile(getMemFile(bucket));
+    memFile->flushToDisk(CHECK_NON_DIRTY_FILE_FOR_SPACE);
+    CPPUNIT_ASSERT_EQUAL(file_size,
+                         ((SimpleMemFileIOBuffer&)memFile->getMemFileIO()).
+                         getFileHandle().getFileSize());
+}
+
+namespace {
+
+const vespalib::LazyFile&
+getFileHandle(const MemFile& mf1)
+{
+    return dynamic_cast<const SimpleMemFileIOBuffer&>(
+            mf1.getMemFileIO()).getFileHandle();
+}
+
+const LoggingLazyFile&
+getLoggerFile(const MemFile& file)
+{
+    return dynamic_cast<const LoggingLazyFile&>(getFileHandle(file));
+}
+
+}
+
+void
+MemFileTest::testNoFileWriteOnNoOpCompaction()
+{
+    BucketId bucket(16, 4);
+    env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>(
+            new LoggingLazyFile::Factory());
+
+    // Feed some unique puts, none of which can be compacted away.
+    for (uint32_t i = 0; i < 2; i++) {
+        document::Document::SP doc(createRandomDocumentAtLocation(
+                                           4, i, 10, 100));
+
+        doPut(doc, bucket, Timestamp((1000 + i * 200)*1000000), 0);
+    }
+    flush(bucket);
+
+    MemFilePtr file(getMemFile(bucket));
+
+    size_t opsBeforeFlush = getLoggerFile(*file).getOperationCount();
+    file->flushToDisk(CHECK_NON_DIRTY_FILE_FOR_SPACE);
+    size_t opsAfterFlush = getLoggerFile(*file).getOperationCount();
+
+    // Disk should not have been touched, since no slots have been
+    // compacted away.
+    if (opsBeforeFlush != opsAfterFlush) {
+        std::cerr << "\n" << getLoggerFile(*file).toString() << "\n";
+    }
+    CPPUNIT_ASSERT_EQUAL(opsBeforeFlush, opsAfterFlush);
+}
+
+void
+MemFileTest::testAddSlotWhenDiskFull()
+{
+    {
+        MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+        AutoFlush af(file);
+        {
+            // Add a dummy-slot that can later be removed
+            Document::SP doc(createRandomDocumentAtLocation(4));
+            file->addPutSlot(*doc, Timestamp(1001));
+        }
+    }
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    AutoFlush af(file);
+    PartitionMonitor* mon = env().getDirectory().getPartition().getMonitor();
+    // Set disk to 99% full
+    mon->setStatOncePolicy();
+    mon->setMaxFillness(.98f);
+    mon->overrideRealStat(512, 100000, 99000);
+    CPPUNIT_ASSERT(mon->isFull());
+
+    // Test that addSlot with a non-persisted Put fails
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        try {
+            file->addPutSlot(*doc, Timestamp(10003));
+            CPPUNIT_ASSERT(false);
+        } catch (vespalib::IoException& e) {
+            CPPUNIT_ASSERT_EQUAL(vespalib::IoException::NO_SPACE, e.getType());
+        }
+    }
+
+    // Slots with valid header and body locations should also
+    // not fail, as these are added when the file is loaded
+    {
+        // Just steal parts from existing slot to ensure they're persisted
+        const MemSlot* existing = file->getSlotAtTime(Timestamp(1001));
+
+        MemSlot slot(existing->getGlobalId(),
+                     Timestamp(1005),
+                     existing->getLocation(HEADER),
+                     existing->getLocation(BODY),
+                     IN_USE,
+                     0x1234);
+        file->addSlot(slot);
+    }
+
+    // Removes should not fail when disk is full
+    {
+        file->addRemoveSlot(*file->getSlotAtTime(Timestamp(1001)), Timestamp(1003));
+    }
+}
+
+void
+MemFileTest::testGetSerializedSize() {
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       4, 1234, 1024, 1024));
+
+    std::string val("Header");
+    doc->setValue(doc->getField("hstringval"),
+                  document::StringFieldValue(val));
+
+    doPut(doc, document::BucketId(16, 4), framework::MicroSecTime(1000));
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+    file->ensureBodyBlockCached();
+    const MemSlot* slot = file->getSlotAtTime(framework::MicroSecTime(1000));
+    CPPUNIT_ASSERT(slot != 0);
+
+    vespalib::nbostream serializedHeader;
+    doc->serializeHeader(serializedHeader);
+
+    vespalib::nbostream serializedBody;
+    doc->serializeBody(serializedBody);
+
+    CPPUNIT_ASSERT_EQUAL(uint32_t(serializedHeader.size()),
+                         file->getSerializedSize(*slot, HEADER));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(serializedBody.size()),
+                         file->getSerializedSize(*slot, BODY));
+}
+
+void
+MemFileTest::testGetBucketInfo()
+{
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       4, 1234, 100, 100));
+    doc->setValue(doc->getField("content"),
+                  document::StringFieldValue("foo"));
+    document::Document::SP doc2(createRandomDocumentAtLocation(
+                                       4, 1235, 100, 100));
+    doc2->setValue(doc->getField("content"),
+                   document::StringFieldValue("bar"));
+
+    doPut(doc, document::BucketId(16, 4), framework::MicroSecTime(1000));
+    flush(document::BucketId(16, 4));
+
+    doPut(doc2, document::BucketId(16, 4), framework::MicroSecTime(1001));
+    flush(document::BucketId(16, 4));
+
+    // Do remove which should only add a single meta entry
+    doRemove(doc->getId(), Timestamp(1002), 0);
+    flush(document::BucketId(16, 4));
+
+    MemFilePtr file(getMemFile(document::BucketId(16, 4)));
+
+    CPPUNIT_ASSERT_EQUAL(3u, file->getSlotCount());
+    uint32_t maxHeaderExtent = (*file)[1].getLocation(HEADER)._pos
+                               + (*file)[1].getLocation(HEADER)._size;
+    uint32_t maxBodyExtent = (*file)[1].getLocation(BODY)._pos
+                             + (*file)[1].getLocation(BODY)._size;
+
+    uint32_t wantedUsedSize = 64 + 40*3 + maxHeaderExtent + maxBodyExtent;
+    BucketInfo info = file->getBucketInfo();
+    CPPUNIT_ASSERT_EQUAL(1u, info.getDocumentCount());
+    CPPUNIT_ASSERT_EQUAL(3u, info.getEntryCount());
+    CPPUNIT_ASSERT_EQUAL(wantedUsedSize, info.getUsedSize());
+    uint32_t wantedUniqueSize = (*file)[1].getLocation(HEADER)._size
+                                + (*file)[1].getLocation(BODY)._size;
+    CPPUNIT_ASSERT_EQUAL(wantedUniqueSize, info.getDocumentSize());
+}
+
+void
+MemFileTest::testCopySlotsPreservesLocationSharing()
+{
+    document::BucketId bucket(16, 4);
+    // Feed two puts to same document (identical seed). These should not
+    // share any blocks. Note: implicit sec -> microsec conversion.
+    feedDocument(1234, 1000); // slot 0
+    auto docId = feedDocument(1234, 1001); // slot 1
+    // Update only header of last version of document. This should share
+    // slot body block 2 with that slot 1.
+    auto update = createHeaderUpdate(docId, document::IntFieldValue(5678));
+    doUpdate(bucket, update, Timestamp(1002 * 1000000), 0);
+    // Feed a remove for doc in slot 2. This should share the header block of
+    // slot 3 with the newest document in slot 2.
+    doRemove(docId, Timestamp(1003 * 1000000), 0);
+    flush(bucket);
+
+    {
+        MemFilePtr src(getMemFile(document::BucketId(16, 4)));
+        MemFilePtr dest(getMemFile(document::BucketId(17, 4)));
+        std::vector<Timestamp> timestamps {
+            Timestamp(1000 * 1000000),
+            Timestamp(1001 * 1000000),
+            Timestamp(1002 * 1000000),
+            Timestamp(1003 * 1000000)
+        };
+        std::vector<const MemSlot*> slots {
+            src->getSlotAtTime(Timestamp(1000 * 1000000)),
+            src->getSlotAtTime(Timestamp(1001 * 1000000)),
+            src->getSlotAtTime(Timestamp(1002 * 1000000)),
+            src->getSlotAtTime(Timestamp(1003 * 1000000))
+        };
+        dest->copySlotsFrom(*src, slots);
+        dest->flushToDisk();
+        CPPUNIT_ASSERT_EQUAL(uint32_t(4), dest->getSlotCount());
+
+        DataLocation header[4];
+        DataLocation body[4];
+        for (int i = 0; i < 4; ++i) {
+            const MemSlot* slot = dest->getSlotAtTime(timestamps[i]);
+            header[i] = slot->getLocation(HEADER);
+            body[i] = slot->getLocation(BODY);
+        }
+        CPPUNIT_ASSERT(!(header[0] == header[1]));
+
+        CPPUNIT_ASSERT_EQUAL(body[2], body[1]);
+        CPPUNIT_ASSERT_EQUAL(header[3], header[2]);
+    }
+}
+
+void
+MemFileTest::testFlushingToNonExistingFileAlwaysRunsCompaction()
+{
+    document::BucketId bucket(16, 4);
+
+    setMaxDocumentVersionsOption(1);
+    feedSameDocNTimes(10);
+    flush(bucket);
+
+    // Max version limit is 1, flushing should have compacted it down.
+    MemFilePtr file(getMemFile(bucket));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount());
+}
+
+void
+MemFileTest::testOrderDocSchemeDocumentsCanBeAddedToFile()
+{
+    // Quick explanation of the esoteric and particular values chosen below:
+    // orderdoc mangles the MSB of the bucket ID based on the document ID's
+    // ordering parameters and thus its bucket cannot be directly deduced from
+    // the generated GID. The values given here specify a document whose GID
+    // bits differ from those generated by the document and where a GID-only
+    // bucket ownership check would fail (nuking the node with an assertion).
+    // We have to make sure cases do not trigger false positives.
+    document::BucketId bucket(0x84000000ee723751);
+    auto doc = createDocument("the quick red fox trips over a hedge",
+                              "orderdoc(3,1):storage_test:group1:9:9");
+    doPut(std::shared_ptr<Document>(std::move(doc)),
+          bucket,
+          Timestamp(1000000 * 1234));
+    flush(bucket);
+
+    MemFilePtr file(getMemFile(bucket));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount());
+    // Ideally we'd test the failure case as well, but that'd require framework
+    // support for death tests.
+}
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.cpp b/memfilepersistence/src/tests/spi/memfiletestutils.cpp
new file mode 100644
index 00000000000..1e882ccbe6b
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/memfiletestutils.cpp
@@ -0,0 +1,455 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h>
+#include <tests/spi/memfiletestutils.h>
+#include <tests/spi/simulatedfailurefile.h>
+#include <vespa/memfilepersistence/memfile/memfilecache.h>
+#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h>
+#include <sys/time.h>
+
+using document::DocumentType;
+
+namespace storage {
+namespace memfile {
+
+namespace {
+    spi::LoadType defaultLoadType(0, "default");
+}
+
+namespace {
+    vdstestlib::DirConfig initialize(uint32_t numDisks) {
+        system(vespalib::make_string("rm -rf vdsroot").c_str());
+        for (uint32_t i = 0; i < numDisks; i++) {
+            system(vespalib::make_string("mkdir -p vdsroot/disks/d%d", i).c_str());
+        }
+        vdstestlib::DirConfig config(getStandardConfig(true));
+        return config;
+    }
+
+    template<typename T>
+    struct ConfigReader : public T::Subscriber
+    {
+        T config;
+
+        ConfigReader(const std::string& configId) {
+            T::subscribe(configId, *this);
+        }
+        void configure(const T& c) { config = c; }
+    };
+}
+
+MemFileTestEnvironment::MemFileTestEnvironment(
+        uint32_t numDisks,
+        framework::ComponentRegister& reg,
+        const document::DocumentTypeRepo& repo)
+    : _config(initialize(numDisks)),
+      _provider(reg, _config.getConfigId())
+{
+    _provider.setDocumentRepo(repo);
+    _provider.getPartitionStates();
+}
+
+MemFileTestUtils::MemFileTestUtils()
+{
+}
+
+MemFileTestUtils::~MemFileTestUtils()
+{
+}
+
+void
+MemFileTestUtils::setupDisks(uint32_t numDisks) {
+    tearDown();
+    _componentRegister.reset(
+            new framework::defaultimplementation::ComponentRegisterImpl);
+    _clock.reset(new FakeClock);
+    _componentRegister->setClock(*_clock);
+    _memoryManager.reset(
+            new framework::defaultimplementation::MemoryManager(
+                framework::defaultimplementation::AllocationLogic::UP(
+                    new framework::defaultimplementation::SimpleMemoryLogic(
+                        *_clock, 1024 * 1024 * 1024))));
+    _componentRegister->setMemoryManager(*_memoryManager);
+    _env.reset(new MemFileTestEnvironment(numDisks,
+                                          *_componentRegister,
+                                          *getTypeRepo()));
+}
+
+Environment&
+MemFileTestUtils::env()
+{
+    return static_cast<MemFilePersistenceProvider&>(
+            getPersistenceProvider()).getEnvironment();
+}
+
+MemFilePersistenceProvider&
+MemFileTestUtils::getPersistenceProvider()
+{
+    return _env->_provider;
+}
+
+MemFilePersistenceThreadMetrics&
+MemFileTestUtils::getMetrics()
+{
+    return getPersistenceProvider().getMetrics();
+}
+
+std::string
+MemFileTestUtils::getMemFileStatus(const document::BucketId& id,
+                                   uint32_t disk)
+{
+    MemFilePtr file(getMemFile(id, disk));
+    std::ostringstream ost;
+    ost << id << ": " << file->getSlotCount() << "," << file->getDisk();
+    return ost.str();
+}
+
+std::string
+MemFileTestUtils::getModifiedBuckets()
+{
+    spi::BucketIdListResult result(
+            getPersistenceProvider().getModifiedBuckets());
+    const spi::BucketIdListResult::List& list(result.getList());
+    std::ostringstream ss;
+    for (size_t i = 0; i < list.size(); ++i) {
+        if (i != 0) {
+            ss << ",";
+        }
+        ss << std::hex << list[i].getId();
+    }
+    return ss.str();
+}
+
+MemFilePtr
+MemFileTestUtils::getMemFile(const document::BucketId& id, uint16_t disk)
+{
+    return env()._cache.get(id, env(), env().getDirectory(disk));
+}
+
+spi::Result
+MemFileTestUtils::flush(const document::BucketId& id, uint16_t disk)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    return getPersistenceProvider().flush(
+            spi::Bucket(id, spi::PartitionId(disk)), context);
+}
+
+document::Document::SP
+MemFileTestUtils::doPutOnDisk(
+        uint16_t disk,
+        uint32_t location,
+        Timestamp timestamp,
+        uint32_t minSize,
+        uint32_t maxSize)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                             location, timestamp.getTime(), minSize, maxSize));
+    getPersistenceProvider().put(
+            spi::Bucket(document::BucketId(16, location), spi::PartitionId(disk)),
+            spi::Timestamp(timestamp.getTime()),
+            doc,
+            context);
+    return doc;
+}
+
+bool
+MemFileTestUtils::doRemoveOnDisk(
+        uint16_t disk,
+        const document::BucketId& bucketId,
+        const document::DocumentId& docId,
+        Timestamp timestamp,
+        OperationHandler::RemoveType persistRemove)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    if (persistRemove == OperationHandler::PERSIST_REMOVE_IF_FOUND) {
+        spi::RemoveResult result = getPersistenceProvider().removeIfFound(
+            spi::Bucket(bucketId, spi::PartitionId(disk)),
+            spi::Timestamp(timestamp.getTime()),
+            docId,
+            context);
+        return result.wasFound();
+    }
+    spi::RemoveResult result = getPersistenceProvider().remove(
+            spi::Bucket(bucketId, spi::PartitionId(disk)),
+            spi::Timestamp(timestamp.getTime()),
+            docId,
+            context);
+
+    return result.wasFound();
+}
+
+bool
+MemFileTestUtils::doUnrevertableRemoveOnDisk(
+        uint16_t disk,
+        const document::BucketId& bucketId,
+        const DocumentId& docId,
+        Timestamp timestamp)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    spi::RemoveResult result =
+        getPersistenceProvider().remove(
+                spi::Bucket(bucketId, spi::PartitionId(disk)),
+                spi::Timestamp(timestamp.getTime()),
+                docId, context);
+
+    return result.wasFound();
+}
+
+spi::GetResult
+MemFileTestUtils::doGetOnDisk(
+        uint16_t disk,
+        const document::BucketId& bucketId,
+        const document::DocumentId& docId,
+        const document::FieldSet& fields)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    return getPersistenceProvider().get(
+            spi::Bucket(bucketId, spi::PartitionId(disk)),
+            fields, docId, context);
+}
+
+document::DocumentUpdate::SP
+MemFileTestUtils::createBodyUpdate(
+        const document::DocumentId& docId,
+        const document::FieldValue& updateValue)
+{
+    const DocumentType*
+        docType(getTypeRepo()->getDocumentType("testdoctype1"));
+    document::DocumentUpdate::SP update(
+            new document::DocumentUpdate(*docType, docId));
+    std::shared_ptr<document::AssignValueUpdate> assignUpdate(
+            new document::AssignValueUpdate(updateValue));
+    document::FieldUpdate fieldUpdate(docType->getField("content"));
+    fieldUpdate.addUpdate(*assignUpdate);
+    update->addUpdate(fieldUpdate);
+    return update;
+}
+
+document::DocumentUpdate::SP
+MemFileTestUtils::createHeaderUpdate(
+        const document::DocumentId& docId,
+        const document::FieldValue& updateValue)
+{
+    const DocumentType*
+        docType(getTypeRepo()->getDocumentType("testdoctype1"));
+    document::DocumentUpdate::SP update(
+            new document::DocumentUpdate(*docType, docId));
+    std::shared_ptr<document::AssignValueUpdate> assignUpdate(
+            new document::AssignValueUpdate(updateValue));
+    document::FieldUpdate fieldUpdate(docType->getField("headerval"));
+    fieldUpdate.addUpdate(*assignUpdate);
+    update->addUpdate(fieldUpdate);
+    return update;
+}
+
+void
+MemFileTestUtils::doPut(const document::Document::SP& doc,
+                        Timestamp time,
+                        uint16_t disk,
+                        uint16_t usedBits)
+{
+    document::BucketId bucket(
+            getBucketIdFactory().getBucketId(doc->getId()));
+    bucket.setUsedBits(usedBits);
+    doPut(doc, bucket, time, disk);
+}
+
+void
+MemFileTestUtils::doPut(const document::Document::SP& doc,
+                        document::BucketId bid,
+                        Timestamp time,
+                        uint16_t disk)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    getPersistenceProvider().put(spi::Bucket(bid, spi::PartitionId(disk)),
+                                 spi::Timestamp(time.getTime()), doc, context);
+}
+
+spi::UpdateResult
+MemFileTestUtils::doUpdate(document::BucketId bid,
+                           const document::DocumentUpdate::SP& update,
+                           Timestamp time,
+                           uint16_t disk)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    return getPersistenceProvider().update(
+            spi::Bucket(bid, spi::PartitionId(disk)),
+            spi::Timestamp(time.getTime()), update, context);
+}
+
+void
+MemFileTestUtils::doRemove(const document::DocumentId& id, Timestamp time,
+                           uint16_t disk, bool unrevertableRemove,
+                           uint16_t usedBits)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    document::BucketId bucket(getBucketIdFactory().getBucketId(id));
+    bucket.setUsedBits(usedBits);
+
+    if (unrevertableRemove) {
+        getPersistenceProvider().remove(
+                spi::Bucket(bucket, spi::PartitionId(disk)),
+                spi::Timestamp(time.getTime()),
+                id, context);
+    } else {
+        spi::RemoveResult result = getPersistenceProvider().removeIfFound(
+                spi::Bucket(bucket, spi::PartitionId(disk)),
+                spi::Timestamp(time.getTime()),
+                id, context);
+
+        if (!result.wasFound()) {
+            throw vespalib::IllegalStateException(
+                    "Attempted to remove non-existing doc " + id.toString(),
+                    VESPA_STRLOC);
+        }
+    }
+}
+
+void
+MemFileTestUtils::copyHeader(document::Document& dest,
+                             const document::Document& src)
+{
+    // FIXME(vekterli): temporary solution while we don't have
+    // fieldset pruning functionality in Document.
+    //dest.setHeaderPtr(src.getHeaderPtr());
+    vespalib::nbostream originalBodyStream;
+    dest.serializeBody(originalBodyStream);
+
+    vespalib::nbostream headerStream;
+    src.serializeHeader(headerStream);
+    document::ByteBuffer hbuf(headerStream.peek(), headerStream.size());
+    dest.deserializeHeader(*getTypeRepo(), hbuf);
+    // deserializeHeader clears fields struct, so have to re-set body
+    document::ByteBuffer bbuf(originalBodyStream.peek(),
+                              originalBodyStream.size());
+    dest.deserializeBody(*getTypeRepo(), bbuf);
+}
+
+void
+MemFileTestUtils::copyBody(document::Document& dest,
+                           const document::Document& src)
+{
+    // FIXME(vekterli): temporary solution while we don't have
+    // fieldset pruning functionality in Document.
+    //dest.setBodyPtr(src.getBodyPtr());
+    vespalib::nbostream stream;
+    src.serializeBody(stream);
+    document::ByteBuffer buf(stream.peek(), stream.size());
+    dest.deserializeBody(*getTypeRepo(), buf);
+}
+
+void
+MemFileTestUtils::clearBody(document::Document& doc)
+{
+    // FIXME(vekterli): temporary solution while we don't have
+    // fieldset pruning functionality in Document.
+    //doc->getBody().clear();
+    vespalib::nbostream stream;
+    doc.serializeHeader(stream);
+    doc.deserialize(*getTypeRepo(), stream);
+}
+
+void
+MemFileTestUtils::createTestBucket(const document::BucketId& bucket,
+                                   uint16_t disk)
+{
+
+    uint32_t opsPerType = 2;
+    uint32_t numberOfLocations = 2;
+    uint32_t minDocSize = 0;
+    uint32_t maxDocSize = 128;
+
+    for (uint32_t useHeaderOnly = 0; useHeaderOnly < 2; ++useHeaderOnly) {
+        bool headerOnly = (useHeaderOnly == 1);
+        for (uint32_t optype=0; optype < 4; ++optype) {
+            for (uint32_t i=0; i<opsPerType; ++i) {
+                uint32_t seed = useHeaderOnly * 10000 + optype * 1000 + i + 1;
+                uint64_t location = (seed % numberOfLocations);
+                location <<= 32;
+                location += (bucket.getRawId() & 0xffffffff);
+                document::Document::SP doc(
+                        createRandomDocumentAtLocation(
+                            location, seed, minDocSize, maxDocSize));
+                if (headerOnly) {
+                    clearBody(*doc);
+                }
+                doPut(doc, Timestamp(seed), disk, bucket.getUsedBits());
+                if (optype == 0) { // Regular put
+                } else if (optype == 1) { // Overwritten later in time
+                    Document::SP doc2(new Document(*doc));
+                    doc2->setValue(doc2->getField("content"),
+                                   document::StringFieldValue("overwritten"));
+                    doPut(doc2, Timestamp(seed + 500),
+                          disk, bucket.getUsedBits());
+                } else if (optype == 2) { // Removed
+                    doRemove(doc->getId(), Timestamp(seed + 500), disk, false,
+                             bucket.getUsedBits());
+                } else if (optype == 3) { // Unrevertable removed
+                    doRemove(doc->getId(), Timestamp(seed), disk, true,
+                             bucket.getUsedBits());
+                }
+            }
+        }
+    }
+    flush(bucket, disk);
+}
+
+void
+MemFileTestUtils::simulateIoErrorsForSubsequentlyOpenedFiles(
+        const IoErrors& errs)
+{
+    std::unique_ptr<SimulatedFailureLazyFile::Factory> factory(
+            new SimulatedFailureLazyFile::Factory);
+    factory->setWriteOpsBeforeFailure(errs._afterWrites);
+    factory->setReadOpsBeforeFailure(errs._afterReads);
+    env()._lazyFileFactory = std::move(factory);
+}
+
+void
+MemFileTestUtils::unSimulateIoErrorsForSubsequentlyOpenedFiles()
+{
+    env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>(
+            new DefaultLazyFileFactory(0));
+}
+
+std::string
+MemFileTestUtils::stringifyFields(const document::Document& doc) const
+{
+    using namespace document;
+    std::vector<std::string> output;
+    const StructFieldValue& fields(doc.getFields());
+    for (StructFieldValue::const_iterator
+             it(fields.begin()), e(fields.end());
+         it != e; ++it)
+    {
+        std::ostringstream ss;
+        const Field& f(it.field());
+        ss << f.getName() << ": ";
+        FieldValue::UP val(fields.getValue(f));
+        if (val.get()) {
+            ss << val->toString();
+        } else {
+            ss << "(null)";
+        }
+        output.push_back(ss.str());
+    }
+    std::ostringstream ret;
+    std::sort(output.begin(), output.end());
+    std::copy(output.begin(), output.end(),
+              std::ostream_iterator<std::string>(ret, "\n"));
+    return ret.str();
+}
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.h b/memfilepersistence/src/tests/spi/memfiletestutils.h
new file mode 100644
index 00000000000..a13b902a214
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/memfiletestutils.h
@@ -0,0 +1,294 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class storage::memfile::MemFileTestUtils
+ * \ingroup memfile
+ *
+ * \brief Utilities for unit tests of the MemFile layer.
+ *
+ * The memfile layer typically needs a MemFileEnvironment object that must be
+ * set up. This class creates such an object to be used by unit tests. Other
+ * utilities useful for only MemFile testing can be added here too.
+ */
+
+#pragma once
+
+#include <vespa/memfilepersistence/memfile/memfilecache.h>
+#include <tests/testhelper.h>
+#include <vespa/persistence/spi/persistenceprovider.h>
+#include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h>
+#include <vespa/document/base/testdocman.h>
+#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
+#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h>
+#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
+
+namespace storage {
+namespace memfile {
+
+struct FakeClock : public framework::Clock {
+public:
+    typedef std::unique_ptr<FakeClock> UP;
+
+    framework::MicroSecTime _absoluteTime;
+
+    FakeClock() {};
+
+    virtual void addSecondsToTime(uint32_t nr) {
+        _absoluteTime += framework::MicroSecTime(nr * uint64_t(1000000));
+    }
+
+    virtual framework::MicroSecTime getTimeInMicros() const {
+        return _absoluteTime;
+    }
+    virtual framework::MilliSecTime getTimeInMillis() const {
+        return getTimeInMicros().getMillis();
+    }
+    virtual framework::SecondTime getTimeInSeconds() const {
+        return getTimeInMicros().getSeconds();
+    }
+};
+
+struct MemFileTestEnvironment {
+    MemFileTestEnvironment(uint32_t numDisks,
+                           framework::ComponentRegister& reg,
+                           const document::DocumentTypeRepo& repo);
+
+    vdstestlib::DirConfig _config;
+    MemFilePersistenceProvider _provider;
+};
+
+class MemFileTestUtils : public Types, public document::TestDocMan, public CppUnit::TestFixture {
+private:
+        // This variables are kept in test class. Instances that needs to be
+        // unique per test needs to be setup in setupDisks and cleared in
+        // tearDown
+    document::BucketIdFactory _bucketIdFactory;
+    framework::defaultimplementation::ComponentRegisterImpl::UP _componentRegister;
+    FakeClock::UP _clock;
+    framework::defaultimplementation::MemoryManager::UP _memoryManager;
+    std::unique_ptr<MemFileTestEnvironment> _env;
+
+public:
+    MemFileTestUtils();
+    virtual ~MemFileTestUtils();
+
+    void setupDisks(uint32_t disks);
+
+    void tearDown() {
+        _env.reset();
+        _componentRegister.reset();
+        _memoryManager.reset();
+        _clock.reset();
+    }
+
+    std::string getMemFileStatus(const document::BucketId& id, uint32_t disk = 0);
+
+    std::string getModifiedBuckets();
+
+    /**
+       Flushes all cached data to disk and updates the bucket database accordingly.
+    */
+    void flush();
+
+    FakeClock& getFakeClock() { return *_clock; }
+
+    spi::Result flush(const document::BucketId& id, uint16_t disk = 0);
+
+    MemFilePersistenceProvider& getPersistenceProvider();
+
+    MemFilePtr getMemFile(const document::BucketId& id, uint16_t disk = 0);
+
+    Environment& env();
+
+    MemFilePersistenceThreadMetrics& getMetrics();
+
+    MemFileTestEnvironment& getEnv() { return *_env; }
+
+    /**
+       Performs a put to the given disk.
+       Returns the document that was inserted.
+    */
+    document::Document::SP doPutOnDisk(
+            uint16_t disk,
+            uint32_t location,
+            Timestamp timestamp,
+            uint32_t minSize = 0,
+            uint32_t maxSize = 128);
+
+    document::Document::SP doPut(
+            uint32_t location,
+            Timestamp timestamp,
+            uint32_t minSize = 0,
+            uint32_t maxSize = 128)
+        { return doPutOnDisk(0, location, timestamp, minSize, maxSize); }
+
+    /**
+       Performs a remove to the given disk.
+       Returns the new doccount if document was removed, or -1 if not found.
+    */
+    bool doRemoveOnDisk(
+            uint16_t disk,
+            const document::BucketId& bid,
+            const document::DocumentId& id,
+            Timestamp timestamp,
+            OperationHandler::RemoveType persistRemove);
+
+    bool doRemove(
+            const document::BucketId& bid,
+            const document::DocumentId& id,
+            Timestamp timestamp,
+            OperationHandler::RemoveType persistRemove) {
+        return doRemoveOnDisk(0, bid, id, timestamp, persistRemove);
+    }
+
+    bool doUnrevertableRemoveOnDisk(uint16_t disk,
+                                    const document::BucketId& bid,
+                                    const DocumentId& id,
+                                    Timestamp timestamp);
+
+    bool doUnrevertableRemove(const document::BucketId& bid,
+                              const DocumentId& id,
+                              Timestamp timestamp)
+    {
+        return doUnrevertableRemoveOnDisk(0, bid, id, timestamp);
+    }
+
+    virtual const document::BucketIdFactory& getBucketIdFactory() const
+        { return _bucketIdFactory; }
+
+    document::BucketIdFactory& getBucketIdFactory()
+        { return _bucketIdFactory; }
+
+    /**
+     * Do a remove toward storage set up in test environment.
+     *
+     * @id Document to remove.
+     * @disk If set, use this disk, otherwise lookup in bucket db.
+     * @unrevertableRemove If set, instead of adding put, turn put to remove.
+     * @usedBits Generate bucket to use from docid using this amount of bits.
+     */
+    void doRemove(const DocumentId& id, Timestamp, uint16_t disk,
+                  bool unrevertableRemove = false, uint16_t usedBits = 16);
+
+    spi::GetResult doGetOnDisk(
+            uint16_t disk,
+            const document::BucketId& bucketId,
+            const document::DocumentId& docId,
+            const document::FieldSet& fields);
+
+    spi::GetResult doGet(
+            const document::BucketId& bucketId,
+            const document::DocumentId& docId,
+            const document::FieldSet& fields)
+        { return doGetOnDisk(0, bucketId, docId, fields); }
+
+    document::DocumentUpdate::SP createBodyUpdate(
+            const document::DocumentId& id,
+            const document::FieldValue& updateValue);
+
+    document::DocumentUpdate::SP createHeaderUpdate(
+            const document::DocumentId& id,
+            const document::FieldValue& updateValue);
+
+    virtual const document::DocumentTypeRepo::SP getTypeRepo() const
+    { return document::TestDocMan::getTypeRepoSP(); }
+
+    /**
+     * Do a put toward storage set up in test environment.
+     *
+     * @doc Document to put. Use TestDocMan to generate easily.
+     * @disk If set, use this disk, otherwise lookup in bucket db.
+     * @usedBits Generate bucket to use from docid using this amount of bits.
+     */
+    void doPut(const Document::SP& doc, Timestamp,
+               uint16_t disk, uint16_t usedBits = 16);
+
+    void doPut(const document::Document::SP& doc,
+               document::BucketId bid,
+               Timestamp time,
+               uint16_t disk = 0);
+
+    spi::UpdateResult doUpdate(document::BucketId bid,
+                               const document::DocumentUpdate::SP& update,
+                               Timestamp time,
+                               uint16_t disk = 0);
+
+    /**
+     * Create a test bucket with various content representing most states a
+     * bucket can represent. (Such that tests have a nice test bucket to use
+     * that require operations to handle all the various bucket contents.
+     *
+     * @disk If set, use this disk, otherwise lookup in bucket db.
+     */
+    void createTestBucket(const BucketId&, uint16_t disk = 0xffff);
+
+    /**
+     * In-place modify doc so that it has no more body fields.
+     */
+    void clearBody(document::Document& doc);
+
+    /**
+     * Copy all header data from src into dest, replacing any
+     * header fields it may already have there. NOTE: this will
+     * also overwrite document ID, type etc!
+     */
+    void copyHeader(document::Document& dest,
+                    const document::Document& src);
+
+    /**
+     * Copy all body data from src into dest, replacing any
+     * body fields it may already have there.
+     */
+    void copyBody(document::Document& dest,
+                  const document::Document& src);
+
+    std::string stringifyFields(const Document& doc) const;
+
+    struct IoErrors {
+        int _afterReads;
+        int _afterWrites;
+
+        IoErrors()
+            : _afterReads(0),
+              _afterWrites(0)
+        {
+        }
+
+        IoErrors& afterReads(int n) {
+            _afterReads = n;
+            return *this;
+        }
+
+        IoErrors& afterWrites(int n) {
+            _afterWrites = n;
+            return *this;
+        }
+    };
+
+    /**
+     * Replaces internal LazyFile factory so that it produces LazyFile
+     * implementations that trigger I/O exceptions on read/write. Optionally,
+     * can supply a parameter setting explicit bounds on how many operations
+     * are allowed on a file before trigging exceptions from there on out. A
+     * bound of -1 in practice means "don't fail ever" while 0 means "fail the
+     * next op of that type".
+     */
+    void simulateIoErrorsForSubsequentlyOpenedFiles(
+            const IoErrors& errs = IoErrors());
+
+    /**
+     * Replace internal LazyFile factory with the default, non-failing impl.
+     */
+    void unSimulateIoErrorsForSubsequentlyOpenedFiles();
+};
+
+class SingleDiskMemFileTestUtils : public MemFileTestUtils
+{
+public:
+    void setUp() {
+        setupDisks(1);
+    }
+};
+
+} // memfile
+} // storage
+
diff --git a/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp b/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp
new file mode 100644
index 00000000000..a5d1c50d043
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp
@@ -0,0 +1,1110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/mapper/memfilemapper.h>
+#include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h>
+#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h>
+#include <tests/spi/memfiletestutils.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/memfilepersistence/mapper/locationreadplanner.h>
+#include <tests/spi/simulatedfailurefile.h>
+#include <tests/spi/options_builder.h>
+
+namespace storage {
+namespace memfile {
+
+struct MemFileV1SerializerTest : public SingleDiskMemFileTestUtils
+{
+    void tearDown();
+    void setUpPartialWriteEnvironment();
+    void resetConfig(uint32_t minimumFileSize, uint32_t minimumFileHeaderBlockSize);
+    void doTestPartialWriteRemove(bool readAll);
+    void doTestPartialWriteUpdate(bool readAll);
+
+    void testWriteReadSingleDoc();
+    void testWriteReadPartial();
+    void testWriteReadPartialRemoved();
+    void testPartialWritePutHeaderOnly();
+    void testPartialWritePut();
+    void testPartialWriteRemoveCached();
+    void testPartialWriteRemoveNotCached();
+    void testPartialWriteUpdateCached();
+    void testPartialWriteUpdateNotCached();
+    void testPartialWriteTooMuchFreeSpace();
+    void testPartialWriteNotEnoughFreeSpace();
+    void testWriteReadSingleRemovedDoc();
+    void testLocationDiskIoPlannerSimple();
+    void testLocationDiskIoPlannerMergeReads();
+    void testLocationDiskIoPlannerAlignReads();
+    void testLocationDiskIoPlannerOneDocument();
+    void testSeparateReadsForHeaderAndBody();
+    void testLocationsRemappedConsistently();
+    void testHeaderBufferTooSmall();
+
+    /*std::unique_ptr<MemFile> createMemFile(FileSpecification& file,
+                                         bool callLoadFile)
+    {
+        return std::unique_ptr<MemFile>(new MemFile(file, env(), callLoadFile));
+        }*/
+
+    CPPUNIT_TEST_SUITE(MemFileV1SerializerTest);
+    CPPUNIT_TEST(testWriteReadSingleDoc);
+    CPPUNIT_TEST(testWriteReadPartial);
+    CPPUNIT_TEST(testWriteReadPartialRemoved);
+    CPPUNIT_TEST(testWriteReadSingleRemovedDoc);
+    CPPUNIT_TEST(testPartialWritePutHeaderOnly);
+    CPPUNIT_TEST(testPartialWritePut);
+    CPPUNIT_TEST(testPartialWriteRemoveCached);
+    CPPUNIT_TEST(testPartialWriteRemoveNotCached);
+    CPPUNIT_TEST(testPartialWriteUpdateCached);
+    CPPUNIT_TEST(testPartialWriteUpdateNotCached);
+    CPPUNIT_TEST(testLocationDiskIoPlannerSimple);
+    CPPUNIT_TEST(testLocationDiskIoPlannerMergeReads);
+    CPPUNIT_TEST(testLocationDiskIoPlannerAlignReads);
+    CPPUNIT_TEST(testLocationDiskIoPlannerOneDocument);
+    CPPUNIT_TEST(testSeparateReadsForHeaderAndBody);
+    CPPUNIT_TEST(testPartialWriteTooMuchFreeSpace);
+    CPPUNIT_TEST(testPartialWriteNotEnoughFreeSpace);
+    CPPUNIT_TEST(testLocationsRemappedConsistently);
+    CPPUNIT_TEST(testHeaderBufferTooSmall);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MemFileV1SerializerTest);
+
+namespace {
+
+const vespalib::LazyFile&
+getFileHandle(const MemFile& mf1)
+{
+    return static_cast<const SimpleMemFileIOBuffer&>(
+            mf1.getMemFileIO()).getFileHandle();
+}
+
+const LoggingLazyFile&
+getLoggerFile(const MemFile& file)
+{
+    return static_cast<const LoggingLazyFile&>(getFileHandle(file));
+}
+
+bool isContentEqual(MemFile& mf1, MemFile& mf2,
+                    bool requireEqualContentCached, std::ostream& error)
+{
+    MemFile::const_iterator it1(
+            mf1.begin(Types::ITERATE_GID_UNIQUE | Types::ITERATE_REMOVED));
+    MemFile::const_iterator it2(
+            mf2.begin(Types::ITERATE_GID_UNIQUE | Types::ITERATE_REMOVED));
+    while (true) {
+        if (it1 == mf1.end() && it2 == mf2.end()) {
+            return true;
+        }
+        if (it1 == mf1.end() || it2 == mf2.end()) {
+            error << "Different amount of GID unique slots";
+            return false;
+        }
+        if (it1->getTimestamp() != it2->getTimestamp()) {
+            error << "Different timestamps";
+            return false;
+            }
+        if (it1->getGlobalId() != it2->getGlobalId()) {
+            error << "Different gids";
+            return false;
+        }
+        if (it1->getPersistedFlags() != it2->getPersistedFlags()) {
+            error << "Different persisted flags";
+            return false;
+        }
+        if (requireEqualContentCached) {
+            if (mf1.partAvailable(*it1, Types::BODY)
+                ^ mf2.partAvailable(*it2, Types::BODY)
+                || mf1.partAvailable(*it1, Types::HEADER)
+                ^ mf2.partAvailable(*it2, Types::HEADER))
+            {
+                error << "Difference in cached content: ";
+                return false;
+            }
+        }
+
+        if (mf1.partAvailable(*it1, Types::HEADER) &&
+            mf2.partAvailable(*it2, Types::HEADER))
+        {
+            document::Document::UP doc1 = mf1.getDocument(*it1, Types::ALL);
+            document::Document::UP doc2 = mf2.getDocument(*it2, Types::ALL);
+
+            CPPUNIT_ASSERT(doc1.get());
+            CPPUNIT_ASSERT(doc2.get());
+
+            if (*doc1 != *doc2) {
+                error << "Documents different: Expected:\n"
+                      << doc1->toString(true) << "\nActual:\n"
+                      << doc2->toString(true) << "\n";
+                return false;
+            }
+        }
+        ++it1;
+        ++it2;
+    }
+}
+
+bool
+validateMemFileStructure(const MemFile& mf, std::ostream& error)
+{
+    const SimpleMemFileIOBuffer& ioBuf(
+            dynamic_cast<const SimpleMemFileIOBuffer&>(mf.getMemFileIO()));
+    const FileInfo& fileInfo(ioBuf.getFileInfo());
+    if (fileInfo.getFileSize() % 512) {
+        error << "File size is not a multiple of 512 bytes";
+        return false;
+    }
+    if (fileInfo.getBlockIndex(Types::BODY) % 512) {
+        error << "Body start index is not a multiple of 512 bytes";
+        return false;
+    }
+    if (fileInfo.getBlockSize(Types::BODY) % 512) {
+        error << "Body size is not a multiple of 512 bytes";
+        return false;
+    }
+    return true;
+}
+
+}
+
+void
+MemFileV1SerializerTest::tearDown() {
+    //_memFile.reset();
+}
+
+/**
+ * Adjust minimum slotfile size values to avoid rewriting file
+ * when we want to get a partial write
+ */
+void
+MemFileV1SerializerTest::setUpPartialWriteEnvironment()
+{
+    resetConfig(4096, 2048);
+}
+
+void
+MemFileV1SerializerTest::resetConfig(uint32_t minimumFileSize,
+                                     uint32_t minimumFileHeaderBlockSize)
+{
+    using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig;
+    using MemFileConfigBuilder
+        = vespa::config::storage::StorMemfilepersistenceConfigBuilder;
+
+    MemFileConfigBuilder persistenceConfig(
+            *env().acquireConfigReadLock().memFilePersistenceConfig());
+    persistenceConfig.minimumFileHeaderBlockSize = minimumFileHeaderBlockSize;
+    persistenceConfig.minimumFileSize = minimumFileSize;
+    auto newCfg = std::unique_ptr<MemFileConfig>(
+            new MemFileConfig(persistenceConfig));
+    env().acquireConfigWriteLock().setMemFilePersistenceConfig(
+            std::move(newCfg));
+}
+
+struct DummyMemFileIOInterface : MemFileIOInterface {
+    Document::UP getDocumentHeader(const document::DocumentTypeRepo&,
+                                   DataLocation) const
+    {
+        return Document::UP();
+    }
+
+    document::DocumentId getDocumentId(DataLocation) const {
+        return document::DocumentId("");
+    }
+
+    void readBody(const document::DocumentTypeRepo&,
+                  DataLocation,
+                  Document&) const
+    {
+    }
+    DataLocation addDocumentIdOnlyHeader(
+            const DocumentId&,
+            const document::DocumentTypeRepo&)
+    {
+        return DataLocation();
+    }
+    DataLocation addHeader(const Document&) { return DataLocation(); }
+    DataLocation addBody(const Document&) { return DataLocation(); }
+    void clear(DocumentPart) {}
+    bool verifyConsistent() const { return true; }
+    void move(const FileSpecification&) {}
+    DataLocation copyCache(const MemFileIOInterface&,
+                           DocumentPart,
+                           DataLocation)
+    {
+        return DataLocation();
+    }
+
+    void close() {};
+    bool isCached(DataLocation, DocumentPart) const { return false; }
+    bool isPersisted(DataLocation, DocumentPart) const { return false; }
+    uint32_t getSerializedSize(DocumentPart,
+                               DataLocation) const { return 0; }
+
+    void ensureCached(Environment&,
+                      DocumentPart,
+                      const std::vector<DataLocation>&)
+    {}
+
+    size_t getCachedSize(DocumentPart) const { return 0; }
+};
+
+#define VESPA_MEMFILEV1_SETUP_SOURCE \
+    system("rm -f testfile.0"); \
+    document::Document::SP doc(createRandomDocumentAtLocation(4)); \
+    FileSpecification file(document::BucketId(16, 4), env().getDirectory(0), "testfile.0");                                  \
+    MemFile source(file, env());
+
+#define VESPA_MEMFILEV1_DIFF(source, target) \
+    "\nSource:\n" + source.toString(true) \
+    + "\nTarget:\n" + target.toString(true)
+
+#define VESPA_MEMFILEV1_VALIDATE_STRUCTURE(mfile) \
+{ \
+    std::ostringstream validateErr; \
+    if (!validateMemFileStructure(mfile, validateErr)) { \
+        CPPUNIT_FAIL(validateErr.str()); \
+    } \
+}
+
+#define VESPA_MEMFILEV1_ASSERT_SERIALIZATION(sourceMemFile) \
+env()._memFileMapper.flush(sourceMemFile, env()); \
+VESPA_MEMFILEV1_VALIDATE_STRUCTURE(sourceMemFile) \
+MemFile target(file, env()); \
+VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target) \
+{ \
+    target.ensureBodyBlockCached(); \
+    target.getBucketInfo(); \
+    std::ostringstream diff; \
+    if (!isContentEqual(sourceMemFile, target, true, diff)) { \
+        std::string msg = "MemFiles not content equal: " + diff.str() \
+                        + VESPA_MEMFILEV1_DIFF(sourceMemFile, target); \
+        CPPUNIT_FAIL(msg); \
+    } \
+}
+
+void
+MemFileV1SerializerTest::testWriteReadSingleDoc()
+{
+    VESPA_MEMFILEV1_SETUP_SOURCE;
+    source.addPutSlot(*doc, Timestamp(1001));
+    std::string foo(VESPA_MEMFILEV1_DIFF(source, source));
+    VESPA_MEMFILEV1_ASSERT_SERIALIZATION(source);
+}
+
+void
+MemFileV1SerializerTest::testWriteReadPartial()
+{
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    std::map<Timestamp, Document::SP> docs;
+    {
+        MemFile source(file, env());
+
+        for (int i = 0; i < 50; ++i) {
+            Document::SP doc(createRandomDocumentAtLocation(4, i, 1000, 2000));
+            source.addPutSlot(*doc, Timestamp(1001 + i));
+            docs[Timestamp(1001 + i)] = doc;
+        }
+
+        env()._memFileMapper.flush(source, env());
+        VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source);
+    }
+
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options).maximumReadThroughGap(1024).build());
+    env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>(
+            new LoggingLazyFile::Factory());
+
+    MemFile target(file, env());
+
+    std::vector<Timestamp> timestamps;
+
+    for (int i = 0; i < 50; i+=4) {
+        timestamps.push_back(Timestamp(1001 + i));
+    }
+    CPPUNIT_ASSERT_EQUAL(size_t(13), timestamps.size());
+
+    getLoggerFile(target).operations.clear();
+    target.ensureDocumentCached(timestamps, false);
+    // Headers are small enough that they get read in 1 op + 13 body reads
+    CPPUNIT_ASSERT_EQUAL(14, (int)getLoggerFile(target).operations.size());
+
+    for (std::size_t i = 0; i < timestamps.size(); ++i) {
+        const MemSlot* slot = target.getSlotAtTime(timestamps[i]);
+        CPPUNIT_ASSERT(slot);
+        CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER));
+        CPPUNIT_ASSERT(target.partAvailable(*slot, BODY));
+        CPPUNIT_ASSERT_EQUAL(*docs[timestamps[i]], *target.getDocument(*slot, ALL));
+    }
+    VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target);
+}
+
+void
+MemFileV1SerializerTest::testWriteReadPartialRemoved()
+{
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    MemFile source(file, env());
+
+    for (int i = 0; i < 50; ++i) {
+        Document::SP doc(createRandomDocumentAtLocation(4, i, 1000, 2000));
+        source.addPutSlot(*doc, Timestamp(1001 + i));
+        source.addRemoveSlot(*source.getSlotAtTime(Timestamp(1001 + i)),
+                             Timestamp(2001 + i));
+    }
+
+    env()._memFileMapper.flush(source, env());
+    VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source);
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options).maximumReadThroughGap(1024).build());
+    env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>(
+            new LoggingLazyFile::Factory);
+
+    MemFile target(file, env());
+
+    std::vector<Timestamp> timestamps;
+
+    for (int i = 0; i < 50; i+=4) {
+        timestamps.push_back(Timestamp(2001 + i));
+    }
+
+    getLoggerFile(target).operations.clear();
+    target.ensureDocumentCached(timestamps, false);
+    // All removed; should only read header locations
+    CPPUNIT_ASSERT_EQUAL(1, (int)getLoggerFile(target).operations.size());
+
+    for (std::size_t i = 0; i < timestamps.size(); ++i) {
+        const MemSlot* slot = target.getSlotAtTime(timestamps[i]);
+        const MemSlot* removedPut(
+                target.getSlotAtTime(timestamps[i] - Timestamp(1000)));
+        CPPUNIT_ASSERT(slot);
+        CPPUNIT_ASSERT(removedPut);
+        CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER));
+        CPPUNIT_ASSERT_EQUAL(removedPut->getLocation(HEADER),
+                             slot->getLocation(HEADER));
+        CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), slot->getLocation(BODY));
+    }
+    VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target);
+}
+
+void MemFileV1SerializerTest::testWriteReadSingleRemovedDoc()
+{
+    VESPA_MEMFILEV1_SETUP_SOURCE;
+    source.addPutSlot(*doc, Timestamp(1001));
+    source.addRemoveSlot(
+            *source.getSlotAtTime(Timestamp(1001)), Timestamp(2001));
+    VESPA_MEMFILEV1_ASSERT_SERIALIZATION(source);
+}
+
+/**
+ * Write a single put with no body to the memfile and ensure it is
+ * persisted properly without a body block
+ */
+void
+MemFileV1SerializerTest::testPartialWritePutHeaderOnly()
+{
+    setUpPartialWriteEnvironment();
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    document::Document::SP doc(createRandomDocumentAtLocation(4));
+    {
+        MemFile source(file, env());
+        source.addPutSlot(*doc, Timestamp(1001));
+        env()._memFileMapper.flush(source, env());
+        VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source);
+    }
+    {
+        // Have to put a second time since the first one will always
+        // rewrite the entire file
+        MemFile target(file, env());
+        Document::SP doc2(createRandomDocumentAtLocation(4));
+        clearBody(*doc2);
+        target.addPutSlot(*doc2, Timestamp(1003));
+        env()._memFileMapper.flush(target, env());
+        VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target);
+    }
+    {
+        MemFile target(file, env());
+        target.ensureBodyBlockCached();
+        CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount());
+
+        const MemSlot& slot = *target.getSlotAtTime(Timestamp(1003));
+        CPPUNIT_ASSERT(slot.getLocation(HEADER)._pos > 0);
+        CPPUNIT_ASSERT(slot.getLocation(HEADER)._size > 0);
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(0, 0), slot.getLocation(BODY));
+        VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target);
+    }
+}
+
+
+
+
+void
+MemFileV1SerializerTest::testLocationDiskIoPlannerSimple()
+{
+    std::vector<MemSlot> slots;
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1001),
+                        DataLocation(0, 1024),
+                        DataLocation(4096, 512), 0, 0));
+    }
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1003),
+                        DataLocation(1024, 1024),
+                        DataLocation(8192, 512), 0, 0));
+    }
+
+    std::vector<DataLocation> headers;
+    std::vector<DataLocation> bodies;
+    headers.push_back(slots[0].getLocation(HEADER));
+    bodies.push_back(slots[0].getLocation(BODY));
+
+    DummyMemFileIOInterface dummyIo;
+    {
+        LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 100, 0);
+
+        CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size());
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(0, 1024),
+                planner.getIoOperations()[0]);
+    }
+    {
+        LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 100, 4096);
+
+        CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size());
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(8192, 512), // + block index
+                planner.getIoOperations()[0]);
+    }
+}
+
+void
+MemFileV1SerializerTest::testLocationDiskIoPlannerMergeReads()
+{
+    std::vector<MemSlot> slots;
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1001),
+                        DataLocation(0, 1024),
+                        DataLocation(5120, 512), 0, 0));
+    }
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1002),
+                        DataLocation(2048, 1024),
+                        DataLocation(7168, 512), 0, 0));
+    }
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1003),
+                        DataLocation(1024, 1024),
+                        DataLocation(9216, 512), 0, 0));
+    }
+
+    std::vector<DataLocation> headers;
+    std::vector<DataLocation> bodies;
+    for (int i = 0; i < 2; ++i) {
+        headers.push_back(slots[i].getLocation(HEADER));
+        bodies.push_back(slots[i].getLocation(BODY));
+    }
+
+    DummyMemFileIOInterface dummyIo;
+    {
+        LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 1025, 0);
+
+        CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size());
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(0, 3072),
+                planner.getIoOperations()[0]);
+    }
+
+    {
+        LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 1025, 0);
+
+        CPPUNIT_ASSERT_EQUAL(2, (int)planner.getIoOperations().size());
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(5120, 512),
+                planner.getIoOperations()[0]);
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(7168, 512),
+                planner.getIoOperations()[1]);
+    }
+}
+
+void
+MemFileV1SerializerTest::testLocationDiskIoPlannerOneDocument()
+{
+    std::vector<MemSlot> slots;
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1001),
+                        DataLocation(0, 1024),
+                        DataLocation(5120, 512), 0, 0));
+    }
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1002),
+                        DataLocation(2048, 1024),
+                        DataLocation(7168, 512), 0, 0));
+    }
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1003),
+                        DataLocation(1024, 1024),
+                        DataLocation(9216, 512), 0, 0));
+    }
+
+    std::vector<DataLocation> headers;
+    std::vector<DataLocation> bodies;
+    headers.push_back(slots[1].getLocation(HEADER));
+    bodies.push_back(slots[1].getLocation(BODY));
+
+    DummyMemFileIOInterface dummyIo;
+    {
+        LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 1000, 0);
+        CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size());
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(2048, 1024),
+                planner.getIoOperations()[0]);
+    }
+
+    {
+        LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 1000, 0);
+        CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size());
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(7168, 512),
+                planner.getIoOperations()[0]);
+    }
+}
+
+void
+MemFileV1SerializerTest::testLocationDiskIoPlannerAlignReads()
+{
+    std::vector<MemSlot> slots;
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1001),
+                        DataLocation(7, 100),
+                        DataLocation(5000, 500), 0, 0));
+    }
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1002),
+                        DataLocation(2000, 100),
+                        DataLocation(7000, 500), 0, 0));
+    }
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1003),
+                        DataLocation(110, 200),
+                        DataLocation(9000, 500), 0, 0));
+    }
+
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        slots.push_back(
+                MemSlot(
+                        doc->getId().getGlobalId(),
+                        Timestamp(1004),
+                        DataLocation(3000, 100),
+                        DataLocation(11000, 500), 0, 0));
+    }
+
+    std::vector<DataLocation> headers;
+    std::vector<DataLocation> bodies;
+    for (int i = 0; i < 2; ++i) {
+        headers.push_back(slots[i].getLocation(HEADER));
+        bodies.push_back(slots[i].getLocation(BODY));
+    }
+
+    DummyMemFileIOInterface dummyIo;
+    {
+        LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 512, 0);
+        std::vector<DataLocation> expected;
+        expected.push_back(DataLocation(0, 512));
+        expected.push_back(DataLocation(1536, 1024));
+
+        CPPUNIT_ASSERT_EQUAL(expected, planner.getIoOperations());
+    }
+    {
+        LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 512, 0);
+        std::vector<DataLocation> expected;
+        expected.push_back(DataLocation(4608, 1024));
+        expected.push_back(DataLocation(6656, 1024));
+
+        CPPUNIT_ASSERT_EQUAL(expected, planner.getIoOperations());
+    }
+}
+
+// TODO(vekterli): add read planner test with a location cached
+
+void
+MemFileV1SerializerTest::testSeparateReadsForHeaderAndBody()
+{
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    Document::SP doc(createRandomDocumentAtLocation(4, 0, 1000, 2000));
+    {
+        MemFile source(file, env());
+        source.addPutSlot(*doc, Timestamp(1001));
+
+        env()._memFileMapper.flush(source, env());
+    }
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options)
+                .maximumReadThroughGap(1024*1024*100)
+                .build());
+    env()._lazyFileFactory = std::unique_ptr<Environment::LazyFileFactory>(
+            new LoggingLazyFile::Factory());
+
+    MemFile target(file, env());
+
+    std::vector<Timestamp> timestamps;
+    timestamps.push_back(Timestamp(1001));
+
+    getLoggerFile(target).operations.clear();
+    target.ensureDocumentCached(timestamps, false);
+
+    CPPUNIT_ASSERT_EQUAL(2, (int)getLoggerFile(target).operations.size());
+    const MemSlot* slot = target.getSlotAtTime(Timestamp(1001));
+    CPPUNIT_ASSERT(slot);
+    CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER));
+    CPPUNIT_ASSERT(target.partAvailable(*slot, BODY));
+    CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(*slot, ALL));
+
+    CPPUNIT_ASSERT(getMetrics().serialization.headerReadSize.getLast() > 0);
+    CPPUNIT_ASSERT(getMetrics().serialization.bodyReadSize.getLast() > 0);
+}
+
+/**
+ * Write a single put with body to the memfile and ensure it is
+ * persisted properly with both header and body blocks
+ */
+void
+MemFileV1SerializerTest::testPartialWritePut()
+{
+    setUpPartialWriteEnvironment();
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    Document::SP doc(createRandomDocumentAtLocation(4));
+    {
+        MemFile source(file, env());
+        source.addPutSlot(*doc, Timestamp(1001));
+
+        env()._memFileMapper.flush(source, env());
+    }
+
+    {
+        // Have to put a second time since the first one will always
+        // rewrite the entire file
+        MemFile target(file, env());
+        Document::SP doc2(createRandomDocumentAtLocation(4));
+        target.addPutSlot(*doc2, Timestamp(1003));
+        env()._memFileMapper.flush(target, env());
+    }
+    {
+        MemFile target(file, env());
+        target.ensureBodyBlockCached();
+        CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount());
+
+        const MemSlot& slot = *target.getSlotAtTime(Timestamp(1003));
+        CPPUNIT_ASSERT(slot.getLocation(HEADER)._pos > 0);
+        CPPUNIT_ASSERT(slot.getLocation(HEADER)._size > 0);
+
+        CPPUNIT_ASSERT(slot.getLocation(BODY)._size > 0);
+        CPPUNIT_ASSERT(slot.getLocation(BODY)._pos > 0);
+    }
+}
+
+void
+MemFileV1SerializerTest::doTestPartialWriteRemove(bool readAll)
+{
+    setUpPartialWriteEnvironment();
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    Document::SP doc(createRandomDocumentAtLocation(4));
+    {
+        MemFile source(file, env());
+        source.addPutSlot(*doc, Timestamp(1001));
+        env()._memFileMapper.flush(source, env());
+    }
+    {
+        MemFile target(file, env());
+        // Only populate cache before removing if explicitly told so
+        if (readAll) {
+            target.ensureBodyBlockCached();
+        }
+        CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getSlotCount());
+        target.addRemoveSlot(target[0], Timestamp(1003));
+
+        env()._memFileMapper.flush(target, env());
+    }
+    {
+        MemFile target(file, env());
+        target.ensureBodyBlockCached();
+
+        CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount());
+
+        const MemSlot& originalSlot = target[0];
+        const MemSlot& removeSlot = target[1];
+        CPPUNIT_ASSERT(originalSlot.getLocation(HEADER)._size > 0);
+        CPPUNIT_ASSERT(originalSlot.getLocation(BODY)._size > 0);
+        CPPUNIT_ASSERT_EQUAL(
+                originalSlot.getLocation(HEADER),
+                removeSlot.getLocation(HEADER));
+        CPPUNIT_ASSERT_EQUAL(
+                DataLocation(0, 0), removeSlot.getLocation(BODY));
+    }
+}
+
+/**
+ * Ensure that removes get the same header location as the Put
+ * they're removing, and that they get a zero body location
+ */
+void
+MemFileV1SerializerTest::testPartialWriteRemoveCached()
+{
+    doTestPartialWriteRemove(true);
+}
+
+void
+MemFileV1SerializerTest::testPartialWriteRemoveNotCached()
+{
+    doTestPartialWriteRemove(false);
+}
+
+void
+MemFileV1SerializerTest::doTestPartialWriteUpdate(bool readAll)
+{
+    setUpPartialWriteEnvironment();
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    Document::SP doc(createRandomDocumentAtLocation(4));
+    {
+        MemFile source(file, env());
+        source.addPutSlot(*doc, Timestamp(1001));
+        env()._memFileMapper.flush(source, env());
+    }
+
+    Document::SP doc2;
+    {
+        MemFile target(file, env());
+        if (readAll) {
+            target.ensureBodyBlockCached();
+        }
+
+        doc2.reset(new Document(*doc->getDataType(), doc->getId()));
+        clearBody(*doc2);
+        doc2->setValue(doc->getField("hstringval"),
+                       document::StringFieldValue("Some updated content"));
+
+        target.addUpdateSlot(*doc2, *target.getSlotAtTime(Timestamp(1001)),
+                             Timestamp(1003));
+        env()._memFileMapper.flush(target, env());
+    }
+
+    {
+        MemFile target(file, env());
+        CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount());
+        const MemSlot& originalSlot = target[0];
+        const MemSlot& updateSlot = target[1];
+        CPPUNIT_ASSERT(originalSlot.getLocation(HEADER)._size > 0);
+        CPPUNIT_ASSERT(originalSlot.getLocation(BODY)._size > 0);
+        CPPUNIT_ASSERT_EQUAL(
+                originalSlot.getLocation(BODY),
+                updateSlot.getLocation(BODY));
+        CPPUNIT_ASSERT(
+                updateSlot.getLocation(HEADER)
+                != originalSlot.getLocation(HEADER));
+
+        CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(target[0], ALL));
+        copyHeader(*doc, *doc2);
+        CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(target[1], ALL));
+    }
+}
+
+/**
+ * Ensure that header updates keep the same body block
+ */
+void
+MemFileV1SerializerTest::testPartialWriteUpdateCached()
+{
+    doTestPartialWriteUpdate(true);
+}
+
+void
+MemFileV1SerializerTest::testPartialWriteUpdateNotCached()
+{
+    doTestPartialWriteUpdate(false);
+}
+
+void
+MemFileV1SerializerTest::testPartialWriteTooMuchFreeSpace()
+{
+    setUpPartialWriteEnvironment();
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    {
+        MemFile source(file, env());
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        source.addPutSlot(*doc, Timestamp(1001));
+        env()._memFileMapper.flush(source, env());
+    }
+    int64_t sizeBefore;
+    // Append filler to slotfile to make it too big for comfort,
+    // forcing a rewrite to shrink it down
+    {
+        vespalib::File slotfile(file.getPath());
+        slotfile.open(0);
+        CPPUNIT_ASSERT(slotfile.isOpen());
+        sizeBefore = slotfile.getFileSize();
+        slotfile.resize(sizeBefore * 20); // Well over min fill rate of 10%
+    }
+    // Write new slot to file; it should now be rewritten with the
+    // same file size as originally
+    {
+        MemFile source(file, env());
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        source.addPutSlot(*doc, Timestamp(1003));
+        env()._memFileMapper.flush(source, env());
+    }
+    {
+        vespalib::File slotfile(file.getPath());
+        slotfile.open(0);
+        CPPUNIT_ASSERT(slotfile.isOpen());
+        CPPUNIT_ASSERT_EQUAL(
+                sizeBefore,
+                slotfile.getFileSize());
+    }
+    CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization
+                         .fullRewritesDueToDownsizingFile.getValue());
+    CPPUNIT_ASSERT_EQUAL(uint64_t(0), getMetrics().serialization
+                         .fullRewritesDueToTooSmallFile.getValue());
+}
+
+void
+MemFileV1SerializerTest::testPartialWriteNotEnoughFreeSpace()
+{
+    setUpPartialWriteEnvironment();
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    // Write file initially
+    MemFile source(file, env());
+    {
+        Document::SP doc(createRandomDocumentAtLocation(4));
+        source.addPutSlot(*doc, Timestamp(1001));
+        env()._memFileMapper.flush(source, env());
+    }
+
+    uint32_t minFile = 1024 * 512;
+    auto memFileCfg = env().acquireConfigReadLock().memFilePersistenceConfig();
+    resetConfig(minFile, memFileCfg->minimumFileHeaderBlockSize);
+
+    // Create doc bigger than initial minimum filesize,
+    // prompting a full rewrite
+    Document::SP doc(
+            createRandomDocumentAtLocation(4, 0, 4096, 4096));
+    source.addPutSlot(*doc, Timestamp(1003));
+
+    env()._memFileMapper.flush(source, env());
+
+    CPPUNIT_ASSERT_EQUAL(
+            minFile,
+            uint32_t(getFileHandle(source).getFileSize()));
+
+    CPPUNIT_ASSERT_EQUAL(uint64_t(0), getMetrics().serialization
+                         .fullRewritesDueToDownsizingFile.getValue());
+    CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization
+                         .fullRewritesDueToTooSmallFile.getValue());
+
+    // Now, ensure we respect minimum file size and don't try to
+    // "helpfully" rewrite the file again (try to detect full
+    // file rewrite with help from the fact we don't currently
+    // check whether or not the file is < the minimum filesize.
+    // If that changes, so must this)
+    memFileCfg = env().acquireConfigReadLock().memFilePersistenceConfig();
+    resetConfig(2 * minFile, memFileCfg->minimumFileHeaderBlockSize);
+
+    source.addRemoveSlot(*source.getSlotAtTime(Timestamp(1003)),
+                         Timestamp(1005));
+    env()._memFileMapper.flush(source, env());
+
+    CPPUNIT_ASSERT_EQUAL(
+            minFile,
+            uint32_t(getFileHandle(source).getFileSize()));
+
+    CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization
+                         .fullRewritesDueToTooSmallFile.getValue());
+}
+
+// Test that we don't mess up when remapping locations that
+// have already been written during the same operation. That is:
+//   part A is remapped (P1, S1) -> (P2, S2)
+//   part B is remapped (P2, S2) -> (P3, S3)
+// Obviously, part B should not overwrite the location of part A,
+// but this will happen if we don't do the updating in one batch.
+void
+MemFileV1SerializerTest::testLocationsRemappedConsistently()
+{
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+
+    std::map<Timestamp, Document::SP> docs;
+    {
+        MemFile mf(file, env());
+        Document::SP tmpDoc(
+                createRandomDocumentAtLocation(4, 0, 100, 100));
+
+        // Create docs identical in size but differing only in doc ids
+        // By keeping same size but inserting with _lower_ timestamps
+        // for docs that get higher location positions, we ensure that
+        // when the file is rewritten, the lower timestamp slots will
+        // get remapped to locations that match existing locations for
+        // higher timestamp slots.
+        for (int i = 0; i < 2; ++i) {
+            std::ostringstream ss;
+            ss << "doc" << i;
+            DocumentId id(document::UserDocIdString("userdoc:foo:4:" + ss.str()));
+            Document::SP doc(new Document(*tmpDoc->getDataType(), id));
+            doc->getFields() = tmpDoc->getFields();
+            mf.addPutSlot(*doc, Timestamp(1000 - i));
+            docs[Timestamp(1000 - i)] = doc;
+        }
+
+        env()._memFileMapper.flush(mf, env());
+        // Dirty the cache for rewrite
+        {
+            DocumentId id2(document::UserDocIdString("userdoc:foo:4:doc9"));
+            Document::UP doc2(new Document(*tmpDoc->getDataType(), id2));
+            doc2->getFields() = tmpDoc->getFields();
+            mf.addPutSlot(*doc2, Timestamp(2000));
+            docs[Timestamp(2000)] = std::move(doc2);
+        }
+
+        // Force rewrite
+        auto memFileCfg = env().acquireConfigReadLock()
+                            .memFilePersistenceConfig();
+        resetConfig(1024*512, memFileCfg ->minimumFileHeaderBlockSize);
+        env()._memFileMapper.flush(mf, env());
+    }
+
+    MemFile target(file, env());
+    target.ensureBodyBlockCached();
+
+    std::ostringstream err;
+    if (!env()._memFileMapper.verify(target, env(), err)) {
+        std::cerr << err.str() << "\n";
+        CPPUNIT_FAIL("MemFile verification failed");
+    }
+
+    typedef std::map<Timestamp, Document::SP>::iterator Iter;
+    for (Iter it(docs.begin()); it != docs.end(); ++it) {
+        const MemSlot* slot = target.getSlotAtTime(it->first);
+        CPPUNIT_ASSERT(slot);
+        CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER));
+        CPPUNIT_ASSERT(target.partAvailable(*slot, BODY));
+        CPPUNIT_ASSERT_EQUAL(*it->second, *target.getDocument(*slot, ALL));
+    }
+}
+
+/**
+ * Test that we read in the correct header information when we have to read
+ * in two passes to get it in its entirety.
+ */
+void
+MemFileV1SerializerTest::testHeaderBufferTooSmall()
+{
+    system("rm -f testfile.0");
+    FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0");
+    FileInfo wantedInfo;
+    {
+        MemFile f(file, env());
+        // 50*40 bytes of meta list data should be more than sufficient
+        for (size_t i = 0; i < 50; ++i) {
+            Document::SP doc(createRandomDocumentAtLocation(4, i));
+            f.addPutSlot(*doc, Timestamp(1001 + i));
+            env()._memFileMapper.flush(f, env());
+        }
+        SimpleMemFileIOBuffer& io(
+                dynamic_cast<SimpleMemFileIOBuffer&>(f.getMemFileIO()));
+        wantedInfo = io.getFileInfo();
+    }
+
+    // Force initial index read to be too small to contain all metadata,
+    // triggering buffer resize and secondary read.
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options).initialIndexRead(512).build());
+    {
+        MemFile f(file, env());
+        CPPUNIT_ASSERT_EQUAL(uint32_t(50), f.getSlotCount());
+        // Ensure we've read correct file info
+        SimpleMemFileIOBuffer& io(
+                dynamic_cast<SimpleMemFileIOBuffer&>(f.getMemFileIO()));
+        const FileInfo& info(io.getFileInfo());
+        CPPUNIT_ASSERT_EQUAL(wantedInfo.getFileSize(), info.getFileSize());
+        CPPUNIT_ASSERT_EQUAL(wantedInfo.getHeaderBlockStartIndex(),
+                             info.getHeaderBlockStartIndex());
+        CPPUNIT_ASSERT_EQUAL(wantedInfo.getBodyBlockStartIndex(),
+                             info.getBodyBlockStartIndex());
+        CPPUNIT_ASSERT_EQUAL(wantedInfo.getBlockSize(HEADER),
+                             info.getBlockSize(HEADER));
+        CPPUNIT_ASSERT_EQUAL(wantedInfo.getBlockSize(BODY),
+                             info.getBlockSize(BODY));
+    }
+}
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp b/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp
new file mode 100644
index 00000000000..0cf04eadaa2
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp
@@ -0,0 +1,501 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/mapper/memfilemapper.h>
+#include <vespa/memfilepersistence/mapper/memfile_v1_serializer.h>
+#include <vespa/memfilepersistence/mapper/memfile_v1_verifier.h>
+#include <vespa/memfilepersistence/mapper/fileinfo.h>
+#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h>
+#include <tests/spi/memfiletestutils.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <tests/spi/simulatedfailurefile.h>
+
+namespace storage {
+namespace memfile {
+
+struct MemFileV1VerifierTest : public SingleDiskMemFileTestUtils
+{
+    void testVerify();
+
+    void tearDown();
+
+    std::unique_ptr<MemFile> createMemFile(FileSpecification& file,
+                                         bool callLoadFile)
+    {
+        return std::unique_ptr<MemFile>(new MemFile(file, env(), callLoadFile));
+    }
+
+    CPPUNIT_TEST_SUITE(MemFileV1VerifierTest);
+    CPPUNIT_TEST_IGNORED(testVerify);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MemFileV1VerifierTest);
+
+namespace {
+    // A totall uncached memfile with content to use for verify testing
+    std::unique_ptr<MemFile> _memFile;
+
+    // Clear old content. Create new file. Make sure nothing is cached.
+    void prepareBucket(SingleDiskMemFileTestUtils& util,
+                       const FileSpecification& file) {
+        _memFile.reset();
+        util.env()._cache.clear();
+        vespalib::unlink(file.getPath());
+        util.createTestBucket(file.getBucketId(), 0);
+        util.env()._cache.clear();
+        _memFile.reset(new MemFile(file, util.env()));
+        _memFile->getMemFileIO().close();
+
+    }
+
+    // Get copy of header of memfile created
+    Header getHeader() {
+        assert(_memFile.get());
+        vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
+        Header result;
+        file.read(&result, sizeof(Header), 0);
+        return result;
+    }
+
+    MetaSlot getSlot(uint32_t index) {
+        assert(_memFile.get());
+        vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
+        MetaSlot result;
+        file.read(&result, sizeof(MetaSlot),
+                  sizeof(Header) + sizeof(MetaSlot) * index);
+        return result;
+    }
+
+    void setSlot(uint32_t index, MetaSlot slot,
+                 bool updateFileChecksum = true)
+    {
+        (void)updateFileChecksum;
+        assert(_memFile.get());
+        //if (updateFileChecksum) slot.updateFileChecksum();
+        vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
+        file.write(&slot, sizeof(MetaSlot),
+                   sizeof(Header) + sizeof(MetaSlot) * index);
+    }
+
+    void setHeader(const Header& header) {
+        assert(_memFile.get());
+        vespalib::LazyFile file(_memFile->getFile().getPath(), 0);
+        file.write(&header, sizeof(Header), 0);
+    }
+
+    void verifySlotFile(MemFileV1VerifierTest& util,
+                        const std::string& expectedError,
+                        const std::string& message,
+                        int32_t remainingEntries,
+                        bool includeContent = true,
+                        bool includeHeader = true)
+    {
+        assert(_memFile.get());
+        FileSpecification file(_memFile->getFile());
+        _memFile.reset();
+        _memFile = util.createMemFile(file, false);
+        std::ostringstream before;
+        try{
+            util.env()._memFileMapper.loadFile(*_memFile, util.env(), false);
+            _memFile->print(before, true, "");
+        } catch (vespalib::Exception& e) {
+            before << "Unknown. Exception during loadFile\n";
+        }
+        std::ostringstream errors;
+        uint32_t flags = (includeContent ? 0 : Types::DONT_VERIFY_BODY)
+                       | (includeHeader ? 0 : Types::DONT_VERIFY_HEADER);
+        if (util.env()._memFileMapper.verify(
+                    *_memFile, util.env(), errors, flags))
+        {
+            _memFile->print(std::cerr, true, "");
+            std::cerr << errors.str() << "\n";
+            CPPUNIT_FAIL("verify() failed to detect: " + message);
+        }
+        CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(),
+                                       expectedError, errors.str());
+        errors.str("");
+        if (util.env()._memFileMapper.repair(
+                    *_memFile, util.env(), errors, flags))
+        {
+            CPPUNIT_FAIL("repair() failed to detect: " + message
+                         + ": " + errors.str());
+        }
+        CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(),
+                                       expectedError, errors.str());
+        std::ostringstream remainingErrors;
+        if (!util.env()._memFileMapper.verify(
+                    *_memFile, util.env(), remainingErrors, flags))
+        {
+            CPPUNIT_FAIL("verify() returns issue after repair of: "
+                         + message + ": " + remainingErrors.str());
+        }
+        CPPUNIT_ASSERT_MESSAGE(remainingErrors.str(),
+                               remainingErrors.str().size() == 0);
+        if (remainingEntries < 0) {
+            if (_memFile->fileExists()) {
+                CPPUNIT_FAIL(message + ": Expected file to not exist anymore");
+            }
+        } else if (dynamic_cast<SimpleMemFileIOBuffer&>(_memFile->getMemFileIO())
+                   .getFileHandle().getFileSize() == 0)
+        {
+            std::ostringstream ost;
+            ost << "Expected " << remainingEntries << " to remain in file, "
+                << "but file does not exist\n";
+            CPPUNIT_FAIL(message + ": " + ost.str());
+        } else {
+            if (int64_t(_memFile->getSlotCount()) != remainingEntries) {
+                std::ostringstream ost;
+                ost << "Expected " << remainingEntries << " to remain in file, "
+                    << "but found " << _memFile->getSlotCount() << "\n";
+                ost << errors.str() << "\n";
+                ost << "Before: " << before.str() << "\nAfter: ";
+                _memFile->print(ost, true, "");
+                CPPUNIT_FAIL(message + ": " + ost.str());
+            }
+        }
+    }
+}
+
+void
+MemFileV1VerifierTest::tearDown()
+{
+    _memFile.reset(0);
+    SingleDiskMemFileTestUtils::tearDown();
+};
+
+void
+MemFileV1VerifierTest::testVerify()
+{
+    BucketId bucket(16, 0xa);
+    std::unique_ptr<FileSpecification> file;
+    createTestBucket(bucket, 0);
+
+    {
+        MemFilePtr memFilePtr(env()._cache.get(bucket, env(), env().getDirectory()));
+        file.reset(new FileSpecification(memFilePtr->getFile()));
+        env()._cache.clear();
+    }
+    {   // Ensure buildTestFile builds a valid file
+            // Initial file should be fine.
+        MemFile memFile(*file, env());
+        std::ostringstream errors;
+        if (!env()._memFileMapper.verify(memFile, env(), errors)) {
+            memFile.print(std::cerr, false, "");
+            CPPUNIT_FAIL("Slotfile failed verification: " + errors.str());
+        }
+    }
+        // Header tests
+    prepareBucket(*this, *file);
+    Header orgheader(getHeader());
+    {   // Test wrong version
+        Header header(orgheader);
+        header.setVersion(0xc0edbabe);
+        header.updateChecksum();
+        setHeader(header);
+        verifySlotFile(*this,
+                       "400000000000000a.0 is of wrong version",
+                       "Faulty version",
+                       -1);
+    }
+    {   // Test meta data list size bigger than file
+        prepareBucket(*this, *file);
+        Header header(orgheader);
+        header.setMetaDataListSize(0xFFFF);
+        header.updateChecksum();
+        setHeader(header);
+        verifySlotFile(*this,
+                "indicates file is bigger than it physically is",
+                "Too big meta data list size",
+                -1);
+    }
+    {   // Test header block size bigger than file
+        prepareBucket(*this, *file);
+        Header header(orgheader);
+        header.setHeaderBlockSize(0xFFFF);
+        header.updateChecksum();
+        setHeader(header);
+        verifySlotFile(*this,
+                "Header indicates file is bigger than it physically is",
+                "Too big header block size",
+                -1);
+    }
+    {   // Test wrong header crc
+        prepareBucket(*this, *file);
+        Header header(orgheader);
+        header.setMetaDataListSize(4);
+        setHeader(header);
+        verifySlotFile(*this,
+                "Header checksum mismatch",
+                "Wrong header checksum",
+                -1);
+    }
+        // Meta data tests
+    prepareBucket(*this, *file);
+    MetaSlot slot6(getSlot(6));
+    {   // Test extra removes - currently allowed
+        MetaSlot slot7(getSlot(7));
+        MetaSlot s(slot7);
+        s.setTimestamp(Timestamp(s._timestamp.getTime() - 1));
+        s.updateChecksum();
+        setSlot(6, s);
+        s.setTimestamp(Timestamp(s._timestamp.getTime() + 1));
+        s.updateChecksum();
+        setSlot(7, s);
+        std::ostringstream errors;
+        if (!env()._memFileMapper.verify(*_memFile, env(), errors)) {
+            _memFile->print(std::cerr, false, "");
+            std::cerr << errors.str() << "\n";
+            CPPUNIT_FAIL("Supposed to be legal with multiple remove values");
+        }
+        setSlot(7, slot7);
+    }
+    {
+        // Test metadata crc mismatch with "used" flag being accidentally
+        // flipped. Should not inhibit adding of subsequent slots.
+        prepareBucket(*this, *file);
+        MetaSlot s(slot6);
+        s.setUseFlag(false);
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "Slot 6 at timestamp 2001 failed checksum verification",
+                "Crc failure with use flag", 23, false);
+    }
+    {   // Test overlapping documents
+        MetaSlot s(slot6);
+            // Direct overlapping header
+        prepareBucket(*this, *file);
+        s.setHeaderPos(0);
+        s.setHeaderSize(51);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "overlaps with slot",
+                "Direct overlapping header", 6, false, false);
+            // Contained header
+            // (contained bit not valid header so fails on other error now)
+        prepareBucket(*this, *file);
+        s.setHeaderPos(176);
+        s.setHeaderSize(80);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "not big enough to contain a document id",
+                "Contained header", 7, false);
+            // Partly overlapping header
+            // (contained bit not valid header so fails on other error now)
+        prepareBucket(*this, *file);
+        s.setHeaderPos(191);
+        s.setHeaderSize(35);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "not big enough to contain a document id",
+                "Partly overlapping header", 7, false);
+        prepareBucket(*this, *file);
+        s.setHeaderPos(185);
+        s.setHeaderSize(33);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "not big enough to contain a document id",
+                "Partly overlapping header (2)", 7, false);
+            // Direct overlapping body
+        prepareBucket(*this, *file);
+        s = slot6;
+        s.setBodyPos(0);
+        s.setBodySize(136);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "Multiple slots with different gids use same body position",
+                "Directly overlapping body", 6, false);
+            // Contained body
+        prepareBucket(*this, *file);
+        s.setBodyPos(10);
+        s.setBodySize(50);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "overlaps with slot",
+                "Contained body", 6, false);
+        CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1)) == 0);
+            // Overlapping body
+        prepareBucket(*this, *file);
+        s.setBodyPos(160);
+        s.setBodySize(40);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "overlaps with slot",
+                "Overlapping body", 5, false);
+        CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(2)) == 0);
+        CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1501)) == 0);
+            // Overlapping body, verifying bodies
+            // (Bad body bit should be removed first, so only one slot needs
+            // removing)
+        prepareBucket(*this, *file);
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "Body checksum mismatch",
+                "Overlapping body(2)", 7, true);
+    }
+    {   // Test out of bounds
+        MetaSlot s(slot6);
+
+            // Header out of bounds
+        prepareBucket(*this, *file);
+        s.setHeaderPos(500);
+        s.setHeaderSize(5000);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "goes out of bounds",
+                "Header out of bounds", 7, false, false);
+            // Body out of bounds
+        prepareBucket(*this, *file);
+        s = slot6;
+        s.setBodyPos(2400);
+        s.setBodySize(6000);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "goes out of bounds",
+                "Body out of bounds", 7, false);
+    }
+    {   // Test timestamp collision
+        prepareBucket(*this, *file);
+        MetaSlot s(slot6);
+        s.setTimestamp(Timestamp(10002));
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "has same timestamp as slot 5",
+                "Timestamp collision", 6, false);
+    }
+    {   // Test timestamp out of order
+        prepareBucket(*this, *file);
+        MetaSlot s(slot6);
+        s.setTimestamp(Timestamp(38));
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "Slot 6 is out of timestamp order",
+                "Timestamp out of order", 8, false);
+    }
+    {   // Test metadata crc mismatch
+        prepareBucket(*this, *file);
+        MetaSlot s(slot6);
+        s.setTimestamp(Timestamp(40));
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "Slot 6 at timestamp 40 failed checksum verification",
+                "Crc failure", 7, false);
+    }
+    {   // Test used after unused
+        // This might actually lose documents after the unused entries.
+        // The memfile will not know about the documents after unused entry.
+        // If the memfile contains changes and writes metadata back due to this,
+        // the following entries will be missing.
+        // (To prevent this repair would have to add metadata entries, but that
+        // may be problems if repair happens at a time where all header or body
+        // data in the file needs to be cached.)
+        prepareBucket(*this, *file);
+        MetaSlot s(slot6);
+        s.setUseFlag(false);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "Slot 7 found after unused entries",
+                "Used after unused", 6, false);
+    }
+    {   // Test header blob corrupt
+        prepareBucket(*this, *file);
+        MetaSlot s(slot6);
+        s.setHeaderPos(519);
+        s.setHeaderSize(86);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "Header checksum mismatch",
+                "Corrupt header blob.", 7);
+    }
+    {   // Test body blob corrupt
+        prepareBucket(*this, *file);
+        MetaSlot s(slot6);
+        s.setBodyPos(52);
+        s.setBodySize(18);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "Body checksum mismatch",
+                "Corrupt body blob.", 7);
+    }
+    {   // Test too long name for header chunk
+        prepareBucket(*this, *file);
+        MetaSlot s(slot6);
+        s.setHeaderPos(160);
+        s.setHeaderSize(33);
+        s.updateChecksum();
+        setSlot(6, s);
+        verifySlotFile(*this,
+                "header is not big enough to contain a document",
+                "Too long name in header.", 7);
+    }
+    {   // Test wrong file checksum
+// Currently disabled. Currently only possible to calculate file checksum from
+// memfile now, and memfile object wont be valid.
+/*
+            // First test if we actually have less entries at all..
+        prepareBucket(*this, *file);
+        MetaSlot s(getSlot(7));
+        s.setUseFlag(false);
+        s.updateChecksum();
+        setSlot(7, s, false);
+        s = getSlot(8);
+        s.setUseFlag(false);
+        s.updateChecksum();
+        setSlot(8, s, false);
+        verifySlotFile(*this,
+                "File checksum should have been",
+                "Wrong file checksum in file.", 7, false);
+std::cerr << "U\n";
+            // Then test with different timestamp in remaining document
+        prepareBucket(*this, *file);
+        s = getSlot(6);
+        s.setTimestamp(s._timestamp + 1);
+        s.updateChecksum();
+        setSlot(6, s, false);
+        verifySlotFile(*this,
+                "File checksum should have been",
+                "Wrong file checksum in file.", 9, false);
+std::cerr << "V\n";
+            // Then check with different gid
+        prepareBucket(*this, *file);
+        s = getSlot(6);
+        s._gid = GlobalId("sdfsdfsedsdfsdfsd");
+        s.updateChecksum();
+        setSlot(6, s, false);
+        verifySlotFile(*this,
+                "File checksum should have been",
+                "Wrong file checksum in file.", 9, false, false);
+*/
+    }
+    {   // Test that documents not belonging in a bucket is removed
+// Currently disabled. Hard to test. Needs total rewrite
+/*
+        prepareBucket(*this, *file);
+        Blob b(createBlob(43u, "userdoc::0:315", "header", "body"));
+        _memFile->write(b, 80);
+        CPPUNIT_ASSERT_EQUAL(4u, _memFile->getBlobCount());
+        CPPUNIT_ASSERT(_memFile->read(b));
+        verifySlotFile(*this,
+                "belongs in bucket",
+                "Document not belonging there", 9);
+        CPPUNIT_ASSERT_EQUAL(3u, _memFile->getBlobCount());
+*/
+    }
+}
+
+}
+}
diff --git a/memfilepersistence/src/tests/spi/options_builder.h b/memfilepersistence/src/tests/spi/options_builder.h
new file mode 100644
index 00000000000..044e7f1d351
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/options_builder.h
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/memfilepersistence/common/environment.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <memory>
+
+namespace storage {
+namespace memfile {
+
+class OptionsBuilder
+{
+    Options _newOptions;
+public:
+    OptionsBuilder(const Options& opts)
+        : _newOptions(opts)
+    {
+    }
+
+    OptionsBuilder& maximumReadThroughGap(uint32_t readThroughGap) {
+        _newOptions._maximumGapToReadThrough = readThroughGap;
+        return *this;
+    }
+
+    OptionsBuilder& initialIndexRead(uint32_t bytesToRead) {
+        _newOptions._initialIndexRead = bytesToRead;
+        return *this;
+    }
+
+    OptionsBuilder& revertTimePeriod(framework::MicroSecTime revertTime) {
+        _newOptions._revertTimePeriod = revertTime;
+        return *this;
+    }
+
+    OptionsBuilder& defaultRemoveDocType(vespalib::stringref typeName) {
+        _newOptions._defaultRemoveDocType = typeName;
+        return *this;
+    }
+
+    OptionsBuilder& maxDocumentVersions(uint32_t maxVersions) {
+        _newOptions._maxDocumentVersions = maxVersions;
+        return *this;
+    }
+
+    std::unique_ptr<Options> build() const {
+        return std::unique_ptr<Options>(new Options(_newOptions));
+    }
+};
+
+} // memfile
+} // storage
+
diff --git a/memfilepersistence/src/tests/spi/providerconformancetest.cpp b/memfilepersistence/src/tests/spi/providerconformancetest.cpp
new file mode 100644
index 00000000000..526f61a812c
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/providerconformancetest.cpp
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/persistence/conformancetest/conformancetest.h>
+#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h>
+#include <vespa/storageframework/defaultimplementation/clock/realclock.h>
+#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h>
+#include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h>
+#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
+#include <vespa/memfilepersistence/memfile/memfilecache.h>
+#include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h>
+#include <tests/spi/memfiletestutils.h>
+
+LOG_SETUP(".test.dummyimpl");
+
+namespace storage {
+namespace memfile {
+
+struct ProviderConformanceTest : public spi::ConformanceTest {
+    struct Factory : public PersistenceFactory {
+        framework::defaultimplementation::ComponentRegisterImpl _compRegister;
+        framework::defaultimplementation::RealClock _clock;
+        framework::defaultimplementation::MemoryManager _memoryManager;
+        std::unique_ptr<MemFileCache> cache;
+
+        Factory()
+            : _compRegister(),
+              _clock(),
+              _memoryManager(
+                    framework::defaultimplementation::AllocationLogic::UP(
+                        new framework::defaultimplementation::SimpleMemoryLogic(
+                            _clock, 1024 * 1024 * 1024)))
+        {
+            _compRegister.setClock(_clock);
+            _compRegister.setMemoryManager(_memoryManager);
+        }
+
+        spi::PersistenceProvider::UP
+        getPersistenceImplementation(const document::DocumentTypeRepo::SP& repo,
+                                     const document::DocumenttypesConfig&)
+        {
+            system("rm -rf vdsroot");
+            system("mkdir -p vdsroot/disks/d0");
+            vdstestlib::DirConfig config(getStandardConfig(true));
+
+            MemFilePersistenceProvider::UP result(
+                    new MemFilePersistenceProvider(
+                            _compRegister,
+                            config.getConfigId()));
+            result->setDocumentRepo(*repo);
+            return spi::PersistenceProvider::UP(result.release());
+        }
+
+        bool
+        supportsRevert() const
+        {
+            return true;
+        }
+    };
+
+    ProviderConformanceTest()
+        : spi::ConformanceTest(PersistenceFactory::UP(new Factory)) {}
+
+    CPPUNIT_TEST_SUITE(ProviderConformanceTest);
+    DEFINE_CONFORMANCE_TESTS();
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ProviderConformanceTest);
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp b/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp
new file mode 100644
index 00000000000..fbf7badf5e4
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/memfilepersistence/memfile/shared_data_location_tracker.h>
+
+namespace storage {
+namespace memfile {
+
+class SharedDataLocationTrackerTest : public CppUnit::TestFixture
+{
+public:
+    void headerIsPassedDownToCacheAccessor();
+    void bodyIsPassedDownToCacheAccessor();
+    void firstInvocationReturnsNewLocation();
+    void multipleInvocationsForSharedSlotReturnSameLocation();
+
+    CPPUNIT_TEST_SUITE(SharedDataLocationTrackerTest);
+    CPPUNIT_TEST(headerIsPassedDownToCacheAccessor);
+    CPPUNIT_TEST(bodyIsPassedDownToCacheAccessor);
+    CPPUNIT_TEST(firstInvocationReturnsNewLocation);
+    CPPUNIT_TEST(multipleInvocationsForSharedSlotReturnSameLocation);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(SharedDataLocationTrackerTest);
+
+namespace {
+
+using Params = std::pair<Types::DocumentPart, DataLocation>;
+constexpr auto HEADER = Types::HEADER;
+constexpr auto BODY = Types::BODY;
+
+/**
+ * A simple mock of a buffer cache which records all invocations
+ * and returns a location increasing by 100 for each invocation.
+ */
+struct MockBufferCacheCopier : BufferCacheCopier
+{
+    // This is practically _screaming_ for GoogleMock.
+    std::vector<Params> invocations;
+
+    DataLocation doCopyFromSourceToLocal(
+            Types::DocumentPart part,
+            DataLocation sourceLocation) override
+    {
+        Params params(part, sourceLocation);
+        const size_t invocationsBefore = invocations.size();
+        invocations.push_back(params);
+        return DataLocation(invocationsBefore * 100,
+                            invocationsBefore * 100 + 100);
+    }
+};
+
+}
+
+void
+SharedDataLocationTrackerTest::headerIsPassedDownToCacheAccessor()
+{
+    MockBufferCacheCopier cache;
+    SharedDataLocationTracker tracker(cache, HEADER);
+    tracker.getOrCreateSharedLocation({0, 100});
+    CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size());
+    CPPUNIT_ASSERT_EQUAL(Params(HEADER, {0, 100}), cache.invocations[0]);
+}
+
+void
+SharedDataLocationTrackerTest::bodyIsPassedDownToCacheAccessor()
+{
+    MockBufferCacheCopier cache;
+    SharedDataLocationTracker tracker(cache, BODY);
+    tracker.getOrCreateSharedLocation({0, 100});
+    CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size());
+    CPPUNIT_ASSERT_EQUAL(Params(BODY, {0, 100}), cache.invocations[0]);
+}
+
+void
+SharedDataLocationTrackerTest::firstInvocationReturnsNewLocation()
+{
+    MockBufferCacheCopier cache;
+    SharedDataLocationTracker tracker(cache, HEADER);
+    // Auto-incrementing per cache copy invocation.
+    CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100),
+                         tracker.getOrCreateSharedLocation({500, 600}));
+    CPPUNIT_ASSERT_EQUAL(DataLocation(100, 200),
+                         tracker.getOrCreateSharedLocation({700, 800}));
+
+    CPPUNIT_ASSERT_EQUAL(size_t(2), cache.invocations.size());
+    CPPUNIT_ASSERT_EQUAL(Params(HEADER, {500, 600}), cache.invocations[0]);
+    CPPUNIT_ASSERT_EQUAL(Params(HEADER, {700, 800}), cache.invocations[1]);
+}
+
+void
+SharedDataLocationTrackerTest
+   ::multipleInvocationsForSharedSlotReturnSameLocation()
+{
+    MockBufferCacheCopier cache;
+    SharedDataLocationTracker tracker(cache, HEADER);
+    CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100),
+                         tracker.getOrCreateSharedLocation({500, 600}));
+    // Same source location, thus we can reuse the same destination location
+    // as well.
+    CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100),
+                         tracker.getOrCreateSharedLocation({500, 600}));
+
+    CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size());
+    CPPUNIT_ASSERT_EQUAL(Params(HEADER, {500, 600}), cache.invocations[0]);
+}
+
+} // memfile
+} // storage
+
diff --git a/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp b/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp
new file mode 100644
index 00000000000..af0466fafe7
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp
@@ -0,0 +1,663 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <tests/spi/memfiletestutils.h>
+#include <tests/spi/options_builder.h>
+
+namespace storage {
+namespace memfile {
+
+class SimpleMemFileIOBufferTest : public SingleDiskMemFileTestUtils
+{
+    CPPUNIT_TEST_SUITE(SimpleMemFileIOBufferTest);
+    CPPUNIT_TEST(testAddAndReadDocument);
+    CPPUNIT_TEST(testNonExistingLocation);
+    CPPUNIT_TEST(testCopy);
+    CPPUNIT_TEST(testCacheLocation);
+    CPPUNIT_TEST(testPersist);
+    CPPUNIT_TEST(testGetSerializedSize);
+    CPPUNIT_TEST(testRemapLocations);
+    CPPUNIT_TEST(testAlignmentUtilFunctions);
+    CPPUNIT_TEST(testCalculatedCacheSize);
+    CPPUNIT_TEST(testSharedBuffer);
+    CPPUNIT_TEST(testSharedBufferUsage);
+    CPPUNIT_TEST(testHeaderChunkEncoderComputesSizesCorrectly);
+    CPPUNIT_TEST(testHeaderChunkEncoderSerializesIdCorrectly);
+    CPPUNIT_TEST(testHeaderChunkEncoderSerializesHeaderCorrectly);
+    CPPUNIT_TEST(testRemovesCanBeWrittenWithBlankDefaultDocument);
+    CPPUNIT_TEST(testRemovesCanBeWrittenWithIdInferredDoctype);
+    CPPUNIT_TEST(testRemovesWithInvalidDocTypeThrowsException);
+    CPPUNIT_TEST_SUITE_END();
+
+    using BufferType = SimpleMemFileIOBuffer::BufferType;
+    using BufferLP = BufferType::LP;
+    using BufferAllocation = SimpleMemFileIOBuffer::BufferAllocation;
+    using HeaderChunkEncoder = SimpleMemFileIOBuffer::HeaderChunkEncoder;
+    using SimpleMemFileIOBufferUP = std::unique_ptr<SimpleMemFileIOBuffer>;
+
+    BufferAllocation allocateBuffer(size_t sz) {
+        return BufferAllocation(BufferLP(new BufferType(sz)), 0, sz);
+    }
+
+    /**
+     * Create an I/O buffer instance with for a dummy bucket. If removeDocType
+     * is non-empty, remove entries will be written in backwards compatible
+     * mode.
+     */
+    SimpleMemFileIOBufferUP createIoBufferWithDummySpec(
+            vespalib::stringref removeDocType = "");
+
+public:
+    class DummyFileReader : public VersionSerializer {
+    public:
+        virtual FileVersion getFileVersion() { return FileVersion(); }
+        virtual void loadFile(MemFile&, Environment&,
+                              Buffer&, uint64_t ) {}
+        virtual FlushResult flushUpdatesToFile(MemFile&, Environment&) {
+            return FlushResult::TooSmall;
+        }
+        virtual void rewriteFile(MemFile&, Environment&) {}
+        virtual bool verify(MemFile&, Environment&,
+                            std::ostream&, bool,
+                            uint16_t) { return false; };
+        virtual void cacheLocations(MemFileIOInterface&,
+                                    Environment&,
+                                    const Options&,
+                                    DocumentPart,
+                                    const std::vector<DataLocation>&) {}
+    };
+
+    DummyFileReader dfr;
+
+    void testAddAndReadDocument();
+    void testNonExistingLocation();
+    void testCopy();
+    void testCacheLocation();
+    void testPersist();
+    void testGetSerializedSize();
+    void testRemapLocations();
+    void testAlignmentUtilFunctions();
+    void testCalculatedCacheSize();
+    void testSharedBuffer();
+    void testSharedBufferUsage();
+    void testHeaderChunkEncoderComputesSizesCorrectly();
+    void testHeaderChunkEncoderSerializesIdCorrectly();
+    void testHeaderChunkEncoderSerializesHeaderCorrectly();
+    void testRemovesCanBeWrittenWithBlankDefaultDocument();
+    void testRemovesCanBeWrittenWithIdInferredDoctype();
+    void testRemovesWithInvalidDocTypeThrowsException();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(SimpleMemFileIOBufferTest);
+
+
+void
+SimpleMemFileIOBufferTest::testAddAndReadDocument()
+{
+    FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0");
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       123,
+                                       456,
+                                       789,
+                                       1234));
+
+    SimpleMemFileIOBuffer buffer(dfr,
+                                 vespalib::LazyFile::UP(),
+                                 std::unique_ptr<FileInfo>(new FileInfo),
+                                 fileSpec,
+                                 env());
+
+    DataLocation h = buffer.addHeader(*doc);
+    DataLocation b = buffer.addBody(*doc);
+
+    Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), h);
+    buffer.readBody(*getTypeRepo(), b, *newDoc);
+
+    CPPUNIT_ASSERT_EQUAL(*doc, *newDoc);
+    CPPUNIT_ASSERT_EQUAL(true, buffer.isCached(h, HEADER));
+    CPPUNIT_ASSERT_EQUAL(true, buffer.isCached(b, BODY));
+    CPPUNIT_ASSERT_EQUAL(false, buffer.isCached(h, BODY));
+    CPPUNIT_ASSERT_EQUAL(false, buffer.isCached(b, HEADER));
+    CPPUNIT_ASSERT_EQUAL(doc->getId(), buffer.getDocumentId(h));
+}
+
+void
+SimpleMemFileIOBufferTest::testPersist()
+{
+    FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0");
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       123,
+                                       456,
+                                       789,
+                                       1234));
+
+    SimpleMemFileIOBuffer buffer(dfr,
+                                 vespalib::LazyFile::UP(),
+                                 std::unique_ptr<FileInfo>(new FileInfo),
+                                 fileSpec,
+                                 env());
+
+    DataLocation h = buffer.addHeader(*doc);
+    DataLocation b = buffer.addBody(*doc);
+
+    CPPUNIT_ASSERT(!buffer.isPersisted(h, HEADER));
+    CPPUNIT_ASSERT(!buffer.isPersisted(b, BODY));
+
+    buffer.persist(HEADER, h, DataLocation(1000, h.size()));
+    buffer.persist(BODY, b, DataLocation(5000, b.size()));
+
+    Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), DataLocation(1000, h.size()));
+    buffer.readBody(*getTypeRepo(), DataLocation(5000, b.size()), *newDoc);
+
+    CPPUNIT_ASSERT(buffer.isPersisted(DataLocation(1000, h.size()), HEADER));
+    CPPUNIT_ASSERT(buffer.isPersisted(DataLocation(5000, b.size()), BODY));
+
+    CPPUNIT_ASSERT_EQUAL(*doc, *newDoc);
+}
+
+void
+SimpleMemFileIOBufferTest::testCopy()
+{
+    FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0");
+    SimpleMemFileIOBuffer buffer(dfr,
+                                 vespalib::LazyFile::UP(),
+                                 std::unique_ptr<FileInfo>(new FileInfo),
+                                 fileSpec,
+                                 env());
+
+    for (uint32_t i = 0; i < 10; ++i) {
+        document::Document::SP doc(createRandomDocumentAtLocation(
+                                           123,
+                                           456,
+                                           789,
+                                           1234));
+
+        DataLocation h = buffer.addHeader(*doc);
+        DataLocation b = buffer.addBody(*doc);
+
+        SimpleMemFileIOBuffer buffer2(dfr,
+                                      vespalib::LazyFile::UP(),
+                                      std::unique_ptr<FileInfo>(new FileInfo),
+                                      fileSpec,
+                                      env());
+
+        DataLocation h2 = buffer2.copyCache(buffer, HEADER, h);
+        DataLocation b2 = buffer2.copyCache(buffer, BODY, b);
+
+        Document::UP newDoc = buffer2.getDocumentHeader(*getTypeRepo(), h2);
+        buffer2.readBody(*getTypeRepo(), b2, *newDoc);
+
+        CPPUNIT_ASSERT_EQUAL(*doc, *newDoc);
+    }
+}
+
+void
+SimpleMemFileIOBufferTest::testNonExistingLocation()
+{
+    FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0");
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       123,
+                                       456,
+                                       789,
+                                       1234));
+
+    SimpleMemFileIOBuffer buffer(dfr,
+                                 vespalib::LazyFile::UP(),
+                                 std::unique_ptr<FileInfo>(new FileInfo),
+                                 fileSpec,
+                                 env());
+
+    DataLocation h = buffer.addHeader(*doc);
+    DataLocation b = buffer.addBody(*doc);
+
+    buffer.clear(HEADER);
+
+    try {
+        Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), h);
+        CPPUNIT_ASSERT(false);
+    } catch (SimpleMemFileIOBuffer::PartNotCachedException& e) {
+    }
+
+    buffer.clear(BODY);
+
+    try {
+        document::Document newDoc;
+        buffer.readBody(*getTypeRepo(), b, newDoc);
+        CPPUNIT_ASSERT(false);
+    } catch (SimpleMemFileIOBuffer::PartNotCachedException& e) {
+    }
+}
+
+void
+SimpleMemFileIOBufferTest::testCacheLocation()
+{
+    FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0");
+
+    SimpleMemFileIOBuffer buffer(dfr,
+                                 vespalib::LazyFile::UP(),
+                                 FileInfo::UP(new FileInfo(100, 10000, 50000)),
+                                 fileSpec,
+                                 env());
+
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       123,
+                                       456,
+                                       789,
+                                       1234));
+
+    BufferAllocation headerBuf = buffer.serializeHeader(*doc);
+    BufferAllocation bodyBuf = buffer.serializeBody(*doc);
+
+    DataLocation hloc(1234, headerBuf.getSize());
+    DataLocation bloc(5678, bodyBuf.getSize());
+
+    buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0);
+    buffer.cacheLocation(BODY, bloc, bodyBuf.getSharedBuffer(), 0);
+
+    Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), hloc);
+    buffer.readBody(*getTypeRepo(), bloc, *newDoc);
+
+    CPPUNIT_ASSERT_EQUAL(*doc, *newDoc);
+}
+
+void
+SimpleMemFileIOBufferTest::testGetSerializedSize()
+{
+    FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0");
+
+    SimpleMemFileIOBuffer buffer(dfr,
+                                 vespalib::LazyFile::UP(),
+                                 FileInfo::UP(new FileInfo(100, 10000, 50000)),
+                                 fileSpec,
+                                 env());
+
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       123,
+                                       456,
+                                       789,
+                                       1234));
+
+    BufferAllocation headerBuf = buffer.serializeHeader(*doc);
+    BufferAllocation bodyBuf = buffer.serializeBody(*doc);
+
+    DataLocation hloc(1234, headerBuf.getSize());
+    DataLocation bloc(5678, bodyBuf.getSize());
+
+    buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0);
+    buffer.cacheLocation(BODY, bloc, bodyBuf.getSharedBuffer(), 0);
+
+    vespalib::nbostream serializedHeader;
+    doc->serializeHeader(serializedHeader);
+
+    vespalib::nbostream serializedBody;
+    doc->serializeBody(serializedBody);
+
+    CPPUNIT_ASSERT_EQUAL(uint32_t(serializedHeader.size()),
+                         buffer.getSerializedSize(HEADER, hloc));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(serializedBody.size()),
+                         buffer.getSerializedSize(BODY, bloc));
+}
+
+// Test that remapping does not overwrite datalocations that it has
+// already updated
+void
+SimpleMemFileIOBufferTest::testRemapLocations()
+{
+    FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0");
+
+    SimpleMemFileIOBuffer buffer(dfr,
+                                 vespalib::LazyFile::UP(),
+                                 FileInfo::UP(new FileInfo(100, 10000, 50000)),
+                                 fileSpec,
+                                 env());
+
+    document::Document::SP doc(createRandomDocumentAtLocation(
+                                       123,
+                                       100,
+                                       100));
+    BufferAllocation headerBuf = buffer.serializeHeader(*doc);
+    BufferAllocation bodyBuf = buffer.serializeBody(*doc);
+
+    document::Document::SP doc2(createRandomDocumentAtLocation(
+                                       123,
+                                       100,
+                                       100));
+
+    BufferAllocation headerBuf2 = buffer.serializeHeader(*doc2);
+    BufferAllocation bodyBuf2 = buffer.serializeBody(*doc2);
+
+    DataLocation hloc(30000, headerBuf.getSize());
+    DataLocation hloc2(0, headerBuf2.getSize());
+    DataLocation hloc3(10000, hloc2._size);
+
+    buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0);
+    buffer.cacheLocation(HEADER, hloc2, headerBuf2.getSharedBuffer(), 0);
+
+    std::map<DataLocation, DataLocation> remapping;
+    remapping[hloc2] = hloc;
+    remapping[hloc] = hloc3;
+
+    buffer.remapAndPersistAllLocations(HEADER, remapping);
+
+    Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), hloc3);
+    document::ByteBuffer bbuf(bodyBuf.getBuffer(), bodyBuf.getSize());
+    newDoc->deserializeBody(*getTypeRepo(), bbuf);
+
+    CPPUNIT_ASSERT_EQUAL(*doc, *newDoc);
+
+    Document::UP newDoc2 = buffer.getDocumentHeader(*getTypeRepo(), hloc);
+    document::ByteBuffer bbuf2(bodyBuf.getBuffer(), bodyBuf.getSize());
+    newDoc2->deserializeBody(*getTypeRepo(), bbuf2);
+    CPPUNIT_ASSERT_EQUAL(*doc2, *newDoc2);
+}
+
+/**
+ * Not technically a part of SimpleMemFileIOBuffer, but used by it and
+ * currently contained within its header file. Move test somewhere else
+ * if the code itself is moved.
+ */
+void
+SimpleMemFileIOBufferTest::testAlignmentUtilFunctions()
+{
+    using namespace util;
+    CPPUNIT_ASSERT_EQUAL(size_t(0), alignUpPow2<4096>(0));
+    CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(1));
+    CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(512));
+    CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(4096));
+    CPPUNIT_ASSERT_EQUAL(size_t(8192), alignUpPow2<4096>(4097));
+    CPPUNIT_ASSERT_EQUAL(size_t(32), alignUpPow2<16>(20));
+    CPPUNIT_ASSERT_EQUAL(size_t(32), alignUpPow2<32>(20));
+    CPPUNIT_ASSERT_EQUAL(size_t(64), alignUpPow2<64>(20));
+    CPPUNIT_ASSERT_EQUAL(size_t(128), alignUpPow2<128>(20));
+
+    CPPUNIT_ASSERT_EQUAL(uint32_t(0), nextPow2(0));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(1), nextPow2(1));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(4), nextPow2(3));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(16), nextPow2(15));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(64), nextPow2(40));
+    CPPUNIT_ASSERT_EQUAL(uint32_t(64), nextPow2(64));
+}
+
+/**
+ * Test that allocated buffers are correctly reported with their sizes
+ * rounded up to account for mmap overhead.
+ */
+void
+SimpleMemFileIOBufferTest::testCalculatedCacheSize()
+{
+    FileSpecification fileSpec(BucketId(16, 123),
+                               env().getDirectory(), "testfile.0");
+    SimpleMemFileIOBuffer buffer(dfr,
+                                 vespalib::LazyFile::UP(),
+                                 std::unique_ptr<FileInfo>(new FileInfo),
+                                 fileSpec,
+                                 env());
+
+    CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(HEADER));
+    CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(BODY));
+
+    // All buffers are on a 4k page granularity.
+    BufferAllocation sharedHeaderBuffer(allocateBuffer(1500)); // -> 4096
+    buffer.cacheLocation(HEADER, DataLocation(0, 85),
+                         sharedHeaderBuffer.getSharedBuffer(), 0);
+    CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(HEADER));
+
+    buffer.cacheLocation(HEADER, DataLocation(200, 100),
+                         sharedHeaderBuffer.getSharedBuffer(), 85);
+    CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(HEADER));
+
+    BufferAllocation singleHeaderBuffer(allocateBuffer(200)); // -> 4096
+    buffer.cacheLocation(HEADER, DataLocation(0, 100),
+                         singleHeaderBuffer.getSharedBuffer(), 0);
+    CPPUNIT_ASSERT_EQUAL(size_t(8192), buffer.getCachedSize(HEADER));
+
+    BufferAllocation singleBodyBuffer(allocateBuffer(300)); // -> 4096
+    buffer.cacheLocation(BODY, DataLocation(0, 100),
+                         singleBodyBuffer.getSharedBuffer(), 0);
+    CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(BODY));
+
+    buffer.clear(HEADER);
+    CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(HEADER));
+
+    buffer.clear(BODY);
+    CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(BODY));
+}
+
+void
+SimpleMemFileIOBufferTest::testSharedBuffer()
+{
+    typedef SimpleMemFileIOBuffer::SharedBuffer SharedBuffer;
+
+    {
+        SharedBuffer buf(1024);
+        CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getSize());
+        CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getFreeSize());
+        CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getUsedSize());
+        CPPUNIT_ASSERT(buf.hasRoomFor(1024));
+        CPPUNIT_ASSERT(!buf.hasRoomFor(1025));
+
+        CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(13));
+        // Allocation should be rounded up to nearest alignment.
+        // TODO: is this even necessary?
+        CPPUNIT_ASSERT_EQUAL(size_t(16), buf.getUsedSize());
+        CPPUNIT_ASSERT_EQUAL(size_t(1008), buf.getFreeSize());
+        CPPUNIT_ASSERT(buf.hasRoomFor(1008));
+        CPPUNIT_ASSERT(!buf.hasRoomFor(1009));
+        CPPUNIT_ASSERT_EQUAL(size_t(16), buf.allocate(1));
+        CPPUNIT_ASSERT_EQUAL(size_t(24), buf.getUsedSize());
+
+        CPPUNIT_ASSERT_EQUAL(size_t(24), buf.allocate(999));
+        CPPUNIT_ASSERT(!buf.hasRoomFor(1));
+        CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize());
+        CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize());
+    }
+    // Test exact fit.
+    {
+        SharedBuffer buf(1024);
+        CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(1024));
+        CPPUNIT_ASSERT(!buf.hasRoomFor(1));
+        CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize());
+        CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize());
+    }
+    // Test 512-byte alignment.
+    {
+        SharedBuffer buf(1024);
+        CPPUNIT_ASSERT(buf.hasRoomFor(1000, SharedBuffer::ALIGN_512_BYTES));
+        CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(10));
+        CPPUNIT_ASSERT(!buf.hasRoomFor(1000, SharedBuffer::ALIGN_512_BYTES));
+        CPPUNIT_ASSERT(!buf.hasRoomFor(513, SharedBuffer::ALIGN_512_BYTES));
+        CPPUNIT_ASSERT(buf.hasRoomFor(512, SharedBuffer::ALIGN_512_BYTES));
+        CPPUNIT_ASSERT_EQUAL(size_t(512), buf.allocate(512, SharedBuffer::ALIGN_512_BYTES));
+        CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize());
+        CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize());
+    }
+}
+
+void
+SimpleMemFileIOBufferTest::testSharedBufferUsage()
+{
+    FileSpecification fileSpec(BucketId(16, 123),
+                               env().getDirectory(), "testfile.0");
+    SimpleMemFileIOBuffer ioBuf(dfr,
+                                vespalib::LazyFile::UP(),
+                                std::unique_ptr<FileInfo>(new FileInfo),
+                                fileSpec,
+                                env());
+
+    const size_t threshold = SimpleMemFileIOBuffer::WORKING_BUFFER_SIZE;
+
+    // Brand new allocation
+    BufferAllocation ba(ioBuf.allocateBuffer(HEADER, 1));
+    CPPUNIT_ASSERT(ba.buf.get());
+    CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba.pos);
+    CPPUNIT_ASSERT_EQUAL(uint32_t(1), ba.size);
+    // Should reuse buffer, but get other offset
+    BufferAllocation ba2(ioBuf.allocateBuffer(HEADER, 500));
+    CPPUNIT_ASSERT_EQUAL(ba.buf.get(), ba2.buf.get());
+    CPPUNIT_ASSERT_EQUAL(uint32_t(8), ba2.pos);
+    CPPUNIT_ASSERT_EQUAL(uint32_t(500), ba2.size);
+    CPPUNIT_ASSERT_EQUAL(size_t(512), ba2.buf->getUsedSize());
+
+    // Allocate a buffer so big that it should get its own buffer instance
+    BufferAllocation ba3(ioBuf.allocateBuffer(HEADER, threshold));
+    CPPUNIT_ASSERT(ba3.buf.get() != ba2.buf.get());
+    CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba3.pos);
+    CPPUNIT_ASSERT_EQUAL(uint32_t(threshold), ba3.size);
+
+    // But smaller allocs should still be done from working buffer
+    BufferAllocation ba4(ioBuf.allocateBuffer(HEADER, 512));
+    CPPUNIT_ASSERT_EQUAL(ba.buf.get(), ba4.buf.get());
+    CPPUNIT_ASSERT_EQUAL(uint32_t(512), ba4.pos);
+    CPPUNIT_ASSERT_EQUAL(uint32_t(512), ba4.size);
+    CPPUNIT_ASSERT_EQUAL(size_t(1024), ba4.buf->getUsedSize());
+
+    // Allocate lots of smaller buffers from the same buffer until we run out.
+    while (true) {
+        BufferAllocation tmp(ioBuf.allocateBuffer(HEADER, 1024));
+        CPPUNIT_ASSERT_EQUAL(ba.buf.get(), tmp.buf.get());
+        if (!tmp.buf->hasRoomFor(2048)) {
+            break;
+        }
+    }
+    BufferAllocation ba5(ioBuf.allocateBuffer(HEADER, 2048));
+    CPPUNIT_ASSERT(ba5.buf.get() != ba.buf.get());
+    CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba5.pos);
+    CPPUNIT_ASSERT_EQUAL(uint32_t(2048), ba5.size);
+
+    // Allocating for different part should get different buffer.
+    BufferAllocation ba6(ioBuf.allocateBuffer(BODY, 128));
+    CPPUNIT_ASSERT(ba6.buf.get() != ba5.buf.get());
+    CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba6.pos);
+    CPPUNIT_ASSERT_EQUAL(uint32_t(128), ba6.size);
+}
+
+void
+SimpleMemFileIOBufferTest::testHeaderChunkEncoderComputesSizesCorrectly()
+{
+    document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100));
+
+    std::string idString = doc->getId().toString();
+    HeaderChunkEncoder encoder(doc->getId());
+    // Without document, payload is: 3x u32 + doc id string (no zero term).
+    CPPUNIT_ASSERT_EQUAL(sizeof(uint32_t)*3 + idString.size(),
+                         static_cast<size_t>(encoder.encodedSize()));
+
+    encoder.bufferDocument(*doc);
+    vespalib::nbostream stream;
+    doc->serializeHeader(stream);
+    // With document, add size of serialized document to the mix.
+    CPPUNIT_ASSERT_EQUAL(sizeof(uint32_t)*3 + idString.size() + stream.size(),
+                         static_cast<size_t>(encoder.encodedSize()));
+}
+
+SimpleMemFileIOBufferTest::SimpleMemFileIOBufferUP
+SimpleMemFileIOBufferTest::createIoBufferWithDummySpec(
+        vespalib::stringref removeDocType)
+{
+    FileSpecification fileSpec(BucketId(16, 123),
+                               env().getDirectory(), "testfile.0");
+    // Override config.
+    auto options = env().acquireConfigReadLock().options();
+    env().acquireConfigWriteLock().setOptions(
+            OptionsBuilder(*options)
+                .defaultRemoveDocType(removeDocType)
+                .build());
+
+    SimpleMemFileIOBufferUP ioBuf(
+            new SimpleMemFileIOBuffer(
+                dfr,
+                vespalib::LazyFile::UP(),
+                std::unique_ptr<FileInfo>(new FileInfo),
+                fileSpec,
+                env()));
+    return ioBuf;
+}
+
+void
+SimpleMemFileIOBufferTest::testHeaderChunkEncoderSerializesIdCorrectly()
+{
+    document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100));
+    HeaderChunkEncoder encoder(doc->getId());
+
+    SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec());
+
+    BufferAllocation buf(ioBuf->allocateBuffer(HEADER, encoder.encodedSize()));
+    encoder.writeTo(buf);
+    DataLocation newLoc = ioBuf->addLocation(HEADER, buf);
+    document::DocumentId checkId = ioBuf->getDocumentId(newLoc);
+
+    CPPUNIT_ASSERT_EQUAL(doc->getId(), checkId);
+}
+
+void
+SimpleMemFileIOBufferTest::testHeaderChunkEncoderSerializesHeaderCorrectly()
+{
+    document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100));
+    HeaderChunkEncoder encoder(doc->getId());
+    encoder.bufferDocument(*doc);
+
+    SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec());
+    BufferAllocation buf(ioBuf->allocateBuffer(HEADER, encoder.encodedSize()));
+    encoder.writeTo(buf);
+    DataLocation newLoc = ioBuf->addLocation(HEADER, buf);
+    Document::UP checkDoc = ioBuf->getDocumentHeader(*getTypeRepo(), newLoc);
+
+    CPPUNIT_ASSERT_EQUAL(doc->getId(), checkDoc->getId());
+    CPPUNIT_ASSERT_EQUAL(doc->getType(), checkDoc->getType());
+}
+
+void
+SimpleMemFileIOBufferTest::testRemovesCanBeWrittenWithBlankDefaultDocument()
+{
+    SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1"));
+
+    document::DocumentId id("userdoc:yarn:12345:fluff");
+    DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo()));
+    // Despite adding with document id only, we should now actually have a
+    // valid document header. Will fail with a DeserializeException if no
+    // header has been written.
+    Document::UP removeWithHeader(
+            ioBuf->getDocumentHeader(*getTypeRepo(), loc));
+    CPPUNIT_ASSERT_EQUAL(removeWithHeader->getId(), id);
+    CPPUNIT_ASSERT_EQUAL(removeWithHeader->getType(),
+                         *getTypeRepo()->getDocumentType("testdoctype1"));
+}
+
+void
+SimpleMemFileIOBufferTest::testRemovesCanBeWrittenWithIdInferredDoctype()
+{
+    SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1"));
+
+    document::DocumentId id("id:yarn:testdoctype2:n=12345:fluff");
+    DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo()));
+    // Since document id contains an explicit document type, the blank remove
+    // document header should be written with that type instead of the one
+    // provided as default via config.
+    Document::UP removeWithHeader(
+            ioBuf->getDocumentHeader(*getTypeRepo(), loc));
+    CPPUNIT_ASSERT_EQUAL(removeWithHeader->getId(), id);
+    CPPUNIT_ASSERT_EQUAL(removeWithHeader->getType(),
+                         *getTypeRepo()->getDocumentType("testdoctype2"));
+}
+
+void
+SimpleMemFileIOBufferTest::testRemovesWithInvalidDocTypeThrowsException()
+{
+    SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1"));
+
+    document::DocumentId id("id:yarn:nosuchtype:n=12345:fluff");
+    try {
+        DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo()));
+        CPPUNIT_FAIL("No exception thrown on bad doctype");
+    } catch (const vespalib::Exception& e) {
+        CPPUNIT_ASSERT(e.getMessage().find("Could not serialize document "
+                                           "for remove with unknown doctype "
+                                           "'nosuchtype'")
+                       != std::string::npos);
+    }
+}
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/spi/simulatedfailurefile.h b/memfilepersistence/src/tests/spi/simulatedfailurefile.h
new file mode 100644
index 00000000000..1ded927a3d1
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/simulatedfailurefile.h
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <tests/spi/memfiletestutils.h>
+#include <tests/spi/logginglazyfile.h>
+
+namespace storage {
+namespace memfile {
+
+class SimulatedFailureLazyFile : public vespalib::LazyFile
+{
+    mutable int _readOpsBeforeFailure;
+    mutable int _writeOpsBeforeFailure;
+public:
+    class Factory : public Environment::LazyFileFactory {
+    public:
+        Factory()
+            : _readOpsBeforeFailure(-1),
+              _writeOpsBeforeFailure(0)
+        {
+        }
+        vespalib::LazyFile::UP createFile(const std::string& fileName) const {
+            return vespalib::LazyFile::UP(
+                    new SimulatedFailureLazyFile(fileName,
+                            vespalib::File::DIRECTIO,
+                            _readOpsBeforeFailure,
+                            _writeOpsBeforeFailure));
+        }
+
+        void setReadOpsBeforeFailure(int ops) {
+            _readOpsBeforeFailure = ops;
+        }
+
+        void setWriteOpsBeforeFailure(int ops) {
+            _writeOpsBeforeFailure = ops;
+        }
+    private:
+        int _readOpsBeforeFailure;
+        int _writeOpsBeforeFailure;
+    };
+
+    SimulatedFailureLazyFile(
+            const std::string& filename,
+            int flags,
+            int readOpsBeforeFailure,
+            int writeOpsBeforeFailure)
+        : LazyFile(filename, flags),
+          _readOpsBeforeFailure(readOpsBeforeFailure),
+          _writeOpsBeforeFailure(writeOpsBeforeFailure)
+    {
+    }
+
+    off_t write(const void *buf, size_t bufsize, off_t offset)
+    {
+        if (_writeOpsBeforeFailure == 0) {
+            throw vespalib::IoException(
+                    "A simulated I/O write exception was triggered",
+                    vespalib::IoException::CORRUPT_DATA, VESPA_STRLOC);
+        }
+        --_writeOpsBeforeFailure;
+        return vespalib::LazyFile::write(buf, bufsize, offset);
+    }
+
+    size_t read(void *buf, size_t bufsize, off_t offset) const
+    {
+        if (_readOpsBeforeFailure == 0) {
+            throw vespalib::IoException(
+                    "A simulated I/O read exception was triggered",
+                    vespalib::IoException::CORRUPT_DATA, VESPA_STRLOC);
+        }
+        --_readOpsBeforeFailure;
+        return vespalib::LazyFile::read(buf, bufsize, offset);
+    }
+};
+
+} // ns memfile
+} // ns storage
+
diff --git a/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp b/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp
new file mode 100644
index 00000000000..75eab5c2972
--- /dev/null
+++ b/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp
@@ -0,0 +1,213 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/datatype/documenttype.h>
+#include <tests/spi/memfiletestutils.h>
+#include <tests/spi/simulatedfailurefile.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+using document::DocumentType;
+
+namespace storage {
+namespace memfile {
+namespace {
+    spi::LoadType defaultLoadType(0, "default");
+}
+
+class SplitOperationHandlerTest : public SingleDiskMemFileTestUtils
+{
+
+    void doTestMultiDisk(uint16_t sourceDisk,
+                         uint16_t targetDisk0,
+                         uint16_t targetDisk1);
+
+
+    CPPUNIT_TEST_SUITE(SplitOperationHandlerTest);
+    CPPUNIT_TEST(testSimple);
+    CPPUNIT_TEST(testMultiDisk);
+    CPPUNIT_TEST(testMultiDiskNonZeroSourceIndex);
+    CPPUNIT_TEST(testExceptionDuringSplittingEvictsAllBuckets);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testSimple();
+    void testMultiDisk();
+    void testMultiDiskNonZeroSourceIndex();
+    void testExceptionDuringSplittingEvictsAllBuckets();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(SplitOperationHandlerTest);
+
+void
+SplitOperationHandlerTest::testSimple()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    setupDisks(1);
+
+    for (uint32_t i = 0; i < 100; i++) {
+        uint32_t location = 4;
+        if (i % 2 == 0) {
+            location |= (1 << 16);
+        }
+
+        doPut(location, Timestamp(1000 + i));
+    }
+    flush(document::BucketId(16, 4));
+
+    env()._cache.clear();
+
+    document::BucketId sourceBucket = document::BucketId(16, 4);
+    document::BucketId target1 = document::BucketId(17, 4);
+    document::BucketId target2 = document::BucketId(17, 4 | (1 << 16));
+
+    SplitOperationHandler handler(env());
+    spi::Result result = getPersistenceProvider().split(
+            spi::Bucket(sourceBucket, spi::PartitionId(0)),
+            spi::Bucket(target1, spi::PartitionId(0)),
+            spi::Bucket(target2, spi::PartitionId(0)),
+            context);
+
+    env()._cache.clear();
+
+    {
+        MemFilePtr file(handler.getMemFile(sourceBucket, 0));
+        CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount());
+    }
+
+    {
+        MemFilePtr file(handler.getMemFile(target1, 0));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+        for (uint32_t i = 0; i < file->getSlotCount(); ++i) {
+            file->getDocument((*file)[i], ALL);
+        }
+    }
+
+    {
+        MemFilePtr file(handler.getMemFile(target2, 0));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+        for (uint32_t i = 0; i < file->getSlotCount(); ++i) {
+            file->getDocument((*file)[i], ALL);
+        }
+    }
+}
+
+void
+SplitOperationHandlerTest::doTestMultiDisk(uint16_t sourceDisk,
+                                           uint16_t targetDisk0,
+                                           uint16_t targetDisk1)
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    setupDisks(3);
+
+    for (uint32_t i = 0; i < 100; i++) {
+        uint32_t location = 4;
+        if (i % 2 == 0) {
+            location |= (1 << 16);
+        }
+
+        doPutOnDisk(sourceDisk, location, Timestamp(1000 + i));
+    }
+    flush(document::BucketId(16, 4));
+
+    env()._cache.clear();
+
+    document::BucketId sourceBucket = document::BucketId(16, 4);
+    document::BucketId target1 = document::BucketId(17, 4);
+    document::BucketId target2 = document::BucketId(17, 4 | (1 << 16));
+
+    SplitOperationHandler handler(env());
+    spi::Result result = getPersistenceProvider().split(
+            spi::Bucket(sourceBucket, spi::PartitionId(sourceDisk)),
+            spi::Bucket(target1, spi::PartitionId(targetDisk0)),
+            spi::Bucket(target2, spi::PartitionId(targetDisk1)),
+            context);
+
+    env()._cache.clear();
+
+    {
+        MemFilePtr file(handler.getMemFile(sourceBucket, sourceDisk));
+        CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount());
+    }
+
+    {
+        MemFilePtr file(handler.getMemFile(target1, targetDisk0));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+        for (uint32_t i = 0; i < file->getSlotCount(); ++i) {
+            file->getDocument((*file)[i], ALL);
+        }
+    }
+
+    {
+        MemFilePtr file(handler.getMemFile(target2, targetDisk1));
+        CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount());
+        for (uint32_t i = 0; i < file->getSlotCount(); ++i) {
+            file->getDocument((*file)[i], ALL);
+        }
+    }
+}
+
+void
+SplitOperationHandlerTest::testMultiDisk()
+{
+    doTestMultiDisk(0, 1, 2);
+}
+
+void
+SplitOperationHandlerTest::testMultiDiskNonZeroSourceIndex()
+{
+    doTestMultiDisk(1, 2, 0);
+}
+
+void
+SplitOperationHandlerTest::testExceptionDuringSplittingEvictsAllBuckets()
+{
+    spi::Context context(defaultLoadType, spi::Priority(0),
+                         spi::Trace::TraceLevel(0));
+    setupDisks(1);
+
+    for (uint32_t i = 0; i < 100; i++) {
+        uint32_t location = 4;
+        if (i % 2 == 0) {
+            location |= (1 << 16);
+        }
+
+        doPut(location, Timestamp(1000 + i));
+    }
+    flush(document::BucketId(16, 4));
+
+    simulateIoErrorsForSubsequentlyOpenedFiles();
+
+    document::BucketId sourceBucket(16, 4);
+    document::BucketId target1(17, 4);
+    document::BucketId target2(17, 4 | (1 << 16));
+
+    try {
+        SplitOperationHandler handler(env());
+        spi::Result result = getPersistenceProvider().split(
+                spi::Bucket(sourceBucket, spi::PartitionId(0)),
+                spi::Bucket(target1, spi::PartitionId(0)),
+                spi::Bucket(target2, spi::PartitionId(0)),
+                context);
+        CPPUNIT_FAIL("Exception not thrown on flush failure");
+    } catch (std::exception&) {
+    }
+
+    CPPUNIT_ASSERT(!env()._cache.contains(sourceBucket));
+    CPPUNIT_ASSERT(!env()._cache.contains(target1));
+    CPPUNIT_ASSERT(!env()._cache.contains(target2));
+
+    unSimulateIoErrorsForSubsequentlyOpenedFiles();
+
+    // Source must not have been deleted
+    {
+        SplitOperationHandler handler(env());
+        MemFilePtr file(handler.getMemFile(sourceBucket, 0));
+        CPPUNIT_ASSERT_EQUAL(100, (int)file->getSlotCount());
+    }
+}
+
+}
+
+}
diff --git a/memfilepersistence/src/tests/testhelper.cpp b/memfilepersistence/src/tests/testhelper.cpp
new file mode 100644
index 00000000000..40a3512e400
--- /dev/null
+++ b/memfilepersistence/src/tests/testhelper.cpp
@@ -0,0 +1,124 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <tests/testhelper.h>
+
+#include <vespa/log/log.h>
+#include <vespa/vespalib/io/fileutil.h>
+
+LOG_SETUP(".testhelper");
+
+namespace storage {
+
+void addStorageDistributionConfig(vdstestlib::DirConfig& dc)
+{
+    vdstestlib::DirConfig::Config* config;
+    config = &dc.getConfig("stor-distribution", true);
+    config->clear();
+    config->set("group[1]");
+    config->set("group[0].name", "foo");
+    config->set("group[0].index", "0");
+    config->set("group[0].nodes[50]");
+
+    for (uint32_t i = 0; i < 50; i++) {
+        std::ostringstream key; key << "group[0].nodes[" << i << "].index";
+        std::ostringstream val; val << i;
+        config->set(key.str(), val.str());
+    }
+}
+
+vdstestlib::DirConfig getStandardConfig(bool storagenode) {
+    vdstestlib::DirConfig dc;
+    vdstestlib::DirConfig::Config* config;
+    config = &dc.addConfig("stor-cluster");
+    config = &dc.addConfig("load-type");
+    config = &dc.addConfig("bucket");
+    config = &dc.addConfig("messagebus");
+    config = &dc.addConfig("stor-prioritymapping");
+    config = &dc.addConfig("stor-bucketdbupdater");
+    config = &dc.addConfig("metricsmanager");
+    config->set("consumer[1]");
+    config->set("consumer[0].name", "\"status\"");
+    config->set("consumer[0].addedmetrics[1]");
+    config->set("consumer[0].addedmetrics[0]", "\"*\"");
+    config = &dc.addConfig("stor-communicationmanager");
+    config->set("rpcport", "0");
+    config->set("mbusport", "0");
+    config = &dc.addConfig("stor-bucketdb");
+    config->set("chunklevel", "0");
+    config = &dc.addConfig("stor-distributormanager");
+    config = &dc.addConfig("stor-opslogger");
+    config = &dc.addConfig("stor-memfilepersistence");
+    // Easier to see what goes wrong with only 1 thread per disk.
+    config->set("minimum_file_meta_slots", "2");
+    config->set("minimum_file_header_block_size", "368");
+    config->set("minimum_file_size", "4096");
+    config->set("threads[1]");
+    config->set("threads[0].lowestpri 255");
+    config->set("dir_spread", "4");
+    config->set("dir_levels", "0");
+    // Unit tests typically use fake low time values, so don't complain
+    // about them or compact/delete them by default. Override in tests testing that
+    // behavior
+    config = &dc.addConfig("persistence");
+    config->set("keep_remove_time_period", "2000000000");
+    config->set("revert_time_period", "2000000000");
+    config = &dc.addConfig("stor-bouncer");
+    config = &dc.addConfig("stor-integritychecker");
+    config = &dc.addConfig("stor-bucketmover");
+    config = &dc.addConfig("stor-messageforwarder");
+    config = &dc.addConfig("stor-server");
+    config->set("enable_dead_lock_detector", "false");
+    config->set("enable_dead_lock_detector_warnings", "false");
+    config->set("max_merges_per_node", "25");
+    config->set("max_merge_queue_size", "20");
+    config->set("root_folder",
+                    (storagenode ? "vdsroot" : "vdsroot.distributor"));
+    config->set("is_distributor",
+                    (storagenode ? "false" : "true"));
+    config = &dc.addConfig("stor-devices");
+    config->set("root_folder",
+                    (storagenode ? "vdsroot" : "vdsroot.distributor"));
+    config = &dc.addConfig("stor-status");
+    config->set("httpport", "0");
+    config = &dc.addConfig("stor-visitor");
+    config->set("defaultdocblocksize", "8192");
+    // By default, need "old" behaviour of maxconcurrent
+    config->set("maxconcurrentvisitors_fixed", "4");
+    config->set("maxconcurrentvisitors_variable", "0");
+    config = &dc.addConfig("stor-visitordispatcher");
+    addFileConfig(dc, "documenttypes", "config-doctypes.cfg");
+    addStorageDistributionConfig(dc);
+    return dc;
+}
+
+void addFileConfig(vdstestlib::DirConfig& dc,
+                       const std::string& configDefName,
+                       const std::string& fileName)
+{
+    vdstestlib::DirConfig::Config* config;
+    config = &dc.getConfig(configDefName, true);
+    config->clear();
+    std::ifstream in(fileName.c_str());
+    std::string line;
+    while (std::getline(in, line, '\n')) {
+        std::string::size_type pos = line.find(' ');
+        if (pos == std::string::npos) {
+            config->set(line);
+        } else {
+            config->set(line.substr(0, pos), line.substr(pos + 1));
+        }
+    }
+    in.close();
+}
+
+TestName::TestName(const std::string& n)
+    : name(n)
+{
+    LOG(debug, "Starting test %s", name.c_str());
+}
+
+TestName::~TestName() {
+    LOG(debug, "Done with test %s", name.c_str());
+}
+
+} // storage
diff --git a/memfilepersistence/src/tests/testhelper.h b/memfilepersistence/src/tests/testhelper.h
new file mode 100644
index 00000000000..4445086d300
--- /dev/null
+++ b/memfilepersistence/src/tests/testhelper.h
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+#include <vespa/vdstestlib/cppunit/dirconfig.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+
+#include <fstream>
+#include <vespa/fastos/fastos.h>
+#include <sstream>
+
+#define ASSERT_REPLY_COUNT(count, dummylink) \
+    { \
+        std::ostringstream msgost; \
+        if ((dummylink).getNumReplies() != count) { \
+            for (uint32_t ijx=0; ijx<(dummylink).getNumReplies(); ++ijx) { \
+                msgost << (dummylink).getReply(ijx)->toString(true) << "\n"; \
+            } \
+        } \
+        CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \
+                                 (dummylink).getNumReplies()); \
+    }
+#define ASSERT_COMMAND_COUNT(count, dummylink) \
+    { \
+        std::ostringstream msgost; \
+        if ((dummylink).getNumCommands() != count) { \
+            for (uint32_t ijx=0; ijx<(dummylink).getNumCommands(); ++ijx) { \
+                msgost << (dummylink).getCommand(ijx)->toString(true) << "\n"; \
+            } \
+        } \
+        CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \
+                                 (dummylink).getNumCommands()); \
+    }
+
+namespace storage {
+
+void addFileConfig(vdstestlib::DirConfig& dc,
+                   const std::string& configDefName,
+                   const std::string& fileName);
+
+
+void addStorageDistributionConfig(vdstestlib::DirConfig& dc);
+
+vdstestlib::DirConfig getStandardConfig(bool storagenode);
+
+// Class used to print start and end of test. Enable debug when you want to see
+// which test creates what output or where we get stuck
+struct TestName {
+    std::string name;
+    TestName(const std::string& n);
+    ~TestName();
+};
+
+} // storage
+
diff --git a/memfilepersistence/src/tests/testrunner.cpp b/memfilepersistence/src/tests/testrunner.cpp
new file mode 100644
index 00000000000..16027870c47
--- /dev/null
+++ b/memfilepersistence/src/tests/testrunner.cpp
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <iostream>
+#include <vespa/log/log.h>
+#include <vespa/vdstestlib/cppunit/cppunittestrunner.h>
+
+LOG_SETUP("persistencecppunittests");
+
+int
+main(int argc, char **argv)
+{
+    vdstestlib::CppUnitTestRunner testRunner;
+    return testRunner.run(argc, argv);
+}
diff --git a/memfilepersistence/src/tests/tools/.gitignore b/memfilepersistence/src/tests/tools/.gitignore
new file mode 100644
index 00000000000..7e7c0fe7fae
--- /dev/null
+++ b/memfilepersistence/src/tests/tools/.gitignore
@@ -0,0 +1,2 @@
+/.depend
+/Makefile
diff --git a/memfilepersistence/src/tests/tools/CMakeLists.txt b/memfilepersistence/src/tests/tools/CMakeLists.txt
new file mode 100644
index 00000000000..aef718c7633
--- /dev/null
+++ b/memfilepersistence/src/tests/tools/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(memfilepersistence_testtools
+    SOURCES
+    dumpslotfiletest.cpp
+    vdsdisktooltest.cpp
+    DEPENDS
+)
diff --git a/memfilepersistence/src/tests/tools/dumpslotfiletest.cpp b/memfilepersistence/src/tests/tools/dumpslotfiletest.cpp
new file mode 100644
index 00000000000..112f8840e72
--- /dev/null
+++ b/memfilepersistence/src/tests/tools/dumpslotfiletest.cpp
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/config/subscription/configuri.h>
+#include <vespa/document/base/testdocrepo.h>
+#include <vespa/memfilepersistence/tools/dumpslotfile.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/util/programoptions_testutils.h>
+#include <tests/spi/memfiletestutils.h>
+
+#include <vespa/document/config/config-documenttypes.h>
+
+namespace storage {
+namespace memfile {
+
+class DumpSlotFileTest : public SingleDiskMemFileTestUtils
+{
+    CPPUNIT_TEST_SUITE(DumpSlotFileTest);
+    CPPUNIT_TEST(testSimple);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testSimple();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DumpSlotFileTest);
+
+#define ASSERT_MATCH(optstring, pattern) \
+{ \
+    vespalib::AppOptions opts("dumpslotfile " optstring); \
+    std::ostringstream out; \
+    config::ConfigUri configUri(config::ConfigUri::createFromInstance( \
+            document::TestDocRepo::getDefaultConfig())); \
+    std::unique_ptr<document::DocumenttypesConfig> config = config::ConfigGetter<document::DocumenttypesConfig>::getConfig(configUri.getConfigId(), configUri.getContext()); \
+    SlotFileDumper::dump(opts.getArgCount(), opts.getArguments(), \
+                         configUri, out, out); \
+    CPPUNIT_ASSERT_MATCH_REGEX(pattern, out.str()); \
+    output = out.str(); \
+}
+
+void
+DumpSlotFileTest::testSimple()
+{
+    std::string output;
+        // Test syntax page
+    ASSERT_MATCH("--help", ".*Usage: dumpslotfile.*");
+        // Test non-existing file. (Handle as empty file)
+    ASSERT_MATCH("00a.0",
+                 ".*BucketId\\(0x000000000000000a\\)"
+                 ".*document count: 0.*non-existing.*");
+        // Parse bucketid without extension.
+    ASSERT_MATCH("000000000000000a",
+                 ".*BucketId\\(0x000000000000000a\\) "
+                 "\\(extracted from filename\\).*");
+        // Parse invalid bucket id.
+    ASSERT_MATCH("000010000000000g",
+                 ".*Failed to extract bucket id from filename.*");
+        // Test toXml with no data. Thus doesn't require doc config
+    ASSERT_MATCH("--toxml --documentconfig whatevah 000a.0",
+                 ".*<vespafeed>.*");
+        // Test invalid arguments
+    ASSERT_MATCH("--foobar", ".*Invalid option 'foobar'\\..*");
+        // What to show in XML doesn't make sense in non-xml mode
+    ASSERT_MATCH("--includeremoveddocs 0.0",
+                 ".*Options for what to include in XML makes no sense when not "
+                 "printing XML content.*");
+    ASSERT_MATCH("--includeremoveentries 0.0",
+                 ".*Options for what to include in XML makes no sense when not "
+                 "printing XML content.*");
+        // To binary only works for single doc
+    ASSERT_MATCH("--tobinary 0.0",
+                 ".*To binary option only works for a single document.*");
+
+    BucketId bid(1, 0);
+    createTestBucket(bid, 0);
+    ASSERT_MATCH("-nN vdsroot/disks/d0/400000000000000.0",
+                 ".*"
+                 "Unique document count: 8.*"
+                 "Total document size: [0-9]+.*"
+                 "Used size: [0-9]+.*"
+                 "Filename: .*/d0/.*"
+                 "Filesize: 12288.*"
+                 "SlotFileHeader.*"
+                 "[0-9]+ empty entries.*"
+                 "Header block.*"
+                 "Content block.*"
+                 "Slotfile verified.*"
+    );
+    ASSERT_MATCH("vdsroot/disks/d0/400000000000000.0", ".*ff ff ff ff.*");
+
+        // User friendly output
+    ASSERT_MATCH("--friendly -nN vdsroot/disks/d0/400000000000000.0",
+                 ".*id:mail:testdoctype1:n=0:9380.html.*");
+
+    ASSERT_MATCH("--tobinary "
+                 "--docid id:mail:testdoctype1:n=0:doesnotexisthere.html "
+                 "vdsroot/disks/d0/400000000000000.0",
+                 ".*No document with id id:mail:testdoctype1:n=0:doesnotexi.* "
+                 "found.*");
+
+    // Should test XML with content.. But needs document config for it to work.
+    // Should be able to create programmatically from testdocman.
+    ASSERT_MATCH("--toxml --documentconfig '' "
+                 "vdsroot/disks/d0/400000000000000.0",
+                 ".*<vespafeed>\n"
+                 "<document documenttype=\"testdoctype1\" "
+                    "documentid=\"id:mail:testdoctype1:n=0:9639.html\">\n"
+                 "<content>overwritten</content>\n"
+                 "</document>.*");
+
+        // To binary
+    ASSERT_MATCH("--tobinary --docid id:mail:testdoctype1:n=0:9380.html "
+                 "vdsroot/disks/d0/400000000000000.0",
+                 ".*");
+    {
+        TestDocMan docMan;
+        document::ByteBuffer buf(output.c_str(), output.size());
+        document::Document doc(docMan.getTypeRepo(), buf);
+        CPPUNIT_ASSERT_EQUAL(std::string(
+           "<document documenttype=\"testdoctype1\" "
+           "documentid=\"id:mail:testdoctype1:n=0:9380.html\">\n"
+           "<content>To be, or not to be: that is the question:\n"
+           "Whether 'tis nobler in the mind to suffer\n"
+           "The slings and arrows of outrage</content>\n"
+           "</document>"), doc.toXml());
+    }
+
+        // Fail verification
+    {
+        vespalib::LazyFile file("vdsroot/disks/d0/400000000000000.0", 0);
+        file.write("corrupt", 7, 64);
+    }
+    ASSERT_MATCH("-nN vdsroot/disks/d0/400000000000000.0",
+                 ".*lot 0 at timestamp [0-9]+ failed checksum verification.*");
+}
+
+} // memfile
+} // storage
diff --git a/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp b/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp
new file mode 100644
index 00000000000..29e780bc900
--- /dev/null
+++ b/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/config/subscription/configuri.h>
+#include <vespa/memfilepersistence/tools/vdsdisktool.h>
+#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/util/programoptions_testutils.h>
+#include <tests/spi/memfiletestutils.h>
+
+namespace storage {
+namespace memfile {
+
+struct VdsDiskToolTest : public SingleDiskMemFileTestUtils
+{
+    framework::defaultimplementation::FakeClock _clock;
+    DeviceManager::LP _deviceManager;
+
+    void setUp();
+    void setupRoot();
+
+    void testSimple();
+
+    CPPUNIT_TEST_SUITE(VdsDiskToolTest);
+    CPPUNIT_TEST(testSimple);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(VdsDiskToolTest);
+
+#define ASSERT_MATCH(optstring, pattern, exitcode) \
+{ \
+    std::ostringstream out; \
+    int result = 1; \
+    try{ \
+        vespalib::AppOptions opts("vdsdisktool " optstring); \
+        result = VdsDiskTool::run(opts.getArgCount(), opts.getArguments(), \
+                                  "vdsroot", out, out); \
+    } catch (std::exception& e) { \
+        out << "Application aborted with exception:\n" << e.what() << "\n"; \
+    } \
+    CPPUNIT_ASSERT_MATCH_REGEX(pattern, out.str()); \
+    CPPUNIT_ASSERT_EQUAL(exitcode, result); \
+}
+
+namespace {
+    void createDisk(int i) {
+        std::ostringstream path;
+        path << "vdsroot/mycluster/storage/3/disks/d" << i;
+        CPPUNIT_ASSERT_EQUAL(0, system(("mkdir -p " + path.str()).c_str()));
+    }
+}
+
+void
+VdsDiskToolTest::setUp()
+{
+    system("rm -rf vdsroot");
+    _deviceManager.reset(new DeviceManager(
+            DeviceMapper::UP(new SimpleDeviceMapper), _clock));
+}
+
+void
+VdsDiskToolTest::setupRoot()
+{
+    system("rm -rf vdsroot");
+    createDisk(0);
+}
+
+void
+VdsDiskToolTest::testSimple()
+{
+        // Test syntax page
+    ASSERT_MATCH("--help", ".*Usage: vdsdisktool .*", 0);
+        // No VDS installation
+    ASSERT_MATCH("status", ".*No VDS installations found at all.*", 1);
+        // Common setup
+    setupRoot();
+    ASSERT_MATCH("status", ".*Disks on storage node 3 in cluster mycluster:\\s*"
+                           "Disk 0: OK\\s*", 0);
+        // Two disks
+    system("mkdir -p vdsroot/mycluster/storage/3/disks/d1/");
+    ASSERT_MATCH("status", ".*Disks on storage node 3 in cluster mycluster:\\s*"
+                           "Disk 0: OK\\s*"
+                           "Disk 1: OK\\s*", 0);
+        // Two disks, non-continuous indexes
+    system("rm -rf vdsroot/mycluster/storage/3/disks/d1/");
+    system("mkdir -p vdsroot/mycluster/storage/3/disks/d2/");
+    ASSERT_MATCH("status", ".*Disks on storage node 3 in cluster mycluster:\\s*"
+                           "Disk 0: OK\\s*"
+                           "Disk 1: NOT_FOUND - Disk not found during scan.*"
+                           "Disk 2: OK\\s*", 0);
+        // Status file existing
+    setupRoot();
+    createDisk(1);
+    MountPointList mountPoints("vdsroot/mycluster/storage/3",
+                               std::vector<vespalib::string>(),
+                               _deviceManager);
+    mountPoints.scanForDisks();
+    CPPUNIT_ASSERT_EQUAL(2u, mountPoints.getSize());
+    mountPoints[1].addEvent(Device::IO_FAILURE, "Bad", "Found in test");
+    mountPoints.writeToFile();
+    ASSERT_MATCH("status", ".*Disks on storage node 3 in cluster mycluster:\\s*"
+                           "Disk 0: OK\\s*"
+                           "Disk 1: IO_FAILURE - 0 Bad\\s*", 0);
+}
+
+} // memfile
+} // storage