Add classes for mapping from gid to lid, with interface in searchlib.

Add document db referent registry, to handle implicit export of gid to lid
mapping and implicit export of all known attributes to other document types.

22 files changed, 617 insertions, 0 deletions

diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt
index 66db223d7ec..7546389aeb6 100644
--- a/searchcore/CMakeLists.txt
+++ b/searchcore/CMakeLists.txt
@@ -38,6 +38,7 @@ vespa_define_module(
     src/vespa/searchcore/proton/matching
     src/vespa/searchcore/proton/metrics
     src/vespa/searchcore/proton/persistenceengine
+    src/vespa/searchcore/proton/reference
     src/vespa/searchcore/proton/reprocessing
     src/vespa/searchcore/proton/server
     src/vespa/searchcore/proton/summaryengine
@@ -116,6 +117,8 @@ vespa_define_module(
     src/tests/proton/persistenceconformance
     src/tests/proton/persistenceengine
     src/tests/proton/proton
+    src/tests/proton/reference/gid_to_lid_mapper
+    src/tests/proton/reference/document_db_referent_registry
     src/tests/proton/reprocessing/attribute_reprocessing_initializer
     src/tests/proton/reprocessing/document_reprocessing_handler
     src/tests/proton/reprocessing/reprocessing_runner
diff --git a/searchcore/src/tests/proton/reference/document_db_referent_registry/CMakeLists.txt b/searchcore/src/tests/proton/reference/document_db_referent_registry/CMakeLists.txt
new file mode 100644
index 00000000000..d30602ce8e5
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/document_db_referent_registry/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_document_db_referent_registry_test_app TEST
+    SOURCES
+    document_db_referent_registry_test.cpp
+    DEPENDS
+    searchcore_reference
+    searchcore_documentmetastore
+    searchcore_bucketdb
+)
+vespa_add_test(NAME searchcore_document_db_referent_registry_test_app COMMAND searchcore_document_db_referent_registry_test_app)
diff --git a/searchcore/src/tests/proton/reference/document_db_referent_registry/DESC b/searchcore/src/tests/proton/reference/document_db_referent_registry/DESC
new file mode 100644
index 00000000000..03d7d0f167a
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/document_db_referent_registry/DESC
@@ -0,0 +1 @@
+document_db_referent_registry test. Take a look at document_db_referent_registry_test.cpp for details.
diff --git a/searchcore/src/tests/proton/reference/document_db_referent_registry/FILES b/searchcore/src/tests/proton/reference/document_db_referent_registry/FILES
new file mode 100644
index 00000000000..6ba8e0632f0
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/document_db_referent_registry/FILES
@@ -0,0 +1 @@
+document_db_referent_registry_test.cpp
diff --git a/searchcore/src/tests/proton/reference/document_db_referent_registry/document_db_referent_registry_test.cpp b/searchcore/src/tests/proton/reference/document_db_referent_registry/document_db_referent_registry_test.cpp
new file mode 100644
index 00000000000..dd2b14cca16
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/document_db_referent_registry/document_db_referent_registry_test.cpp
@@ -0,0 +1,101 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/searchcore/proton/reference/document_db_referent_registry.h>
+#include <vespa/searchcore/proton/reference/i_document_db_referent.h>
+#include <thread>
+#include <vespa/log/log.h>
+LOG_SETUP("document_db_reference_registry_test");
+
+namespace proton
+{
+
+namespace
+{
+
+
+std::mutex lock;
+std::shared_ptr<IDocumentDBReferent> getFooResult;
+
+void getFooTask(const IDocumentDBReferentRegistry *registry)
+{
+    auto result = registry->getDocumentDBReferent("foo");
+    std::lock_guard<std::mutex> guard(lock);
+    getFooResult = result;
+}
+
+std::shared_ptr<IDocumentDBReferent> checkFooResult()
+{
+    std::lock_guard<std::mutex> guard(lock);
+    return getFooResult;
+}
+
+}
+
+struct MyDocumentDBReferent : public IDocumentDBReferent
+{
+    MyDocumentDBReferent()
+    {
+    }
+    virtual ~MyDocumentDBReferent() { }
+    virtual std::shared_ptr<search::AttributeVector> getAttribute(vespalib::stringref name) override {
+        (void) name;
+        return std::shared_ptr<search::AttributeVector>();
+    }
+    virtual std::shared_ptr<search::IGidToLidMapperFactory> getGidToLidMapperFactory() override {
+        return std::shared_ptr<search::IGidToLidMapperFactory>();
+    }
+};
+
+struct Fixture
+{
+
+    DocumentDBReferentRegistry _registry;
+
+    Fixture()
+        : _registry()
+    {
+    }
+
+    std::shared_ptr<MyDocumentDBReferent>
+    add(vespalib::string name) {
+        auto referent = std::make_shared<MyDocumentDBReferent>();
+        _registry.addDocumentDBReferent(name, referent);
+        return referent;
+    }
+};
+
+TEST_F("Test that we can instantiate registry with two referents", Fixture)
+{
+    auto referentFoo = f.add("foo");
+    auto referentBar = f.add("bar");
+    EXPECT_NOT_EQUAL(referentFoo.get(), referentBar.get());
+    auto referent = f._registry.getDocumentDBReferent("foo");
+    EXPECT_EQUAL(referentFoo.get(), referent.get());
+    referent = f._registry.getDocumentDBReferent("bar");
+    EXPECT_EQUAL(referentBar.get(), referent.get());
+}
+
+TEST_F("Test that we block get until related add is completed", Fixture)
+{
+    const IDocumentDBReferentRegistry &registry = f._registry;
+    std::thread getFooThread(getFooTask, &registry);
+    sleep(1);
+    std::shared_ptr<IDocumentDBReferent> nullResult;
+    EXPECT_EQUAL(nullResult.get(), checkFooResult().get());
+    auto referentFoo = f.add("foo");
+    std::shared_ptr<IDocumentDBReferent> checkResult;
+    for (int retry = 0; retry < 60 && !checkResult; ++retry) {
+        sleep(1);
+        checkResult = checkFooResult();
+    }
+    EXPECT_EQUAL(referentFoo.get(), checkResult.get());
+    getFooThread.join();
+}
+
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/CMakeLists.txt b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/CMakeLists.txt
new file mode 100644
index 00000000000..259f44e101d
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_gid_to_lid_mapper_test_app TEST
+    SOURCES
+    gid_to_lid_mapper_test.cpp
+    DEPENDS
+    searchcore_reference
+    searchcore_documentmetastore
+    searchcore_bucketdb
+)
+vespa_add_test(NAME searchcore_gid_to_lid_mapper_test_app COMMAND searchcore_gid_to_lid_mapper_test_app)
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/DESC b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/DESC
new file mode 100644
index 00000000000..f251a0c3ac2
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/DESC
@@ -0,0 +1 @@
+gid_to_lid_mapper test. Take a look at gid_to_lid_mapper_test.cpp for details.
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/FILES b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/FILES
new file mode 100644
index 00000000000..13c38a92974
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/FILES
@@ -0,0 +1 @@
+gid_to_lid_mapper_test.cpp
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp
new file mode 100644
index 00000000000..3506e88cea5
--- /dev/null
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp
@@ -0,0 +1,144 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/searchcore/proton/reference/gid_to_lid_mapper.h>
+#include <vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.h>
+#include <vespa/log/log.h>
+LOG_SETUP("gid_to_lid_mapper_test");
+
+using document::GlobalId;
+using document::BucketId;
+using document::DocumentId;
+using storage::spi::Timestamp;
+using vespalib::GenerationHandler;
+
+namespace proton {
+
+namespace {
+
+GlobalId toGid(vespalib::stringref docId) {
+    return DocumentId(docId).getGlobalId();
+}
+
+vespalib::string doc1("id:test:music::1");
+vespalib::string doc2("id:test:music::2");
+vespalib::string doc3("id:test:music::3");
+
+static constexpr uint32_t numBucketBits = UINT32_C(20);
+
+BucketId toBucketId(const GlobalId &gid) {
+    BucketId bucketId(gid.convertToBucketId());
+    bucketId.setUsedBits(numBucketBits);
+    return bucketId;
+}
+
+void assertLid(const std::unique_ptr<search::IGidToLidMapper> &mapper, const vespalib::string &docId, uint32_t lid) {
+    EXPECT_EQUAL(lid, mapper->mapGidToLid(toGid(docId)));
+}
+
+}
+
+struct Fixture
+{
+
+    BucketDBOwner::SP _bucketDB;
+    std::shared_ptr<DocumentMetaStore> _dms;
+    Timestamp _timestamp;
+    using generation_t = GenerationHandler::generation_t;
+
+    Fixture()
+        : _bucketDB(std::make_shared<BucketDBOwner>()),
+          _dms(std::make_shared<DocumentMetaStore>(_bucketDB))
+    {
+        populate();
+    }
+
+    void bumpTimeStamp() {
+        _timestamp = Timestamp(_timestamp.getValue() + 1);
+    }
+
+    void put(vespalib::stringref docId, uint32_t lid) {
+        bumpTimeStamp();
+        const GlobalId gid(toGid(docId));
+        _dms->put(gid, toBucketId(gid), _timestamp, lid);
+        _dms->commit();
+    }
+
+    uint32_t put(vespalib::stringref docId) {
+        auto inspectRes = _dms->inspect(toGid(docId));
+        uint32_t lid = inspectRes.getLid();
+        put(docId, lid);
+        return lid;
+    }
+
+    void remove(uint32_t lid) {
+        _dms->remove(lid);
+        _dms->removeComplete(lid);
+        _dms->commit();
+    }
+
+
+    void populate() {
+        put(doc1, 4);
+        put(doc2, 7);
+        _dms->constructFreeList();
+    }
+
+    std::shared_ptr<search::IGidToLidMapperFactory> getGidToLidMapperFactory() {
+        return std::make_shared<GidToLidMapperFactory>(_dms);
+    }
+
+    void assertGenerations(generation_t currentGeneration, generation_t firstUsedGeneration)
+    {
+        (void) currentGeneration;
+        (void) firstUsedGeneration;
+#if 1
+        const GenerationHandler &handler = _dms->getGenerationHandler();
+        EXPECT_EQUAL(currentGeneration, handler.getCurrentGeneration());
+        EXPECT_EQUAL(firstUsedGeneration, handler.getFirstUsedGeneration());
+#endif
+    }
+};
+
+TEST_F("Test that we can use gid mapper to get lids", Fixture)
+{
+    auto factory = f.getGidToLidMapperFactory();
+    auto mapper = factory->getMapper();
+    TEST_DO(assertLid(mapper, doc1, 4));
+    TEST_DO(assertLid(mapper, doc2, 7));
+    TEST_DO(assertLid(mapper, doc3, 0));
+}
+
+TEST_F("Test that mapper holds read guard", Fixture)
+{
+    TEST_DO(f.assertGenerations(3, 3));
+    auto factory = f.getGidToLidMapperFactory();
+    uint32_t lid3 = f.put(doc3);
+    EXPECT_EQUAL(1u, lid3);
+    TEST_DO(assertLid(factory->getMapper(), doc3, 1));
+    TEST_DO(f.assertGenerations(4, 4));
+    // Remove and readd withoug guard, old docid can be reused
+    f.remove(lid3);
+    lid3 = f.put(doc3);
+    EXPECT_EQUAL(1u, lid3);
+    TEST_DO(assertLid(factory->getMapper(), doc3, 1));
+    TEST_DO(f.assertGenerations(7, 7));
+    // Remove and readd withoug guard, old docid cannot be reused
+    auto mapper = factory->getMapper();
+    f.remove(lid3);
+    lid3 = f.put(doc3);
+    EXPECT_EQUAL(2u, lid3);
+    TEST_DO(assertLid(mapper, doc3, 2));
+    TEST_DO(f.assertGenerations(10, 7));
+}
+
+}
+
+TEST_MAIN()
+{
+    TEST_RUN_ALL();
+}
diff --git a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt
new file mode 100644
index 00000000000..a74a1e58221
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(searchcore_reference STATIC
+    SOURCES
+    document_db_referent.cpp
+    document_db_referent_registry.cpp
+    gid_to_lid_mapper.cpp
+    gid_to_lid_mapper_factory.cpp
+    DEPENDS
+)
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_referent.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_referent.cpp
new file mode 100644
index 00000000000..e281a88edbe
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_referent.cpp
@@ -0,0 +1,37 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "document_db_referent.h"
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+
+namespace proton {
+
+DocumentDBReferent::DocumentDBReferent()
+    : _attrMgr(),
+      _dms()
+{
+}
+
+DocumentDBReferent::~DocumentDBReferent()
+{
+}
+
+std::shared_ptr<search::AttributeVector>
+DocumentDBReferent::getAttribute(vespalib::stringref name)
+{
+    search::AttributeGuard::UP guard = _attrMgr->getAttribute(name);
+    if (guard) {
+        return guard->getSP();
+    } else {
+        return std::shared_ptr<search::AttributeVector>();
+    }
+}
+
+std::shared_ptr<search::IGidToLidMapperFactory>
+DocumentDBReferent::getGidToLidMapperFactory()
+{
+    return std::shared_ptr<search::IGidToLidMapperFactory>();
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_referent.h b/searchcore/src/vespa/searchcore/proton/reference/document_db_referent.h
new file mode 100644
index 00000000000..ac1ac6aff02
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_referent.h
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_document_db_referent.h"
+
+namespace search
+{
+
+class IAttributeManager;
+class DocumentMetaStore;
+
+}
+
+namespace proton {
+
+/*
+ * Class for getting target attributes for imported
+ * attributes, and for getting interface for mapping to lids
+ * compatible with the target attributes.
+ */
+class DocumentDBReferent : public IDocumentDBReferent
+{
+    std::shared_ptr<search::IAttributeManager> _attrMgr;
+    std::shared_ptr<search::DocumentMetaStore> _dms;
+public:
+    DocumentDBReferent();
+    virtual ~DocumentDBReferent();
+    virtual std::shared_ptr<search::AttributeVector> getAttribute(vespalib::stringref name) override;
+    virtual std::shared_ptr<search::IGidToLidMapperFactory> getGidToLidMapperFactory() override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_referent_registry.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_referent_registry.cpp
new file mode 100644
index 00000000000..8e538408ea8
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_referent_registry.cpp
@@ -0,0 +1,44 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "document_db_referent_registry.h"
+
+namespace proton {
+
+DocumentDBReferentRegistry::DocumentDBReferentRegistry()
+    : _lock()
+{
+}
+
+DocumentDBReferentRegistry::~DocumentDBReferentRegistry()
+{
+}
+
+std::shared_ptr<IDocumentDBReferent>
+DocumentDBReferentRegistry::getDocumentDBReferent(vespalib::stringref name) const
+{
+    std::unique_lock<std::mutex> guard(_lock);
+    auto itr = _handlers.find(name);
+    while (itr == _handlers.end()) {
+        _cv.wait(guard);
+        itr = _handlers.find(name);
+    }
+    return itr->second;
+}
+
+void
+DocumentDBReferentRegistry::addDocumentDBReferent(vespalib::stringref name, std::shared_ptr<IDocumentDBReferent> referee)
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    _handlers[name] = referee;
+    _cv.notify_all();
+}
+
+void
+DocumentDBReferentRegistry::removeDocumentDBReferent(vespalib::stringref name)
+{
+    std::lock_guard<std::mutex> guard(_lock);
+    _handlers.erase(name);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_referent_registry.h b/searchcore/src/vespa/searchcore/proton/reference/document_db_referent_registry.h
new file mode 100644
index 00000000000..7cd41cfe18b
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_referent_registry.h
@@ -0,0 +1,28 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "i_document_db_referent_registry.h"
+#include <mutex>
+#include <condition_variable>
+#include <map>
+
+namespace proton {
+
+/*
+ * Class implementing a registry of named IDocumentDBReferents.
+ */
+class DocumentDBReferentRegistry : public IDocumentDBReferentRegistry
+{
+    mutable std::mutex _lock;
+    mutable std::condition_variable _cv;
+    std::map<vespalib::string, std::shared_ptr<IDocumentDBReferent>> _handlers;
+public:
+    DocumentDBReferentRegistry();
+    virtual ~DocumentDBReferentRegistry();
+
+    virtual std::shared_ptr<IDocumentDBReferent> getDocumentDBReferent(vespalib::stringref docType) const override;
+    virtual void addDocumentDBReferent(vespalib::stringref name, std::shared_ptr<IDocumentDBReferent> referee) override;
+    virtual void removeDocumentDBReferent(vespalib::stringref name) override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp
new file mode 100644
index 00000000000..99bf79c3ca2
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp
@@ -0,0 +1,30 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "gid_to_lid_mapper.h"
+
+namespace proton {
+
+GidToLidMapper::GidToLidMapper(vespalib::GenerationHandler::Guard &&guard,
+                               const search::IDocumentMetaStore &dms)
+    : _guard(std::move(guard)),
+      _dms(dms)
+{
+}
+
+GidToLidMapper::~GidToLidMapper()
+{
+}
+
+uint32_t
+GidToLidMapper::mapGidToLid(const document::GlobalId &gid) const
+{
+    uint32_t lid = 0;
+    if (_dms.getLid(gid, lid)) {
+        return lid;
+    } else {
+        return 0u;
+    }
+}
+
+} // namespace search
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.h
new file mode 100644
index 00000000000..ebb5402418d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.h
@@ -0,0 +1,26 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/i_gid_to_lid_mapper.h>
+#include <vespa/vespalib/util/generationhandler.h>
+#include <vespa/searchlib/common/idocumentmetastore.h>
+
+namespace proton {
+
+/*
+ * Class for mapping from gid to lid. Instances should be short lived
+ * due to read guards preventing resource reuse.
+ */
+class GidToLidMapper : public search::IGidToLidMapper
+{
+    vespalib::GenerationHandler::Guard _guard;
+    const search::IDocumentMetaStore &_dms;
+public:
+    GidToLidMapper(vespalib::GenerationHandler::Guard &&guard,
+                   const search::IDocumentMetaStore &dms);
+    virtual ~GidToLidMapper();
+    virtual uint32_t mapGidToLid(const document::GlobalId &gid) const override;
+};
+
+} // namespace search
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.cpp
new file mode 100644
index 00000000000..69605ed61a0
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.cpp
@@ -0,0 +1,24 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "gid_to_lid_mapper_factory.h"
+#include "gid_to_lid_mapper.h"
+
+namespace proton {
+
+GidToLidMapperFactory::GidToLidMapperFactory(std::shared_ptr<DocumentMetaStore> dms)
+    : _dms(std::move(dms))
+{
+}
+
+GidToLidMapperFactory::~GidToLidMapperFactory()
+{
+}
+
+std::unique_ptr<search::IGidToLidMapper>
+GidToLidMapperFactory::getMapper() const
+{
+    return std::make_unique<GidToLidMapper>(_dms->getGenerationHandler().takeGuard(), *_dms);
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.h
new file mode 100644
index 00000000000..bfcf2687d0a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.h
@@ -0,0 +1,19 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h>
+#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
+
+namespace proton {
+
+class GidToLidMapperFactory : public search::IGidToLidMapperFactory
+{
+    std::shared_ptr<DocumentMetaStore> _dms;
+public:
+    GidToLidMapperFactory(std::shared_ptr<DocumentMetaStore> dms);
+    virtual ~GidToLidMapperFactory();
+    virtual std::unique_ptr<search::IGidToLidMapper> getMapper() const override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_referent.h b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_referent.h
new file mode 100644
index 00000000000..695539bf591
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_referent.h
@@ -0,0 +1,28 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace search {
+
+class AttributeVector;
+class IGidToLidMapperFactory;
+
+}
+
+namespace proton {
+
+/*
+ * Interface class for getting target attributes for imported
+ * attributes, and for getting interface for mapping to lids
+ * compatible with the target attributes.
+ */
+class IDocumentDBReferent
+{
+public:
+    virtual ~IDocumentDBReferent() { }
+    virtual std::shared_ptr<search::AttributeVector> getAttribute(vespalib::stringref name) = 0;
+    virtual std::shared_ptr<search::IGidToLidMapperFactory> getGidToLidMapperFactory() = 0;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_referent_registry.h b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_referent_registry.h
new file mode 100644
index 00000000000..a7cf06c9592
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_referent_registry.h
@@ -0,0 +1,28 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+class IDocumentDBReferent;
+
+/*
+ * Interface class for a registry of named IDocumentDBReferents.
+ */
+class IDocumentDBReferentRegistry
+{
+public:
+    virtual ~IDocumentDBReferentRegistry() { }
+
+    /*
+     * Get named IDocumentDBReferent.  Block while it doesn't exist.
+     */
+    virtual std::shared_ptr<IDocumentDBReferent> getDocumentDBReferent(vespalib::stringref name) const = 0;
+
+    virtual void addDocumentDBReferent(vespalib::stringref name, std::shared_ptr<IDocumentDBReferent> referee) = 0;
+
+    virtual void removeDocumentDBReferent(vespalib::stringref name) = 0;
+};
+
+} // namespace proton
diff --git a/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h
new file mode 100644
index 00000000000..69bb76e76c5
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h
@@ -0,0 +1,21 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document { class GlobalId; }
+
+namespace search {
+
+/*
+ * Interface class for mapping from gid to lid. Instances should be short
+ * lived due to implementations containing read guards preventing resource
+ * reuse.
+ */
+class IGidToLidMapper
+{
+public:
+    virtual ~IGidToLidMapper() { }
+    virtual uint32_t mapGidToLid(const document::GlobalId &gid) const = 0;
+};
+
+} // namespace search
diff --git a/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper_factory.h b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper_factory.h
new file mode 100644
index 00000000000..fadd0b74020
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper_factory.h
@@ -0,0 +1,19 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search {
+
+class IGidToLidMapper;
+
+/*
+ * Interface to factory class for creating classes mapping from gid to lid.
+ */
+class IGidToLidMapperFactory
+{
+public:
+    virtual ~IGidToLidMapperFactory() { }
+    virtual std::unique_ptr<IGidToLidMapper> getMapper() const = 0;
+};
+
+} // namespace search