summaryrefslogtreecommitdiffstats
path: root/searchlib
diff options
context:
space:
mode:
authorGeir Storli <geirst@verizonmedia.com>2019-06-19 13:04:42 +0000
committerGeir Storli <geirst@verizonmedia.com>2019-06-19 13:06:30 +0000
commita8e57585e422b4758af5c178e252409eb3f43b30 (patch)
tree36e2f3539847d4b086b25c75ccf9c051312bca32 /searchlib
parentd995dda4234ed90e4c9c46688c54c6be409e46bb (diff)
Add template parameter to FieldIndex that specifies whether the underlying posting lists have interleaved features or not.
Currently, we only support posting lists without interleaved features.
Diffstat (limited to 'searchlib')
-rw-r--r--searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp9
-rw-r--r--searchlib/src/tests/memoryindex/field_index/field_index_test.cpp94
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt1
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/field_index.cpp130
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/field_index.h161
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/field_index_base.cpp36
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/field_index_base.h119
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp35
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h23
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp24
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h9
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h2
-rw-r--r--searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp8
-rw-r--r--searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h5
-rw-r--r--searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h2
16 files changed, 389 insertions, 271 deletions
diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp
index df7f80e8601..36e9bde5c9f 100644
--- a/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp
+++ b/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp
@@ -18,10 +18,13 @@ using namespace search::memoryindex;
using search::index::schema::DataType;
using search::test::SearchIteratorVerifier;
+using FieldIndexType = FieldIndex<false>;
+using PostingIteratorType = PostingIterator<false>;
+
class Verifier : public SearchIteratorVerifier {
private:
mutable TermFieldMatchData _tfmd;
- FieldIndex _field_index;
+ FieldIndexType _field_index;
public:
Verifier(const Schema& schema)
@@ -41,8 +44,8 @@ public:
(void) strict;
TermFieldMatchDataArray match_data;
match_data.add(&_tfmd);
- return std::make_unique<PostingIterator>(_field_index.find("a"),
- _field_index.getFeatureStore(), 0, match_data);
+ return std::make_unique<PostingIteratorType>(_field_index.find("a"),
+ _field_index.getFeatureStore(), 0, match_data);
}
};
diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
index 15d97d314a1..f2cc2580cd8 100644
--- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
+++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
@@ -40,8 +40,10 @@ using vespalib::GenerationHandler;
namespace memoryindex {
using test::WrapInserter;
-using PostingList = FieldIndex::PostingList;
+using FieldIndexType = FieldIndex<false>;
+using PostingList = FieldIndexType::PostingList;
using PostingConstItr = PostingList::ConstIterator;
+using PostingIteratorType = PostingIterator<false>;
class MyBuilder : public IndexBuilder {
private:
@@ -197,22 +199,22 @@ assertPostingList(std::vector<uint32_t> &exp, PostingConstItr itr)
return assertPostingList(ss.str(), itr);
}
-FieldIndex::PostingList::Iterator
+FieldIndexType::PostingList::Iterator
find_in_field_index(const vespalib::stringref word,
uint32_t fieldId,
const FieldIndexCollection& fic)
{
- auto* field_index = dynamic_cast<FieldIndex*>(fic.getFieldIndex(fieldId));
+ auto* field_index = dynamic_cast<FieldIndexType*>(fic.getFieldIndex(fieldId));
assert(field_index != nullptr);
return field_index->find(word);
}
-FieldIndex::PostingList::ConstIterator
+FieldIndexType::PostingList::ConstIterator
find_frozen_in_field_index(const vespalib::stringref word,
uint32_t fieldId,
const FieldIndexCollection& fic)
{
- auto* field_index = dynamic_cast<FieldIndex*>(fic.getFieldIndex(fieldId));
+ auto* field_index = dynamic_cast<FieldIndexType*>(fic.getFieldIndex(fieldId));
assert(field_index != nullptr);
return field_index->findFrozen(word);
}
@@ -409,7 +411,7 @@ public:
{
}
- MyDrainRemoves(FieldIndex& field_index)
+ MyDrainRemoves(FieldIndexType& field_index)
: _remover(field_index.getDocumentRemover())
{
}
@@ -487,7 +489,7 @@ make_single_field_schema()
struct FieldIndexTest : public ::testing::Test {
Schema schema;
- FieldIndex idx;
+ FieldIndexType idx;
FieldIndexTest()
: schema(make_single_field_schema()),
idx(schema, 0)
@@ -518,8 +520,8 @@ struct FieldIndexCollectionTest : public ::testing::Test {
}
~FieldIndexCollectionTest() {}
- FieldIndex::PostingList::Iterator find(const vespalib::stringref word,
- uint32_t fieldId) const {
+ FieldIndexType::PostingList::Iterator find(const vespalib::stringref word,
+ uint32_t fieldId) const {
return find_in_field_index(word, fieldId, fic);
}
};
@@ -671,16 +673,16 @@ TEST_F(FieldIndexTest, require_that_posting_iterator_is_working)
TermFieldMatchDataArray matchData;
matchData.add(&tfmd);
{
- PostingIterator itr(idx.find("not"),
- idx.getFeatureStore(),
- 0, matchData);
+ PostingIteratorType itr(idx.find("not"),
+ idx.getFeatureStore(),
+ 0, matchData);
itr.initFullRange();
EXPECT_TRUE(itr.isAtEnd());
}
{
- PostingIterator itr(idx.find("a"),
- idx.getFeatureStore(),
- 0, matchData);
+ PostingIteratorType itr(idx.find("a"),
+ idx.getFeatureStore(),
+ 0, matchData);
itr.initFullRange();
EXPECT_EQ(10u, itr.getDocId());
itr.unpack(10);
@@ -954,12 +956,12 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
TermFieldMatchDataArray matchData;
matchData.add(&tfmd);
{
- PostingIterator itr(findFrozen("not", 0), featureStoreRef(_fic, 0), 0, matchData);
+ PostingIteratorType itr(findFrozen("not", 0), featureStoreRef(_fic, 0), 0, matchData);
itr.initFullRange();
EXPECT_TRUE(itr.isAtEnd());
}
{
- PostingIterator itr(findFrozen("a", 0), featureStoreRef(_fic, 0), 0, matchData);
+ PostingIteratorType itr(findFrozen("a", 0), featureStoreRef(_fic, 0), 0, matchData);
itr.initFullRange();
EXPECT_EQ(10u, itr.getDocId());
itr.unpack(10);
@@ -976,19 +978,19 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
EXPECT_TRUE(itr.isAtEnd());
}
{
- PostingIterator itr(findFrozen("x", 0), featureStoreRef(_fic, 0), 0, matchData);
+ PostingIteratorType itr(findFrozen("x", 0), featureStoreRef(_fic, 0), 0, matchData);
itr.initFullRange();
EXPECT_TRUE(itr.isAtEnd());
}
{
- PostingIterator itr(findFrozen("x", 1), featureStoreRef(_fic, 1), 1, matchData);
+ PostingIteratorType itr(findFrozen("x", 1), featureStoreRef(_fic, 1), 1, matchData);
itr.initFullRange();
EXPECT_EQ(30u, itr.getDocId());
itr.unpack(30);
EXPECT_EQ("{6:2[e=0,w=1,l=6]}", toString(tfmd.getIterator(), true, true));
}
{
- PostingIterator itr(findFrozen("x", 2), featureStoreRef(_fic, 2), 2, matchData);
+ PostingIteratorType itr(findFrozen("x", 2), featureStoreRef(_fic, 2), 2, matchData);
itr.initFullRange();
EXPECT_EQ(30u, itr.getDocId());
itr.unpack(30);
@@ -996,7 +998,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
EXPECT_EQ("{2:1[e=0,w=1,l=2]}", toString(tfmd.getIterator(), true, true));
}
{
- PostingIterator itr(findFrozen("x", 3), featureStoreRef(_fic, 3), 3, matchData);
+ PostingIteratorType itr(findFrozen("x", 3), featureStoreRef(_fic, 3), 3, matchData);
itr.initFullRange();
EXPECT_EQ(30u, itr.getDocId());
itr.unpack(30);
@@ -1193,17 +1195,17 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working)
matchData.add(&tfmd);
{
uint32_t fieldId = _schema.getIndexFieldId("iu");
- PostingIterator itr(findFrozen("not", fieldId),
- featureStoreRef(_fic, fieldId),
- fieldId, matchData);
+ PostingIteratorType itr(findFrozen("not", fieldId),
+ featureStoreRef(_fic, fieldId),
+ fieldId, matchData);
itr.initFullRange();
EXPECT_TRUE(itr.isAtEnd());
}
{
uint32_t fieldId = _schema.getIndexFieldId("iu");
- PostingIterator itr(findFrozen("example", fieldId),
- featureStoreRef(_fic, fieldId),
- fieldId, matchData);
+ PostingIteratorType itr(findFrozen("example", fieldId),
+ featureStoreRef(_fic, fieldId),
+ fieldId, matchData);
itr.initFullRange();
EXPECT_EQ(10u, itr.getDocId());
itr.unpack(10);
@@ -1213,9 +1215,9 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working)
}
{
uint32_t fieldId = _schema.getIndexFieldId("iau");
- PostingIterator itr(findFrozen("example", fieldId),
- featureStoreRef(_fic, fieldId),
- fieldId, matchData);
+ PostingIteratorType itr(findFrozen("example", fieldId),
+ featureStoreRef(_fic, fieldId),
+ fieldId, matchData);
itr.initFullRange();
EXPECT_EQ(10u, itr.getDocId());
itr.unpack(10);
@@ -1226,9 +1228,9 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working)
}
{
uint32_t fieldId = _schema.getIndexFieldId("iwu");
- PostingIterator itr(findFrozen("example", fieldId),
- featureStoreRef(_fic, fieldId),
- fieldId, matchData);
+ PostingIteratorType itr(findFrozen("example", fieldId),
+ featureStoreRef(_fic, fieldId),
+ fieldId, matchData);
itr.initFullRange();
EXPECT_EQ(10u, itr.getDocId());
itr.unpack(10);
@@ -1279,18 +1281,18 @@ TEST_F(CjkInverterTest, require_that_cjk_indexing_is_working)
matchData.add(&tfmd);
uint32_t fieldId = _schema.getIndexFieldId("f0");
{
- PostingIterator itr(findFrozen("not", fieldId),
- featureStoreRef(_fic, fieldId),
- fieldId, matchData);
+ PostingIteratorType itr(findFrozen("not", fieldId),
+ featureStoreRef(_fic, fieldId),
+ fieldId, matchData);
itr.initFullRange();
EXPECT_TRUE(itr.isAtEnd());
}
{
- PostingIterator itr(findFrozen("我就"
- "是那个",
- fieldId),
- featureStoreRef(_fic, fieldId),
- fieldId, matchData);
+ PostingIteratorType itr(findFrozen("我就"
+ "是那个",
+ fieldId),
+ featureStoreRef(_fic, fieldId),
+ fieldId, matchData);
itr.initFullRange();
EXPECT_EQ(10u, itr.getDocId());
itr.unpack(10);
@@ -1299,11 +1301,11 @@ TEST_F(CjkInverterTest, require_that_cjk_indexing_is_working)
EXPECT_TRUE(itr.isAtEnd());
}
{
- PostingIterator itr(findFrozen("大灰"
- "狼",
- fieldId),
- featureStoreRef(_fic, fieldId),
- fieldId, matchData);
+ PostingIteratorType itr(findFrozen("大灰"
+ "狼",
+ fieldId),
+ featureStoreRef(_fic, fieldId),
+ fieldId, matchData);
itr.initFullRange();
EXPECT_EQ(10u, itr.getDocId());
itr.unpack(10);
diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt
index 441fe12c383..c19596692cc 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt
@@ -5,6 +5,7 @@ vespa_add_library(searchlib_memoryindex OBJECT
document_inverter.cpp
feature_store.cpp
field_index.cpp
+ field_index_base.cpp
field_index_collection.cpp
field_index_remover.cpp
field_inverter.cpp
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
index e492e3e7eee..a5dc921cfdf 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
@@ -48,33 +48,24 @@ void set_interleaved_features(DocIdAndFeatures &features)
using datastore::EntryRef;
-vespalib::asciistream &
-operator<<(vespalib::asciistream & os, const FieldIndex::WordKey & rhs)
-{
- os << "wr(" << rhs._wordRef.ref() << ")";
- return os;
-}
-
-FieldIndex::FieldIndex(const index::Schema& schema, uint32_t fieldId)
+template <bool interleaved_features>
+FieldIndex<interleaved_features>::FieldIndex(const index::Schema& schema, uint32_t fieldId)
: FieldIndex(schema, fieldId, index::FieldLengthInfo())
{
}
-FieldIndex::FieldIndex(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info)
- : _wordStore(),
- _numUniqueWords(0),
- _generationHandler(),
- _dict(),
- _postingListStore(),
- _featureStore(schema),
- _fieldId(fieldId),
- _remover(_wordStore),
- _inserter(std::make_unique<OrderedFieldIndexInserter>(*this)),
- _calculator(info)
+template <bool interleaved_features>
+FieldIndex<interleaved_features>::FieldIndex(const index::Schema& schema, uint32_t fieldId,
+ const index::FieldLengthInfo& info)
+ : FieldIndexBase(schema, fieldId, info),
+ _postingListStore()
{
+ using InserterType = OrderedFieldIndexInserter<interleaved_features>;
+ _inserter = std::make_unique<InserterType>(*this);
}
-FieldIndex::~FieldIndex()
+template <bool interleaved_features>
+FieldIndex<interleaved_features>::~FieldIndex()
{
_postingListStore.disableFreeLists();
_postingListStore.disableElemHoldList();
@@ -101,28 +92,31 @@ FieldIndex::~FieldIndex()
trimHoldLists();
}
-FieldIndex::PostingList::Iterator
-FieldIndex::find(const vespalib::stringref word) const
+template <bool interleaved_features>
+typename FieldIndex<interleaved_features>::PostingList::Iterator
+FieldIndex<interleaved_features>::find(const vespalib::stringref word) const
{
DictionaryTree::Iterator itr = _dict.find(WordKey(EntryRef()), KeyComp(_wordStore, word));
if (itr.valid()) {
return _postingListStore.begin(EntryRef(itr.getData()));
}
- return PostingList::Iterator();
+ return typename PostingList::Iterator();
}
-FieldIndex::PostingList::ConstIterator
-FieldIndex::findFrozen(const vespalib::stringref word) const
+template <bool interleaved_features>
+typename FieldIndex<interleaved_features>::PostingList::ConstIterator
+FieldIndex<interleaved_features>::findFrozen(const vespalib::stringref word) const
{
auto itr = _dict.getFrozenView().find(WordKey(EntryRef()), KeyComp(_wordStore, word));
if (itr.valid()) {
return _postingListStore.beginFrozen(EntryRef(itr.getData()));
}
- return PostingList::Iterator();
+ return typename PostingList::Iterator();
}
+template <bool interleaved_features>
void
-FieldIndex::compactFeatures()
+FieldIndex<interleaved_features>::compactFeatures()
{
std::vector<uint32_t> toHold;
@@ -130,7 +124,7 @@ FieldIndex::compactFeatures()
auto itr = _dict.begin();
uint32_t packedIndex = _fieldId;
for (; itr.valid(); ++itr) {
- PostingListStore::RefType pidx(EntryRef(itr.getData()));
+ typename PostingListStore::RefType pidx(EntryRef(itr.getData()));
if (!pidx.valid()) {
continue;
}
@@ -177,8 +171,9 @@ FieldIndex::compactFeatures()
_featureStore.transferHoldLists(generation);
}
+template <bool interleaved_features>
void
-FieldIndex::dump(search::index::IndexBuilder & indexBuilder)
+FieldIndex<interleaved_features>::dump(search::index::IndexBuilder & indexBuilder)
{
vespalib::stringref word;
FeatureStore::DecodeContextCooked decoder(nullptr);
@@ -187,7 +182,7 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder)
_featureStore.setupForField(_fieldId, decoder);
for (auto itr = _dict.begin(); itr.valid(); ++itr) {
const WordKey & wk = itr.getKey();
- PostingListStore::RefType plist(EntryRef(itr.getData()));
+ typename PostingListStore::RefType plist(EntryRef(itr.getData()));
word = _wordStore.getWord(wk._wordRef);
if (!plist.valid()) {
continue;
@@ -225,8 +220,9 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder)
}
}
+template <bool interleaved_features>
vespalib::MemoryUsage
-FieldIndex::getMemoryUsage() const
+FieldIndex<interleaved_features>::getMemoryUsage() const
{
vespalib::MemoryUsage usage;
usage.merge(_wordStore.getMemoryUsage());
@@ -239,17 +235,20 @@ FieldIndex::getMemoryUsage() const
namespace {
+template <bool interleaved_features>
class MemoryTermBlueprint : public SimpleLeafBlueprint {
private:
+ using FieldIndexType = FieldIndex<interleaved_features>;
+ using PostingListIteratorType = typename FieldIndexType::PostingList::ConstIterator;
GenerationHandler::Guard _guard;
- FieldIndex::PostingList::ConstIterator _posting_itr;
+ PostingListIteratorType _posting_itr;
const FeatureStore& _feature_store;
const uint32_t _field_id;
const bool _use_bit_vector;
public:
MemoryTermBlueprint(GenerationHandler::Guard&& guard,
- FieldIndex::PostingList::ConstIterator posting_itr,
+ PostingListIteratorType posting_itr,
const FeatureStore& feature_store,
const FieldSpecBase& field,
uint32_t field_id,
@@ -267,7 +266,8 @@ public:
}
SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray& tfmda, bool) const override {
- auto result = std::make_unique<PostingIterator>(_posting_itr, _feature_store, _field_id, tfmda);
+ using PostingIteratorType = PostingIterator<interleaved_features>;
+ auto result = std::make_unique<PostingIteratorType>(_posting_itr, _feature_store, _field_id, tfmda);
if (_use_bit_vector) {
LOG(debug, "Return BooleanMatchIteratorWrapper: field_id(%u), doc_count(%zu)",
_field_id, _posting_itr.size());
@@ -281,95 +281,99 @@ public:
}
+template <bool interleaved_features>
std::unique_ptr<queryeval::SimpleLeafBlueprint>
-FieldIndex::make_term_blueprint(const vespalib::string& term,
- const queryeval::FieldSpecBase& field,
- uint32_t field_id)
+FieldIndex<interleaved_features>::make_term_blueprint(const vespalib::string& term,
+ const queryeval::FieldSpecBase& field,
+ uint32_t field_id)
{
auto guard = takeGenerationGuard();
auto posting_itr = findFrozen(term);
bool use_bit_vector = field.isFilter();
- return std::make_unique<MemoryTermBlueprint>(std::move(guard), posting_itr, getFeatureStore(),
- field, field_id, use_bit_vector);
+ return std::make_unique<MemoryTermBlueprint<interleaved_features>>
+ (std::move(guard), posting_itr, getFeatureStore(), field, field_id, use_bit_vector);
}
+template
+class FieldIndex<false>;
+
}
namespace search::btree {
template
-class BTreeNodeDataWrap<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::LEAF_SLOTS>;
+class BTreeNodeDataWrap<memoryindex::FieldIndexBase::WordKey, BTreeDefaultTraits::LEAF_SLOTS>;
template
-class BTreeNodeT<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::INTERNAL_SLOTS>;
+class BTreeNodeT<memoryindex::FieldIndexBase::WordKey, BTreeDefaultTraits::INTERNAL_SLOTS>;
#if 0
template
-class BTreeNodeT<memoryindex::FieldIndex::WordKey,
+class BTreeNodeT<memoryindex::FieldIndexBase::WordKey,
BTreeDefaultTraits::LEAF_SLOTS>;
#endif
template
-class BTreeNodeTT<memoryindex::FieldIndex::WordKey,
+class BTreeNodeTT<memoryindex::FieldIndexBase::WordKey,
datastore::EntryRef,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS>;
template
-class BTreeNodeTT<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeNodeTT<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::LEAF_SLOTS>;
template
-class BTreeInternalNode<memoryindex::FieldIndex::WordKey,
+class BTreeInternalNode<memoryindex::FieldIndexBase::WordKey,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS>;
template
-class BTreeLeafNode<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeLeafNode<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::LEAF_SLOTS>;
template
-class BTreeNodeStore<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeNodeStore<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS,
BTreeDefaultTraits::LEAF_SLOTS>;
template
-class BTreeIterator<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeIterator<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
- const memoryindex::FieldIndex::KeyComp,
+ const memoryindex::FieldIndexBase::KeyComp,
BTreeDefaultTraits>;
template
-class BTree<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTree<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
- const memoryindex::FieldIndex::KeyComp,
+ const memoryindex::FieldIndexBase::KeyComp,
BTreeDefaultTraits>;
template
-class BTreeRoot<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeRoot<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
- const memoryindex::FieldIndex::KeyComp,
+ const memoryindex::FieldIndexBase::KeyComp,
BTreeDefaultTraits>;
template
-class BTreeRootBase<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeRootBase<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS,
BTreeDefaultTraits::LEAF_SLOTS>;
template
-class BTreeNodeAllocator<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeNodeAllocator<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS,
BTreeDefaultTraits::LEAF_SLOTS>;
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h
index 9c97ebf3e85..05665945800 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/field_index.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h
@@ -2,20 +2,13 @@
#pragma once
-#include "feature_store.h"
-#include "field_index_remover.h"
-#include "i_field_index.h"
+#include "field_index_base.h"
#include "posting_list_entry.h"
-#include "word_store.h"
-#include <vespa/searchlib/index/docidandfeatures.h>
-#include <vespa/searchlib/index/field_length_calculator.h>
#include <vespa/searchlib/index/indexbuilder.h>
#include <vespa/vespalib/btree/btree.h>
#include <vespa/vespalib/btree/btreenodeallocator.h>
#include <vespa/vespalib/btree/btreeroot.h>
#include <vespa/vespalib/btree/btreestore.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/util/memoryusage.h>
namespace search::memoryindex {
@@ -33,95 +26,24 @@ class IOrderedFieldIndexInserter;
* This information is unpacked and used during ranking.
*
* Elements in the three stores are accessed using 32-bit references / handles.
+ *
+ * The template parameter specifies whether the underlying posting lists have interleaved features or not.
*/
-class FieldIndex : public IFieldIndex {
+template <bool interleaved_features>
+class FieldIndex : public FieldIndexBase {
public:
// Mapping from docid -> feature ref
- using PostingListEntryType = PostingListEntry<false>;
+ using PostingListEntryType = PostingListEntry<interleaved_features>;
using PostingList = btree::BTreeRoot<uint32_t, PostingListEntryType, search::btree::NoAggregated>;
using PostingListStore = btree::BTreeStore<uint32_t, PostingListEntryType,
search::btree::NoAggregated,
std::less<uint32_t>,
btree::BTreeDefaultTraits>;
- using PostingListKeyDataType = PostingListStore::KeyDataType;
-
- struct WordKey {
- datastore::EntryRef _wordRef;
-
- explicit WordKey(datastore::EntryRef wordRef) : _wordRef(wordRef) { }
- WordKey() : _wordRef() { }
-
- friend vespalib::asciistream &
- operator<<(vespalib::asciistream & os, const WordKey & rhs);
- };
-
- class KeyComp {
- private:
- const WordStore &_wordStore;
- const vespalib::stringref _word;
-
- const char *getWord(datastore::EntryRef wordRef) const {
- if (wordRef.valid()) {
- return _wordStore.getWord(wordRef);
- }
- return _word.data();
- }
-
- public:
- KeyComp(const WordStore &wordStore, const vespalib::stringref word)
- : _wordStore(wordStore),
- _word(word)
- { }
-
- bool operator()(const WordKey & lhs, const WordKey & rhs) const {
- int cmpres = strcmp(getWord(lhs._wordRef), getWord(rhs._wordRef));
- return cmpres < 0;
- }
- };
-
- using PostingListPtr = uint32_t;
- using DictionaryTree = btree::BTree<WordKey, PostingListPtr,
- search::btree::NoAggregated,
- const KeyComp>;
-private:
- using GenerationHandler = vespalib::GenerationHandler;
-
- WordStore _wordStore;
- uint64_t _numUniqueWords;
- GenerationHandler _generationHandler;
- DictionaryTree _dict;
- PostingListStore _postingListStore;
- FeatureStore _featureStore;
- uint32_t _fieldId;
- FieldIndexRemover _remover;
- std::unique_ptr<IOrderedFieldIndexInserter> _inserter;
- index::FieldLengthCalculator _calculator;
-
-public:
- datastore::EntryRef addWord(const vespalib::stringref word) {
- _numUniqueWords++;
- return _wordStore.addWord(word);
- }
-
- datastore::EntryRef addFeatures(const index::DocIdAndFeatures &features) {
- return _featureStore.addFeatures(_fieldId, features).first;
- }
-
- FieldIndex(const index::Schema& schema, uint32_t fieldId);
- FieldIndex(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info);
- ~FieldIndex();
- PostingList::Iterator find(const vespalib::stringref word) const;
-
- PostingList::ConstIterator
- findFrozen(const vespalib::stringref word) const;
-
- uint64_t getNumUniqueWords() const override { return _numUniqueWords; }
- const FeatureStore & getFeatureStore() const override { return _featureStore; }
- const WordStore &getWordStore() const override { return _wordStore; }
- IOrderedFieldIndexInserter &getInserter() override { return *_inserter; }
- index::FieldLengthCalculator &get_calculator() override { return _calculator; }
+ using PostingListKeyDataType = typename PostingListStore::KeyDataType;
private:
+ PostingListStore _postingListStore;
+
void freeze() {
_postingListStore.freeze();
_dict.getAllocator().freeze();
@@ -148,18 +70,19 @@ private:
}
public:
- GenerationHandler::Guard takeGenerationGuard() override {
- return _generationHandler.takeGuard();
- }
+ FieldIndex(const index::Schema& schema, uint32_t fieldId);
+ FieldIndex(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info);
+ ~FieldIndex();
+
+ typename PostingList::Iterator find(const vespalib::stringref word) const;
+ typename PostingList::ConstIterator findFrozen(const vespalib::stringref word) const;
void compactFeatures() override;
void dump(search::index::IndexBuilder & indexBuilder) override;
vespalib::MemoryUsage getMemoryUsage() const override;
- DictionaryTree &getDictionaryTree() { return _dict; }
PostingListStore &getPostingListStore() { return _postingListStore; }
- FieldIndexRemover &getDocumentRemover() override { return _remover; }
void commit() override {
_remover.flush();
@@ -179,82 +102,82 @@ public:
namespace search::btree {
extern template
-class BTreeNodeDataWrap<memoryindex::FieldIndex::WordKey,
+class BTreeNodeDataWrap<memoryindex::FieldIndexBase::WordKey,
BTreeDefaultTraits::LEAF_SLOTS>;
extern template
-class BTreeNodeT<memoryindex::FieldIndex::WordKey,
+class BTreeNodeT<memoryindex::FieldIndexBase::WordKey,
BTreeDefaultTraits::INTERNAL_SLOTS>;
#if 0
extern template
-class BTreeNodeT<memoryindex::FieldIndex::WordKey,
+class BTreeNodeT<memoryindex::FieldIndexBase::WordKey,
BTreeDefaultTraits::LEAF_SLOTS>;
#endif
extern template
-class BTreeNodeTT<memoryindex::FieldIndex::WordKey,
+class BTreeNodeTT<memoryindex::FieldIndexBase::WordKey,
datastore::EntryRef,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS>;
extern template
-class BTreeNodeTT<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeNodeTT<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::LEAF_SLOTS>;
extern template
-class BTreeInternalNode<memoryindex::FieldIndex::WordKey,
+class BTreeInternalNode<memoryindex::FieldIndexBase::WordKey,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS>;
extern template
-class BTreeLeafNode<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeLeafNode<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::LEAF_SLOTS>;
extern template
-class BTreeNodeStore<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeNodeStore<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS,
BTreeDefaultTraits::LEAF_SLOTS>;
extern template
-class BTreeIterator<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeIterator<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
- const memoryindex::FieldIndex::KeyComp,
+ const memoryindex::FieldIndexBase::KeyComp,
BTreeDefaultTraits>;
extern template
-class BTree<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTree<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
- const memoryindex::FieldIndex::KeyComp,
+ const memoryindex::FieldIndexBase::KeyComp,
BTreeDefaultTraits>;
extern template
-class BTreeRoot<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeRoot<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
- const memoryindex::FieldIndex::KeyComp,
- BTreeDefaultTraits>;
+ const memoryindex::FieldIndexBase::KeyComp,
+ BTreeDefaultTraits>;
extern template
-class BTreeRootBase<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeRootBase<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
BTreeDefaultTraits::INTERNAL_SLOTS,
BTreeDefaultTraits::LEAF_SLOTS>;
extern template
-class BTreeNodeAllocator<memoryindex::FieldIndex::WordKey,
- memoryindex::FieldIndex::PostingListPtr,
+class BTreeNodeAllocator<memoryindex::FieldIndexBase::WordKey,
+ memoryindex::FieldIndexBase::PostingListPtr,
search::btree::NoAggregated,
- BTreeDefaultTraits::INTERNAL_SLOTS,
- BTreeDefaultTraits::LEAF_SLOTS>;
+ BTreeDefaultTraits::INTERNAL_SLOTS,
+ BTreeDefaultTraits::LEAF_SLOTS>;
}
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_base.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.cpp
new file mode 100644
index 00000000000..ee1fee3d935
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.cpp
@@ -0,0 +1,36 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "field_index_base.h"
+#include "i_ordered_field_index_inserter.h"
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace search::memoryindex {
+
+vespalib::asciistream &
+operator<<(vespalib::asciistream& os, const FieldIndexBase::WordKey& rhs)
+{
+ os << "wr(" << rhs._wordRef.ref() << ")";
+ return os;
+}
+
+FieldIndexBase::FieldIndexBase(const index::Schema& schema, uint32_t fieldId)
+ : FieldIndexBase(schema, fieldId, index::FieldLengthInfo())
+{
+}
+
+FieldIndexBase::FieldIndexBase(const index::Schema& schema, uint32_t fieldId,
+ const index::FieldLengthInfo& info)
+ : _wordStore(),
+ _numUniqueWords(0),
+ _generationHandler(),
+ _dict(),
+ _featureStore(schema),
+ _fieldId(fieldId),
+ _remover(_wordStore),
+ _inserter(),
+ _calculator(info)
+{
+}
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h
new file mode 100644
index 00000000000..7efec1f2ae8
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h
@@ -0,0 +1,119 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "feature_store.h"
+#include "field_index_remover.h"
+#include "i_field_index.h"
+#include "word_store.h"
+#include <vespa/searchlib/index/docidandfeatures.h>
+#include <vespa/searchlib/index/field_length_calculator.h>
+#include <vespa/vespalib/btree/btree.h>
+#include <vespa/vespalib/btree/btreenodeallocator.h>
+#include <vespa/vespalib/btree/btreeroot.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/memoryusage.h>
+
+namespace search::memoryindex {
+
+class IOrderedFieldIndexInserter;
+
+/**
+ * Abstract base class for implementation of memory index for a single field.
+ *
+ * Contains all components that are not dependent of the posting list format.
+ */
+class FieldIndexBase : public IFieldIndex {
+public:
+ /**
+ * Class representing a word used as key in the dictionary.
+ */
+ struct WordKey {
+ datastore::EntryRef _wordRef;
+
+ explicit WordKey(datastore::EntryRef wordRef) : _wordRef(wordRef) { }
+ WordKey() : _wordRef() { }
+
+ friend vespalib::asciistream&
+ operator<<(vespalib::asciistream& os, const WordKey& rhs);
+ };
+
+ /**
+ * Comparator class for words used in the dictionary.
+ */
+ class KeyComp {
+ private:
+ const WordStore& _wordStore;
+ const vespalib::stringref _word;
+
+ const char* getWord(datastore::EntryRef wordRef) const {
+ if (wordRef.valid()) {
+ return _wordStore.getWord(wordRef);
+ }
+ return _word.data();
+ }
+
+ public:
+ KeyComp(const WordStore& wordStore, const vespalib::stringref word)
+ : _wordStore(wordStore),
+ _word(word)
+ { }
+
+ bool operator()(const WordKey& lhs, const WordKey& rhs) const {
+ int cmpres = strcmp(getWord(lhs._wordRef), getWord(rhs._wordRef));
+ return cmpres < 0;
+ }
+ };
+
+ using PostingListPtr = uint32_t;
+ using DictionaryTree = btree::BTree<WordKey, PostingListPtr,
+ search::btree::NoAggregated,
+ const KeyComp>;
+
+protected:
+ using GenerationHandler = vespalib::GenerationHandler;
+
+ WordStore _wordStore;
+ uint64_t _numUniqueWords;
+ GenerationHandler _generationHandler;
+ DictionaryTree _dict;
+ FeatureStore _featureStore;
+ uint32_t _fieldId;
+ FieldIndexRemover _remover;
+ std::unique_ptr<IOrderedFieldIndexInserter> _inserter;
+ index::FieldLengthCalculator _calculator;
+
+ void incGeneration() {
+ _generationHandler.incGeneration();
+ }
+
+public:
+ datastore::EntryRef addWord(const vespalib::stringref word) {
+ _numUniqueWords++;
+ return _wordStore.addWord(word);
+ }
+
+ datastore::EntryRef addFeatures(const index::DocIdAndFeatures& features) {
+ return _featureStore.addFeatures(_fieldId, features).first;
+ }
+
+ FieldIndexBase(const index::Schema& schema, uint32_t fieldId);
+ FieldIndexBase(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info);
+
+ uint64_t getNumUniqueWords() const override { return _numUniqueWords; }
+ const FeatureStore& getFeatureStore() const override { return _featureStore; }
+ const WordStore& getWordStore() const override { return _wordStore; }
+ IOrderedFieldIndexInserter& getInserter() override { return *_inserter; }
+ index::FieldLengthCalculator& get_calculator() override { return _calculator; }
+
+ GenerationHandler::Guard takeGenerationGuard() override {
+ return _generationHandler.takeGuard();
+ }
+
+ DictionaryTree& getDictionaryTree() { return _dict; }
+ FieldIndexRemover& getDocumentRemover() override { return _remover; }
+
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp
index 40b1e8f360f..dc7d35a755d 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp
@@ -34,7 +34,7 @@ FieldIndexCollection::FieldIndexCollection(const Schema& schema, const IFieldLen
{
for (uint32_t fieldId = 0; fieldId < _numFields; ++fieldId) {
const auto& field = schema.getIndexField(fieldId);
- auto fieldIndex = std::make_unique<FieldIndex>(schema, fieldId, inspector.get_field_length_info(field.getName()));
+ auto fieldIndex = std::make_unique<FieldIndex<false>>(schema, fieldId, inspector.get_field_length_info(field.getName()));
_fieldIndexes.push_back(std::move(fieldIndex));
}
}
diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp
index 624ae41b990..c6524a2fc64 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp
@@ -27,7 +27,8 @@ const vespalib::string emptyWord = "";
}
-OrderedFieldIndexInserter::OrderedFieldIndexInserter(FieldIndex &fieldIndex)
+template <bool interleaved_features>
+OrderedFieldIndexInserter<interleaved_features>::OrderedFieldIndexInserter(FieldIndexType& fieldIndex)
: _word(),
_prevDocId(noDocId),
_prevAdd(false),
@@ -39,10 +40,12 @@ OrderedFieldIndexInserter::OrderedFieldIndexInserter(FieldIndex &fieldIndex)
{
}
-OrderedFieldIndexInserter::~OrderedFieldIndexInserter() = default;
+template <bool interleaved_features>
+OrderedFieldIndexInserter<interleaved_features>::~OrderedFieldIndexInserter() = default;
+template <bool interleaved_features>
void
-OrderedFieldIndexInserter::flushWord()
+OrderedFieldIndexInserter<interleaved_features>::flushWord()
{
if (_removes.empty() && _adds.empty()) {
return;
@@ -64,21 +67,24 @@ OrderedFieldIndexInserter::flushWord()
_adds.clear();
}
+template <bool interleaved_features>
void
-OrderedFieldIndexInserter::flush()
+OrderedFieldIndexInserter<interleaved_features>::flush()
{
flushWord();
_listener.flush();
}
+template <bool interleaved_features>
void
-OrderedFieldIndexInserter::commit()
+OrderedFieldIndexInserter<interleaved_features>::commit()
{
_fieldIndex.commit();
}
+template <bool interleaved_features>
void
-OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word)
+OrderedFieldIndexInserter<interleaved_features>::setNextWord(const vespalib::stringref word)
{
// TODO: Adjust here if zero length words should be legal.
assert(_word < word);
@@ -103,9 +109,10 @@ OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word)
assert(_word == wordStore.getWord(_dItr.getKey()._wordRef));
}
+template <bool interleaved_features>
void
-OrderedFieldIndexInserter::add(uint32_t docId,
- const index::DocIdAndFeatures &features)
+OrderedFieldIndexInserter<interleaved_features>::add(uint32_t docId,
+ const index::DocIdAndFeatures &features)
{
assert(docId != noDocId);
assert(_prevDocId == noDocId || _prevDocId < docId ||
@@ -117,8 +124,9 @@ OrderedFieldIndexInserter::add(uint32_t docId,
_prevAdd = true;
}
+template <bool interleaved_features>
void
-OrderedFieldIndexInserter::remove(uint32_t docId)
+OrderedFieldIndexInserter<interleaved_features>::remove(uint32_t docId)
{
assert(docId != noDocId);
assert(_prevDocId == noDocId || _prevDocId < docId);
@@ -127,8 +135,9 @@ OrderedFieldIndexInserter::remove(uint32_t docId)
_prevAdd = false;
}
+template <bool interleaved_features>
void
-OrderedFieldIndexInserter::rewind()
+OrderedFieldIndexInserter<interleaved_features>::rewind()
{
assert(_removes.empty() && _adds.empty());
_word = "";
@@ -137,10 +146,14 @@ OrderedFieldIndexInserter::rewind()
_dItr.begin();
}
+template <bool interleaved_features>
datastore::EntryRef
-OrderedFieldIndexInserter::getWordRef() const
+OrderedFieldIndexInserter<interleaved_features>::getWordRef() const
{
return _dItr.getKey()._wordRef;
}
+template
+class OrderedFieldIndexInserter<false>;
+
}
diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h
index 529b1d6d6a7..0e04b126f32 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h
@@ -18,27 +18,30 @@ class IFieldIndexInsertListener;
* and for each word updating the posting list with docId adds / removes.
*
* Insert order must be properly sorted, first by word, then by docId.
+ *
+ * The template parameter specifies whether the posting lists of the field index have interleaved features or not.
*/
+template <bool interleaved_features>
class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter {
private:
vespalib::stringref _word;
uint32_t _prevDocId;
bool _prevAdd;
- using DictionaryTree = FieldIndex::DictionaryTree;
- using PostingListStore = FieldIndex::PostingListStore;
- using KeyComp = FieldIndex::KeyComp;
- using WordKey = FieldIndex::WordKey;
- using PostingListEntryType = FieldIndex::PostingListEntryType;
- using PostingListKeyDataType = FieldIndex::PostingListKeyDataType;
- FieldIndex &_fieldIndex;
- DictionaryTree::Iterator _dItr;
+ using FieldIndexType = FieldIndex<interleaved_features>;
+ using DictionaryTree = typename FieldIndexType::DictionaryTree;
+ using PostingListStore = typename FieldIndexType::PostingListStore;
+ using KeyComp = typename FieldIndexType::KeyComp;
+ using WordKey = typename FieldIndexType::WordKey;
+ using PostingListEntryType = typename FieldIndexType::PostingListEntryType;
+ using PostingListKeyDataType = typename FieldIndexType::PostingListKeyDataType;
+ FieldIndexType& _fieldIndex;
+ typename DictionaryTree::Iterator _dItr;
IFieldIndexInsertListener &_listener;
// Pending changes to posting list for (_word)
std::vector<uint32_t> _removes;
std::vector<PostingListKeyDataType> _adds;
-
static constexpr uint32_t noFieldId = std::numeric_limits<uint32_t>::max();
static constexpr uint32_t noDocId = std::numeric_limits<uint32_t>::max();
@@ -50,7 +53,7 @@ private:
void flushWord();
public:
- OrderedFieldIndexInserter(FieldIndex &fieldIndex);
+ OrderedFieldIndexInserter(FieldIndexType& fieldIndex);
~OrderedFieldIndexInserter() override;
void setNextWord(const vespalib::stringref word) override;
void add(uint32_t docId, const index::DocIdAndFeatures &features) override;
diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp
index bad3d47b2b9..0e84c2b7968 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp
@@ -13,10 +13,11 @@ LOG_SETUP(".searchlib.memoryindex.posting_iterator");
namespace search::memoryindex {
-PostingIterator::PostingIterator(FieldIndex::PostingList::ConstIterator itr,
- const FeatureStore & featureStore,
- uint32_t packedIndex,
- const fef::TermFieldMatchDataArray & matchData) :
+template <bool interleaved_features>
+PostingIterator<interleaved_features>::PostingIterator(PostingListIteratorType itr,
+ const FeatureStore& featureStore,
+ uint32_t packedIndex,
+ const fef::TermFieldMatchDataArray& matchData) :
queryeval::RankedSearchIteratorBase(matchData),
_itr(itr),
_featureStore(featureStore),
@@ -25,10 +26,12 @@ PostingIterator::PostingIterator(FieldIndex::PostingList::ConstIterator itr,
_featureStore.setupForField(packedIndex, _featureDecoder);
}
-PostingIterator::~PostingIterator() {}
+template <bool interleaved_features>
+PostingIterator<interleaved_features>::~PostingIterator() = default;
+template <bool interleaved_features>
void
-PostingIterator::initRange(uint32_t begin, uint32_t end)
+PostingIterator<interleaved_features>::initRange(uint32_t begin, uint32_t end)
{
SearchIterator::initRange(begin, end);
_itr.lower_bound(begin);
@@ -40,8 +43,9 @@ PostingIterator::initRange(uint32_t begin, uint32_t end)
clearUnpacked();
}
+template <bool interleaved_features>
void
-PostingIterator::doSeek(uint32_t docId)
+PostingIterator<interleaved_features>::doSeek(uint32_t docId)
{
if (getUnpacked()) {
clearUnpacked();
@@ -54,8 +58,9 @@ PostingIterator::doSeek(uint32_t docId)
}
}
+template <bool interleaved_features>
void
-PostingIterator::doUnpack(uint32_t docId)
+PostingIterator<interleaved_features>::doUnpack(uint32_t docId)
{
if (!_matchData.valid() || getUnpacked()) {
return;
@@ -69,6 +74,9 @@ PostingIterator::doUnpack(uint32_t docId)
setUnpacked();
}
+template
+class PostingIterator<false>;
+
}
namespace search::btree {
diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h
index de337ef49f3..f029c837cf7 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h
@@ -9,10 +9,15 @@ namespace search::memoryindex {
/**
* Search iterator for memory field index posting list.
+ *
+ * The template parameter specifies whether the wrapped posting list has interleaved features or not.
*/
+template <bool interleaved_features>
class PostingIterator : public queryeval::RankedSearchIteratorBase {
private:
- FieldIndex::PostingList::ConstIterator _itr;
+ using FieldIndexType = FieldIndex<interleaved_features>;
+ using PostingListIteratorType = typename FieldIndexType::PostingList::ConstIterator;
+ PostingListIteratorType _itr;
const FeatureStore &_featureStore;
FeatureStore::DecodeContextCooked _featureDecoder;
@@ -25,7 +30,7 @@ public:
* @param packedIndex the field or field collection owning features.
* @param matchData the match data to unpack features into.
**/
- PostingIterator(FieldIndex::PostingList::ConstIterator itr,
+ PostingIterator(PostingListIteratorType itr,
const FeatureStore &featureStore,
uint32_t packedIndex,
const fef::TermFieldMatchDataArray &matchData);
diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h b/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h
index b3f488d4ed7..373de21e836 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h
@@ -9,7 +9,7 @@ namespace search::memoryindex {
/**
* Entry per document in memory index posting list.
*/
-template <bool has_interleaved_features>
+template <bool interleaved_features>
class PostingListEntry {
mutable datastore::EntryRef _features; // reference to compressed features
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp
index d3d9e5e0bcb..a4996c931e2 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp
@@ -129,10 +129,10 @@ search::queryeval::SearchIterator *
FakeMemTreeOcc::
createIterator(const fef::TermFieldMatchDataArray &matchData) const
{
- return new search::memoryindex::PostingIterator(_tree.begin(_allocator),
- _mgr._featureStore,
- _packedIndex,
- matchData);
+ return new search::memoryindex::PostingIterator<false>(_tree.begin(_allocator),
+ _mgr._featureStore,
+ _packedIndex,
+ matchData);
}
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h
index 599d2b37ab2..69114611fe6 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h
@@ -13,8 +13,9 @@ namespace search::fakedata {
class FakeMemTreeOccMgr : public FakeWord::RandomizedWriter {
public:
- using Tree = memoryindex::FieldIndex::PostingList;
- using PostingListEntryType = memoryindex::FieldIndex::PostingListEntryType;
+ // TODO: Create implementation for "interleaved features" posting list as well.
+ using Tree = memoryindex::FieldIndex<false>::PostingList;
+ using PostingListEntryType = memoryindex::FieldIndex<false>::PostingListEntryType;
using NodeAllocator = Tree::NodeAllocatorType;
using FeatureStore = memoryindex::FeatureStore;
using EntryRef = datastore::EntryRef;
diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h
index 647f624daea..268bf834d21 100644
--- a/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h
+++ b/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h
@@ -20,7 +20,7 @@ public:
{
}
- WrapInserter(FieldIndex& field_index)
+ WrapInserter(IFieldIndex& field_index)
: _inserter(field_index.getInserter())
{
}