Merge pull request #9563 from vespa-engine/toregge/use-match-data-tagging-when-constructing-iterator

Use match data tagging when constructing search iterator

9 files changed, 46 insertions, 22 deletions

diff --git a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp
index fa49e1ba3af..7e24511435b 100644
--- a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp
+++ b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp
@@ -76,13 +76,22 @@ public:
         return SearchIterator::UP(_fp->createIterator(_tfmda));
     }
 private:
-    mutable TermFieldMatchDataArray _tfmda;
+    TermFieldMatchData _tfmd;
+    TermFieldMatchDataArray _tfmda;
     FakePosting::SP _fp;
 };
 
 Verifier::Verifier(FakePosting::SP fp)
-    : _fp(std::move(fp))
-{ }
+    : _tfmd(),
+      _tfmda(),
+      _fp(std::move(fp))
+{
+    if (_fp) {
+        _tfmd.setNeedNormalFeatures(_fp->enable_unpack_normal_features());
+        _tfmd.setNeedCheapFeatures(_fp->enable_unpack_cheap_features());
+    }
+    _tfmda.add(&_tfmd);
+}
 
 Verifier::~Verifier() = default;
 
diff --git a/searchlib/src/tests/postinglistbm/posting_list_test.cpp b/searchlib/src/tests/postinglistbm/posting_list_test.cpp
index a464e019bac..2022ffb1b8e 100644
--- a/searchlib/src/tests/postinglistbm/posting_list_test.cpp
+++ b/searchlib/src/tests/postinglistbm/posting_list_test.cpp
@@ -25,6 +25,8 @@ validate_posting_list_for_word(const FakePosting& posting, const FakeWord& word)
     TermFieldMatchDataArray tfmda;
     tfmda.add(&md);
 
+    md.setNeedNormalFeatures(posting.enable_unpack_normal_features());
+    md.setNeedCheapFeatures(posting.enable_unpack_cheap_features());
     std::unique_ptr<SearchIterator> iterator(posting.createIterator(tfmda));
     if (posting.hasWordPositions()) {
         word.validate(iterator.get(), tfmda, posting.enable_unpack_normal_features(), posting.has_cheap_features() && posting.enable_unpack_cheap_features(), false);
diff --git a/searchlib/src/tests/postinglistbm/stress_runner.cpp b/searchlib/src/tests/postinglistbm/stress_runner.cpp
index b084e4bd49d..5353008965d 100644
--- a/searchlib/src/tests/postinglistbm/stress_runner.cpp
+++ b/searchlib/src/tests/postinglistbm/stress_runner.cpp
@@ -214,6 +214,8 @@ makeSomePostings(FPFactory *postingFactory,
             TermFieldMatchDataArray tfmda;
             tfmda.add(&md);
 
+            md.setNeedNormalFeatures(posting->enable_unpack_normal_features());
+            md.setNeedCheapFeatures(posting->enable_unpack_cheap_features());
             std::unique_ptr<SearchIterator> iterator(posting->createIterator(tfmda));
             if (posting->hasWordPositions()) {
                 if (stride != 0) {
diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp
index 68056720f2a..6a3ea9254d2 100644
--- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "zcposocciterators.h"
+#include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include "zc4_posting_params.h"
 
 namespace search::diskindex {
@@ -75,19 +76,8 @@ create_zc_posocc_iterator(const PostingListCounts &counts, bitcompression::Posit
 std::unique_ptr<search::queryeval::SearchIterator>
 create_zc_posocc_iterator(bool bigEndian, const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data)
 {
-    bool unpack_normal_features = true;
-    bool unpack_cheap_features = true;
-    if (bigEndian) {
-        return create_zc_posocc_iterator<true>(counts, start, bit_length, posting_params, fields_params, match_data, unpack_normal_features, unpack_cheap_features);
-    } else {
-        return create_zc_posocc_iterator<false>(counts, start, bit_length, posting_params, fields_params, match_data, unpack_normal_features, unpack_cheap_features);
-    }
-}
-
-// Temporarily for unit testing.
-std::unique_ptr<search::queryeval::SearchIterator>
-create_zc_posocc_iterator(bool bigEndian, const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data, bool unpack_normal_features, bool unpack_cheap_features)
-{
+    bool unpack_normal_features = match_data.valid() ? match_data[0]->needs_normal_features() : false;
+    bool unpack_cheap_features = match_data.valid() ? match_data[0]->needs_cheap_features() : false;
     if (bigEndian) {
         return create_zc_posocc_iterator<true>(counts, start, bit_length, posting_params, fields_params, match_data, unpack_normal_features, unpack_cheap_features);
     } else {
diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h
index 6b26cb22ba8..47c5455f259 100644
--- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h
+++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h
@@ -48,10 +48,6 @@ public:
 std::unique_ptr<search::queryeval::SearchIterator>
 create_zc_posocc_iterator(bool bigEndian, const index::PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data);
 
-// Temporarily for unit testing.
-std::unique_ptr<search::queryeval::SearchIterator>
-create_zc_posocc_iterator(bool bigEndian, const index::PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data, bool unpack_normal_features, bool unpack_cheap_features);
-
 extern template class ZcRareWordPosOccIterator<false, false>;
 extern template class ZcRareWordPosOccIterator<false, true>;
 extern template class ZcRareWordPosOccIterator<true, false>;
diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
index 2b7b67e8365..4de84866edf 100644
--- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
+++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
@@ -20,7 +20,7 @@ TermFieldMatchData::TermFieldMatchData() :
 TermFieldMatchData::TermFieldMatchData(const TermFieldMatchData & rhs) :
     _docId(rhs._docId),
     _fieldId(rhs._fieldId),
-    _flags(UNPACK_ALL_FEATURES_MASK),
+    _flags(rhs._flags),
     _sz(0),
     _numOccs(0),
     _fieldLength(0)
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
index b2e3b2e6ba3..cf22554a4ed 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
@@ -20,6 +20,20 @@ using search::index::Schema;
 
 namespace search::memoryindex {
 
+namespace {
+
+void set_cheap_features(DocIdAndFeatures &features)
+{
+    // Set cheap features based on normal features.
+    // TODO: Update when proper cheap features are present in memory index.
+    assert(!features.elements().empty());
+    const auto &element = features.elements().front();
+    features.set_field_length(element.getElementLen());
+    features.set_num_occs(element.getNumOccs());
+}
+
+}
+
 using datastore::EntryRef;
 
 vespalib::asciistream &
@@ -172,6 +186,7 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder)
                 _featureStore.setupForReadFeatures(featureRef, decoder);
                 decoder.readFeatures(features);
                 features.set_doc_id(docId);
+                set_cheap_features(features);
                 indexBuilder.add_document(features);
             }
         } else {
@@ -184,6 +199,7 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder)
                 _featureStore.setupForReadFeatures(featureRef, decoder);
                 decoder.readFeatures(features);
                 features.set_doc_id(docId);
+                set_cheap_features(features);
                 indexBuilder.add_document(features);
             }
         }
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp
index ef4edf86c59..7892f9fa70b 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp
@@ -31,6 +31,8 @@ public:
           _itr()
     {
         _tfmda.add(&_md);
+        _md.setNeedNormalFeatures(posting.enable_unpack_normal_features());
+        _md.setNeedCheapFeatures(posting.enable_unpack_cheap_features());
         _itr.reset(posting.createIterator(_tfmda));
     }
     ~IteratorState() {}
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp
index 237f9f2b242..ac2c0203ac8 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp
@@ -781,7 +781,14 @@ SearchIterator *
 FakeZc4SkipPosOcc<bigEndian>::
 createIterator(const TermFieldMatchDataArray &matchData) const
 {
-    return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData, _unpack_normal_features, _unpack_cheap_features).release();
+    if (matchData.valid()) {
+        assert(_unpack_normal_features == matchData[0]->needs_normal_features());
+        assert(_unpack_cheap_features == matchData[0]->needs_cheap_features());
+    } else {
+        assert(!_unpack_normal_features);
+        assert(!_unpack_cheap_features);
+    }
+return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData).release();
 }
 
 template <bool bigEndian>