diff options
Diffstat (limited to 'searchlib')
9 files changed, 46 insertions, 22 deletions
diff --git a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp index fa49e1ba3af..7e24511435b 100644 --- a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp +++ b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp @@ -76,13 +76,22 @@ public: return SearchIterator::UP(_fp->createIterator(_tfmda)); } private: - mutable TermFieldMatchDataArray _tfmda; + TermFieldMatchData _tfmd; + TermFieldMatchDataArray _tfmda; FakePosting::SP _fp; }; Verifier::Verifier(FakePosting::SP fp) - : _fp(std::move(fp)) -{ } + : _tfmd(), + _tfmda(), + _fp(std::move(fp)) +{ + if (_fp) { + _tfmd.setNeedNormalFeatures(_fp->enable_unpack_normal_features()); + _tfmd.setNeedCheapFeatures(_fp->enable_unpack_cheap_features()); + } + _tfmda.add(&_tfmd); +} Verifier::~Verifier() = default; diff --git a/searchlib/src/tests/postinglistbm/posting_list_test.cpp b/searchlib/src/tests/postinglistbm/posting_list_test.cpp index a464e019bac..2022ffb1b8e 100644 --- a/searchlib/src/tests/postinglistbm/posting_list_test.cpp +++ b/searchlib/src/tests/postinglistbm/posting_list_test.cpp @@ -25,6 +25,8 @@ validate_posting_list_for_word(const FakePosting& posting, const FakeWord& word) TermFieldMatchDataArray tfmda; tfmda.add(&md); + md.setNeedNormalFeatures(posting.enable_unpack_normal_features()); + md.setNeedCheapFeatures(posting.enable_unpack_cheap_features()); std::unique_ptr<SearchIterator> iterator(posting.createIterator(tfmda)); if (posting.hasWordPositions()) { word.validate(iterator.get(), tfmda, posting.enable_unpack_normal_features(), posting.has_cheap_features() && posting.enable_unpack_cheap_features(), false); diff --git a/searchlib/src/tests/postinglistbm/stress_runner.cpp b/searchlib/src/tests/postinglistbm/stress_runner.cpp index b084e4bd49d..5353008965d 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.cpp +++ b/searchlib/src/tests/postinglistbm/stress_runner.cpp @@ -214,6 +214,8 @@ makeSomePostings(FPFactory *postingFactory, TermFieldMatchDataArray tfmda; tfmda.add(&md); + md.setNeedNormalFeatures(posting->enable_unpack_normal_features()); + md.setNeedCheapFeatures(posting->enable_unpack_cheap_features()); std::unique_ptr<SearchIterator> iterator(posting->createIterator(tfmda)); if (posting->hasWordPositions()) { if (stride != 0) { diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp index 68056720f2a..6a3ea9254d2 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "zcposocciterators.h" +#include <vespa/searchlib/fef/termfieldmatchdata.h> #include "zc4_posting_params.h" namespace search::diskindex { @@ -75,19 +76,8 @@ create_zc_posocc_iterator(const PostingListCounts &counts, bitcompression::Posit std::unique_ptr<search::queryeval::SearchIterator> create_zc_posocc_iterator(bool bigEndian, const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data) { - bool unpack_normal_features = true; - bool unpack_cheap_features = true; - if (bigEndian) { - return create_zc_posocc_iterator<true>(counts, start, bit_length, posting_params, fields_params, match_data, unpack_normal_features, unpack_cheap_features); - } else { - return create_zc_posocc_iterator<false>(counts, start, bit_length, posting_params, fields_params, match_data, unpack_normal_features, unpack_cheap_features); - } -} - -// Temporarily for unit testing. -std::unique_ptr<search::queryeval::SearchIterator> -create_zc_posocc_iterator(bool bigEndian, const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data, bool unpack_normal_features, bool unpack_cheap_features) -{ + bool unpack_normal_features = match_data.valid() ? match_data[0]->needs_normal_features() : false; + bool unpack_cheap_features = match_data.valid() ? match_data[0]->needs_cheap_features() : false; if (bigEndian) { return create_zc_posocc_iterator<true>(counts, start, bit_length, posting_params, fields_params, match_data, unpack_normal_features, unpack_cheap_features); } else { diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h index 6b26cb22ba8..47c5455f259 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h @@ -48,10 +48,6 @@ public: std::unique_ptr<search::queryeval::SearchIterator> create_zc_posocc_iterator(bool bigEndian, const index::PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data); -// Temporarily for unit testing. -std::unique_ptr<search::queryeval::SearchIterator> -create_zc_posocc_iterator(bool bigEndian, const index::PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data, bool unpack_normal_features, bool unpack_cheap_features); - extern template class ZcRareWordPosOccIterator<false, false>; extern template class ZcRareWordPosOccIterator<false, true>; extern template class ZcRareWordPosOccIterator<true, false>; diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp index 2b7b67e8365..4de84866edf 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp @@ -20,7 +20,7 @@ TermFieldMatchData::TermFieldMatchData() : TermFieldMatchData::TermFieldMatchData(const TermFieldMatchData & rhs) : _docId(rhs._docId), _fieldId(rhs._fieldId), - _flags(UNPACK_ALL_FEATURES_MASK), + _flags(rhs._flags), _sz(0), _numOccs(0), _fieldLength(0) diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index b2e3b2e6ba3..cf22554a4ed 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -20,6 +20,20 @@ using search::index::Schema; namespace search::memoryindex { +namespace { + +void set_cheap_features(DocIdAndFeatures &features) +{ + // Set cheap features based on normal features. + // TODO: Update when proper cheap features are present in memory index. + assert(!features.elements().empty()); + const auto &element = features.elements().front(); + features.set_field_length(element.getElementLen()); + features.set_num_occs(element.getNumOccs()); +} + +} + using datastore::EntryRef; vespalib::asciistream & @@ -172,6 +186,7 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) _featureStore.setupForReadFeatures(featureRef, decoder); decoder.readFeatures(features); features.set_doc_id(docId); + set_cheap_features(features); indexBuilder.add_document(features); } } else { @@ -184,6 +199,7 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) _featureStore.setupForReadFeatures(featureRef, decoder); decoder.readFeatures(features); features.set_doc_id(docId); + set_cheap_features(features); indexBuilder.add_document(features); } } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp index ef4edf86c59..7892f9fa70b 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp @@ -31,6 +31,8 @@ public: _itr() { _tfmda.add(&_md); + _md.setNeedNormalFeatures(posting.enable_unpack_normal_features()); + _md.setNeedCheapFeatures(posting.enable_unpack_cheap_features()); _itr.reset(posting.createIterator(_tfmda)); } ~IteratorState() {} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp index 237f9f2b242..ac2c0203ac8 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp @@ -781,7 +781,14 @@ SearchIterator * FakeZc4SkipPosOcc<bigEndian>:: createIterator(const TermFieldMatchDataArray &matchData) const { - return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData, _unpack_normal_features, _unpack_cheap_features).release(); + if (matchData.valid()) { + assert(_unpack_normal_features == matchData[0]->needs_normal_features()); + assert(_unpack_cheap_features == matchData[0]->needs_cheap_features()); + } else { + assert(!_unpack_normal_features); + assert(!_unpack_cheap_features); + } +return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData).release(); } template <bool bigEndian> |