Revert "Revert "Balder/do not use your own docidlimit""

11 files changed, 42 insertions, 75 deletions

diff --git a/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h
index ba1193bc065..2965e3796bf 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h
@@ -6,8 +6,7 @@
 
 #include <memory>
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 /**
  * Interface used to expose searchable data to the matching
@@ -64,5 +63,4 @@ public:
     virtual ~ISearchContext() {}
 };
 
-} // namespace matching
-} // namespace proton
+}
diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp
index 7ff9c59bc09..c00ab1270e3 100644
--- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp
+++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp
@@ -412,7 +412,7 @@ BitVectorTest::checkSearch(AttributePtr v,
                            bool checkStride)
 {
     (void) checkStride;
-    sb->initFullRange();
+    sb->initRange(1, v->getCommittedDocIdLimit());
     sb->seek(1u);
     uint32_t docId = sb->getDocId();
     uint32_t lastDocId = 0;
@@ -497,7 +497,7 @@ BitVectorTest::test(BasicType bt,
         if (!enableOnlyBitVector) {
             checkSearch(v, std::move(dwsi), md, 2, 1022, 205, !filter, true);
         } else {
-            dwsi->initFullRange();
+            dwsi->initRange(1, v->getCommittedDocIdLimit());
             EXPECT_TRUE(dwsi->isAtEnd());
         }
     }
diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
index 3688e3f8d0f..332892aeaea 100644
--- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
+++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
@@ -232,6 +232,7 @@ TEST_F("createSearchContext() returns an imported search context", SingleStringA
     // Iterator specifics are tested in imported_search_context_test, so just make sure
     // we get the expected iterator functionality. In this case, a non-strict iterator.
     auto iter = ctx->createIterator(&match, false);
+    iter->initRange(1, f.imported_attr->getNumDocs());
     EXPECT_FALSE(iter->seek(DocId(1)));
     EXPECT_FALSE(iter->seek(DocId(2)));
     EXPECT_FALSE(iter->seek(DocId(3)));
diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
index c0b36dc1441..5811ca9cc4d 100644
--- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
+++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
@@ -5,8 +5,7 @@
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include <vespa/searchcommon/attribute/search_context_params.h>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 using fef::TermFieldMatchData;
 using vespalib::Trinary;
@@ -22,7 +21,7 @@ struct Fixture : ImportedAttributeFixture {
             bool strict) {
         auto iter = ctx.createIterator(&match, strict);
         assert(iter.get() != nullptr);
-        iter->initRange(DocId(1), reference_attr->getNumDocs() + 1);
+        iter->initRange(DocId(1), reference_attr->getNumDocs());
         return iter;
     }
 
@@ -333,7 +332,6 @@ TEST_F("queryTerm() returns term context was created with", WsetValueFixture) {
     EXPECT_EQUAL(std::string("helloworld"), std::string(ctx->queryTerm().getTerm()));
 }
 
-} // attribute
-} // search
+}
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp
index 903807d2b0b..a6e06af1b71 100644
--- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp
+++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp
@@ -203,7 +203,7 @@ Result do_search(IAttributeManager &attribute_manager, const Node &node, bool st
     bp->fetchPostings(strict);
     SearchIterator::UP iterator = bp->createSearch(*match_data, strict);
     ASSERT_TRUE(iterator.get() != nullptr);
-    iterator->initFullRange();
+    iterator->initRange(1, num_docs);
     extract_posting_info(result, iterator->getPostingInfo());
     extract_wand_params(result, dynamic_cast<ParallelWeakAndSearch*>(iterator.get()));
     result.iterator_dump = iterator->asString();
diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp
index 6fa87741f1d..c22807f8bf5 100644
--- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp
+++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp
@@ -3,7 +3,6 @@
 
 #include <vespa/searchlib/attribute/attribute_blueprint_factory.h>
 #include <vespa/searchlib/attribute/attribute_weighted_set_blueprint.h>
-#include <vespa/searchlib/attribute/iattributemanager.h>
 #include <vespa/searchlib/attribute/attributecontext.h>
 #include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/searchlib/attribute/extendableattributes.h>
@@ -11,18 +10,12 @@
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/query/tree/simplequery.h>
-#include <vespa/searchlib/queryeval/field_spec.h>
-#include <vespa/searchlib/queryeval/searchiterator.h>
-#include <vespa/searchlib/queryeval/blueprint.h>
 #include <vespa/searchlib/queryeval/fake_result.h>
 #include <vespa/searchlib/queryeval/weighted_set_term_search.h>
 #include <vespa/searchlib/queryeval/fake_requestcontext.h>
-#include <memory>
-#include <string>
-#include <map>
+
 
 #include <vespa/searchlib/attribute/enumstore.hpp>
-#include <vespa/searchlib/attribute/singlestringattribute.h>
 
 using namespace search;
 using namespace search::query;
@@ -169,7 +162,7 @@ struct WS {
         bp->fetchPostings(strict);
         SearchIterator::UP sb = bp->createSearch(*md, strict);
         FakeResult result;
-        sb->initFullRange();
+        sb->initRange(1, 10);
         for (uint32_t docId = 1; docId < 10; ++docId) {
             if (sb->seek(docId)) {
                 sb->unpack(docId);
diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp
index a79c35632be..20e5b592501 100644
--- a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp
+++ b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp
@@ -432,7 +432,7 @@ ResultSetPtr
 SearchContextTest::performSearch(SearchIterator & sb, uint32_t numDocs)
 {
     HitCollector hc(numDocs, numDocs, 0);
-    sb.initFullRange();
+    sb.initRange(1, numDocs);
     // assume strict toplevel search object located at start
     for (sb.seek(1u); ! sb.isAtEnd(); sb.seek(sb.getDocId() + 1)) {
         hc.addHit(sb.getDocId(), 0.0);
@@ -723,7 +723,7 @@ SearchContextTest::testStrictSearchIterator(SearchContext & threeHits,
     { // search for value with 3 hits
         threeHits.fetchPostings(true);
         SearchBasePtr sb = threeHits.createIterator(&dummy, true);
-        sb->initFullRange();
+        sb->initRange(1, threeHits.attribute().getCommittedDocIdLimit());
         EXPECT_TRUE(typeTester.matches(*sb));
         EXPECT_TRUE(sb->getDocId() == sb->beginId() ||
                     sb->getDocId() == 1u);
@@ -744,7 +744,7 @@ SearchContextTest::testStrictSearchIterator(SearchContext & threeHits,
     { // search for value with no hits
         noHits.fetchPostings(true);
         SearchBasePtr sb = noHits.createIterator(&dummy, true);
-        sb->initFullRange();
+        sb->initRange(1, noHits.attribute().getCommittedDocIdLimit());
         ASSERT_TRUE(typeTester.matches(*sb));
         EXPECT_TRUE(sb->getDocId() == sb->beginId() ||
                    sb->isAtEnd());
diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp
index 79f45a2a700..92c0fe5d37c 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp
@@ -21,25 +21,16 @@ AttributeIteratorBase::visitMembers(vespalib::ObjectVisitor &visitor) const
     visit(visitor, "tfmd.docId", _matchData->getDocId());
 }
 
-FilterAttributeIterator::FilterAttributeIterator(fef::TermFieldMatchData * matchData, uint32_t docIdLimit)
-    : AttributeIteratorBase(matchData),
-      _docIdLimit(docIdLimit)
+FilterAttributeIterator::FilterAttributeIterator(fef::TermFieldMatchData * matchData)
+    : AttributeIteratorBase(matchData)
 {
     _matchPosition->setElementWeight(1);
 }
 
 void
-FilterAttributeIterator::visitMembers(vespalib::ObjectVisitor &visitor) const
-{
-    AttributeIteratorBase::visitMembers(visitor);
-    visit(visitor, "docIdLimit", _docIdLimit);
-}
-
-void
 AttributeIterator::visitMembers(vespalib::ObjectVisitor &visitor) const
 {
     AttributeIteratorBase::visitMembers(visitor);
-    visit(visitor, "docIdLimit", _docIdLimit);
     visit(visitor, "weight", _weight);
 }
 
@@ -53,7 +44,6 @@ FlagAttributeIterator::doUnpack(uint32_t docId)
 AttributePostingListIterator:: AttributePostingListIterator(bool hasWeight, TermFieldMatchData *matchData)
     : AttributeIteratorBase(matchData),
       _hasWeight(hasWeight)
-      // _hasWeight(_searchContext.attribute().hasWeightedSetType())
 {
 }
 
diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h
index a7ec31c1f37..09db5c334db 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h
@@ -49,26 +49,22 @@ public:
 class AttributeIterator : public AttributeIteratorBase
 {
 public:
-    AttributeIterator(fef::TermFieldMatchData * matchData, uint32_t docIdLimit)
+    AttributeIterator(fef::TermFieldMatchData * matchData)
         : AttributeIteratorBase(matchData),
-          _docIdLimit(docIdLimit),
           _weight(1)
     { }
 protected:
     void visitMembers(vespalib::ObjectVisitor &visitor) const override;
     void doUnpack(uint32_t docId) override;
-    uint32_t   _docIdLimit;
     int32_t    _weight;
 };
 
 class FilterAttributeIterator : public AttributeIteratorBase
 {
 public:
-    FilterAttributeIterator(fef::TermFieldMatchData * matchData, uint32_t docIdLimit);
+    FilterAttributeIterator(fef::TermFieldMatchData * matchData);
 protected:
-    void visitMembers(vespalib::ObjectVisitor &visitor) const override;
     void doUnpack(uint32_t docId) override;
-    uint32_t   _docIdLimit;
 };
 
 template <typename SC>
@@ -89,7 +85,6 @@ public:
     bool seekFast(uint32_t docId) const { return _searchContext.cmp(docId); }
 };
 
-
 template <typename SC>
 class FilterAttributeIteratorT : public FilterAttributeIterator
 {
@@ -121,10 +116,10 @@ template <typename SC>
 class AttributeIteratorStrict : public AttributeIteratorT<SC>
 {
 private:
-    using AttributeIteratorT<SC>::_docIdLimit;
     using AttributeIteratorT<SC>::_searchContext;
     using AttributeIteratorT<SC>::setDocId;
     using AttributeIteratorT<SC>::setAtEnd;
+    using AttributeIteratorT<SC>::isAtEnd;
     using AttributeIteratorT<SC>::_weight;
     using Trinary=vespalib::Trinary;
     void doSeek(uint32_t docId) override;
@@ -140,10 +135,10 @@ template <typename SC>
 class FilterAttributeIteratorStrict : public FilterAttributeIteratorT<SC>
 {
 private:
-    using FilterAttributeIteratorT<SC>::_docIdLimit;
     using FilterAttributeIteratorT<SC>::_searchContext;
     using FilterAttributeIteratorT<SC>::setDocId;
     using FilterAttributeIteratorT<SC>::setAtEnd;
+    using FilterAttributeIteratorT<SC>::isAtEnd;
     using Trinary=vespalib::Trinary;
     void doSeek(uint32_t docId) override;
     Trinary is_strict() const override { return Trinary::True; }
@@ -330,11 +325,11 @@ template <typename SC>
 class FlagAttributeIteratorT : public FlagAttributeIterator
 {
 private:
+    using Attribute = typename SC::Attribute;
     void doSeek(uint32_t docId) override;
 
 protected:
     const SC & _sc;
-    uint32_t   _docIdLimit;
 
     void or_hits_into(BitVector &result, uint32_t begin_id) override;
     void and_hits_into(BitVector &result, uint32_t begin_id) override;
@@ -343,9 +338,7 @@ protected:
 public:
     FlagAttributeIteratorT(const SC &sc, fef::TermFieldMatchData * matchData)
         : FlagAttributeIterator(matchData),
-          _sc(sc),
-          _docIdLimit(static_cast<const typename SC::Attribute &>
-                      (sc.attribute()).getCommittedDocIdLimit())
+          _sc(sc)
     { }
 
     void initRange(uint32_t begin, uint32_t end) override {
@@ -361,10 +354,11 @@ template <typename SC>
 class FlagAttributeIteratorStrict : public FlagAttributeIteratorT<SC>
 {
 private:
-    using FlagAttributeIteratorT<SC>::_docIdLimit;
     using FlagAttributeIteratorT<SC>::_sc;
     using FlagAttributeIteratorT<SC>::setDocId;
     using FlagAttributeIteratorT<SC>::setAtEnd;
+    using FlagAttributeIteratorT<SC>::isAtEnd;
+    using Attribute = typename SC::Attribute;
     using Trinary=vespalib::Trinary;
     void doSeek(uint32_t docId) override;
     Trinary is_strict() const override { return Trinary::True; }
@@ -376,4 +370,3 @@ public:
 };
 
 } // namespace search
-
diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp
index 3c94283e867..87c2cc3e800 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp
@@ -62,7 +62,6 @@ void AttributePostingListIteratorT<PL>::initRange(uint32_t begin, uint32_t end)
     }
 }
 
-
 template <typename PL>
 template<typename... Args>
 FilterAttributePostingListIteratorT<PL>::
@@ -208,15 +207,15 @@ FilterAttributeIteratorT<SC>::visitMembers(vespalib::ObjectVisitor &visitor) con
 
 template <typename SC>
 AttributeIteratorT<SC>::AttributeIteratorT(const SC &searchContext, fef::TermFieldMatchData *matchData)
-        : AttributeIterator(matchData, searchContext.attribute().getCommittedDocIdLimit()),
-          _searchContext(searchContext)
+    : AttributeIterator(matchData),
+      _searchContext(searchContext)
 { }
 
 
 template <typename SC>
 FilterAttributeIteratorT<SC>::FilterAttributeIteratorT(const SC &searchContext, fef::TermFieldMatchData *matchData)
-        : FilterAttributeIterator(matchData, searchContext._attr.getCommittedDocIdLimit()),
-          _searchContext(searchContext)
+    : FilterAttributeIterator(matchData),
+      _searchContext(searchContext)
 { }
 
 
@@ -225,11 +224,10 @@ void
 FlagAttributeIteratorStrict<SC>::doSeek(uint32_t docId)
 {
     const SC & sc(_sc);
-    const typename SC::Attribute &attr =
-            static_cast<const typename SC::Attribute &>(sc.attribute());
+    const Attribute &attr = static_cast<const Attribute &>(sc.attribute());
     for (int i = sc._low; (i <= sc._high); ++i) {
         const BitVector * bv = attr.getBitVector(i);
-        if ((bv != NULL) && docId < _docIdLimit && bv->testBit(docId)) {
+        if ((bv != NULL) && !isAtEnd(docId) && bv->testBit(docId)) {
             setDocId(docId);
             return;
         }
@@ -238,12 +236,12 @@ FlagAttributeIteratorStrict<SC>::doSeek(uint32_t docId)
     uint32_t minNextBit(search::endDocId);
     for (int i = sc._low; (i <= sc._high); ++i) {
         const BitVector * bv = attr.getBitVector(i);
-        if (bv != NULL && docId < _docIdLimit) {
+        if (bv != NULL && !isAtEnd(docId)) {
             uint32_t nextBit = bv->getNextTrueBit(docId);
             minNextBit = std::min(nextBit, minNextBit);
         }
     }
-    if (minNextBit < _docIdLimit) {
+    if (!isAtEnd(minNextBit)) {
         setDocId(minNextBit);
     } else {
         setAtEnd();
@@ -255,11 +253,10 @@ void
 FlagAttributeIteratorT<SC>::doSeek(uint32_t docId)
 {
     const SC & sc(_sc);
-    const typename SC::Attribute &attr =
-            static_cast<const typename SC::Attribute &>(sc.attribute());
+    const Attribute &attr = static_cast<const Attribute &>(sc.attribute());
     for (int i = sc._low; (i <= sc._high); ++i) {
         const BitVector * bv = attr.getBitVector(i);
-        if ((bv != NULL) && docId < _docIdLimit && bv->testBit(docId)) {
+        if ((bv != NULL) && !isAtEnd(docId) && bv->testBit(docId)) {
             setDocId(docId);
             return;
         }
@@ -271,7 +268,7 @@ void
 FlagAttributeIteratorT<SC>::or_hits_into(BitVector &result, uint32_t begin_id) {
     (void) begin_id;
     const SC & sc(_sc);
-    const typename SC::Attribute &attr = static_cast<const typename SC::Attribute &>(sc.attribute());
+    const Attribute &attr = static_cast<const Attribute &>(sc.attribute());
     for (int i = sc._low; (i <= sc._high); ++i) {
         const BitVector * bv = attr.getBitVector(i);
         if (bv != NULL) {
@@ -284,7 +281,7 @@ template <typename SC>
 void
 FlagAttributeIteratorT<SC>::and_hits_into(BitVector &result, uint32_t begin_id) {
     const SC & sc(_sc);
-    const typename SC::Attribute &attr = static_cast<const typename SC::Attribute &>(sc.attribute());
+    const Attribute &attr = static_cast<const Attribute &>(sc.attribute());
     if (sc._low == sc._high) {
         const BitVector * bv = attr.getBitVector(sc._low);
         if (bv != NULL) {
@@ -303,7 +300,7 @@ template <typename SC>
 std::unique_ptr<BitVector>
 FlagAttributeIteratorT<SC>::get_hits(uint32_t begin_id) {
     const SC & sc(_sc);
-    const typename SC::Attribute &attr = static_cast<const typename SC::Attribute &>(sc.attribute());
+    const Attribute &attr = static_cast<const Attribute &>(sc.attribute());
     int i = sc._low;
     BitVector::UP result;
     for (;!result && i < sc._high; ++i) {
@@ -332,7 +329,7 @@ template <typename SC>
 void
 AttributeIteratorT<SC>::doSeek(uint32_t docId)
 {
-    if (__builtin_expect(docId >= _docIdLimit, false)) {
+    if (isAtEnd(docId)) {
         setAtEnd();
     } else if (_searchContext.cmp(docId, _weight)) {
         setDocId(docId);
@@ -343,7 +340,7 @@ template <typename SC>
 void
 FilterAttributeIteratorT<SC>::doSeek(uint32_t docId)
 {
-    if (__builtin_expect(docId >= _docIdLimit, false)) {
+    if (isAtEnd(docId)) {
         setAtEnd();
     } else if (_searchContext.cmp(docId)) {
         setDocId(docId);
@@ -354,7 +351,7 @@ template <typename SC>
 void
 AttributeIteratorStrict<SC>::doSeek(uint32_t docId)
 {
-    for (uint32_t nextId = docId; nextId < _docIdLimit; ++nextId) {
+    for (uint32_t nextId = docId; !isAtEnd(nextId); ++nextId) {
         if (_searchContext.cmp(nextId, _weight)) {
             setDocId(nextId);
             return;
@@ -367,7 +364,7 @@ template <typename SC>
 void
 FilterAttributeIteratorStrict<SC>::doSeek(uint32_t docId)
 {
-    for (uint32_t nextId = docId; nextId < _docIdLimit; ++nextId) {
+    for (uint32_t nextId = docId; !isAtEnd(nextId); ++nextId) {
         if (_searchContext.cmp(nextId)) {
             setDocId(nextId);
             return;
@@ -382,7 +379,6 @@ AttributeIteratorT<SC>::or_hits_into(BitVector & result, uint32_t begin_id) {
     AttributeIteratorBase::or_hits_into(_searchContext, result, begin_id);
 }
 
-
 template <typename SC>
 void
 FilterAttributeIteratorT<SC>::or_hits_into(BitVector & result, uint32_t begin_id) {
@@ -395,7 +391,6 @@ AttributeIteratorT<SC>::get_hits(uint32_t begin_id) {
     return AttributeIteratorBase::get_hits(_searchContext, begin_id);
 }
 
-
 template <typename SC>
 BitVector::UP
 FilterAttributeIteratorT<SC>::get_hits(uint32_t begin_id) {
@@ -414,5 +409,4 @@ FilterAttributeIteratorT<SC>::and_hits_into(BitVector & result, uint32_t begin_i
     AttributeIteratorBase::and_hits_into(_searchContext, result, begin_id);
 }
 
-
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/queryeval/searchable.h b/searchlib/src/vespa/searchlib/queryeval/searchable.h
index 078ad7f065d..8faef2be08b 100644
--- a/searchlib/src/vespa/searchlib/queryeval/searchable.h
+++ b/searchlib/src/vespa/searchlib/queryeval/searchable.h
@@ -2,10 +2,10 @@
 
 #pragma once
 
-#include <vector>
 #include "field_spec.h"
 #include "blueprint.h"
 #include <vespa/searchlib/queryeval/irequestcontext.h>
+#include <vector>
 
 namespace search {