Merge pull request #7998 from vespa-engine/toregge/compact-lid-space-on-source-selector

Compact lid space on source selector.

4 files changed, 28 insertions, 6 deletions

diff --git a/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp b/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp
index a223b87a73f..1eda0b7cb67 100644
--- a/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp
+++ b/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp
@@ -41,7 +41,7 @@ private:
     void requireThatSelectorCanCloneAndSubtract();
     void requireThatSelectorCanCloneAndSubtract();
     template <typename SelectorType>
-    void requireThatSelectorCanSaveAndLoad();
+    void requireThatSelectorCanSaveAndLoad(bool compactLidSpace);
     void requireThatSelectorCanSaveAndLoad();
     template <typename SelectorType>
     void requireThatCompleteSourceRangeIsHandled();
@@ -140,12 +140,15 @@ Test::requireThatSelectorCanCloneAndSubtract()
 
 template <typename SelectorType>
 void
-Test::requireThatSelectorCanSaveAndLoad()
+Test::requireThatSelectorCanSaveAndLoad(bool compactLidSpace)
 {
     SelectorType selector(default_source, base_file_name2);
     setSources(selector);
     selector.setBaseId(base_id);
     selector.setSource(maxDocId + 1, default_source);
+    if (compactLidSpace) {
+        selector.compactLidSpace(maxDocId - 4);
+    }
 
     FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str());
     FastOS_FileInterface::MakeDirIfNotPresentOrExit(index_dir.c_str());
@@ -155,9 +158,13 @@ Test::requireThatSelectorCanSaveAndLoad()
     save_info->save(TuneFileAttributes(), DummyFileHeaderContext());
     typename SelectorType::UP
         selector2(SelectorType::load(base_file_name));
-    testSourceSelector(docs, arraysize(docs), default_source, *selector2);
+    testSourceSelector(docs, arraysize(docs) - compactLidSpace, default_source, *selector2);
     EXPECT_EQUAL(base_id, selector2->getBaseId());
-    EXPECT_EQUAL(maxDocId + 2, selector2->getDocIdLimit());
+    if (compactLidSpace) {
+        EXPECT_EQUAL(maxDocId - 4, selector2->getDocIdLimit());
+    } else {
+        EXPECT_EQUAL(maxDocId + 2, selector2->getDocIdLimit());
+    }
 
     FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str());
 }
@@ -165,7 +172,8 @@ Test::requireThatSelectorCanSaveAndLoad()
 void
 Test::requireThatSelectorCanSaveAndLoad()
 {
-    requireThatSelectorCanSaveAndLoad<FixedSourceSelector>();
+    requireThatSelectorCanSaveAndLoad<FixedSourceSelector>(false);
+    requireThatSelectorCanSaveAndLoad<FixedSourceSelector>(true);
 }
 
 template <typename SelectorType>
diff --git a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp
index ae2f0234e60..385ffc95c40 100644
--- a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp
@@ -88,4 +88,10 @@ FixedSourceSelector::setSource(uint32_t docId, queryeval::Source source)
     _source.commit();
 }
 
+void
+FixedSourceSelector::compactLidSpace(uint32_t lidLimit)
+{
+    _source.compactLidSpace(lidLimit + 1);
+}
+
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.h b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.h
index 4dbfc22c270..f92bb1e0eb2 100644
--- a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.h
+++ b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.h
@@ -38,8 +38,9 @@ public:
     // Inherit doc from ISourceSelector
     void setSource(uint32_t docId, queryeval::Source source) final override;
     uint32_t getDocIdLimit() const final override {
-        return _source.getNumDocs() - 1;
+        return _source.getCommittedDocIdLimit() - 1;
     }
+    void compactLidSpace(uint32_t lidLimit) override;
     std::unique_ptr<IIterator> createIterator() const final override {
         return std::make_unique<Iterator>(*this);
     }
diff --git a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h
index 88a3cb57a8a..7151a785c26 100644
--- a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h
+++ b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h
@@ -73,6 +73,13 @@ public:
     virtual uint32_t getDocIdLimit() const = 0;
 
     /**
+     * Sets the lid limit in this selector.
+     *
+     * @param lidLimit the new lid limit (one above highest valid doc id).
+     */
+    virtual void compactLidSpace(uint32_t lidLimit) = 0;
+
+    /**
      * Create a new iterator over the data held by this source
      * selector.
      *