enable re-using termwise bitvectors in second phase

detach match data from rank program and refactor match loop
re-use bitvector in termwise search if range is the same

4 files changed, 45 insertions, 4 deletions

diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp
index 0ff7e0899e9..e55bd7081a1 100644
--- a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp
+++ b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp
@@ -141,7 +141,8 @@ TEST("test overrides")
     overrides.add("value(1,2,3).2",  "6.0");
     overrides.add("bogus(feature)", "10.0");
 
-    rankProgram->setup(mdl, queryEnv, overrides);
+    MatchData::UP match_data = mdl.createMatchData();
+    rankProgram->setup(*match_data, queryEnv, overrides);
 
     std::map<vespalib::string, feature_t> res = Utils::getAllFeatures(*rankProgram, 2);
 
diff --git a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
index 5c0f09a1203..3ae5932889a 100644
--- a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
+++ b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/fef/test/indexenvironment.h>
 #include <vespa/searchlib/fef/test/queryenvironment.h>
 #include <vespa/searchlib/fef/test/plugin/unbox.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
 #include <vespa/searchlib/fef/rank_program.h>
 #include <vespa/searchlib/fef/verify_feature.h>
 #include <vespa/eval/eval/value_type.h>
@@ -87,10 +88,11 @@ struct Fixture {
             return vespalib::eval::error_value;        
         }
         MatchDataLayout mdl;
+        MatchData::UP md = mdl.createMatchData();
         QueryEnvironment queryEnv(&indexEnv);
         Properties overrides;
         RankProgram program(resolver);
-        program.setup(mdl, queryEnv, overrides);        
+        program.setup(*md, queryEnv, overrides);        
         auto result = program.get_seeds();
         EXPECT_EQUAL(1u, result.num_features());
         EXPECT_TRUE(!result.is_object(0)); // verifies auto-unboxing
diff --git a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp
index 8e23731e961..7e28178e5f7 100644
--- a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp
+++ b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp
@@ -6,6 +6,7 @@
 #include <vespa/searchlib/features/rankingexpressionfeature.h>
 #include <vespa/searchlib/fef/blueprintfactory.h>
 #include <vespa/searchlib/fef/indexproperties.h>
+#include <vespa/searchlib/fef/matchdatalayout.h>
 #include <vespa/searchlib/fef/test/indexenvironment.h>
 #include <vespa/searchlib/fef/test/queryenvironment.h>
 #include <vespa/searchlib/fef/test/plugin/sum.h>
@@ -69,10 +70,11 @@ struct Fixture {
     IndexEnvironment indexEnv;
     BlueprintResolver::SP resolver;
     Properties overrides;
+    MatchData::UP match_data;
     RankProgram program;
     size_t track_cnt;
     Fixture() : factory(), indexEnv(), resolver(new BlueprintResolver(factory, indexEnv)),
-                overrides(), program(resolver), track_cnt(0)
+                overrides(), match_data(), program(resolver), track_cnt(0)
     {
         factory.addPrototype(Blueprint::SP(new BoxingBlueprint()));
         factory.addPrototype(Blueprint::SP(new DocidBlueprint()));
@@ -107,7 +109,8 @@ struct Fixture {
         ASSERT_TRUE(resolver->compile());
         MatchDataLayout mdl;
         QueryEnvironment queryEnv(&indexEnv);
-        program.setup(mdl, queryEnv, overrides);
+        match_data = mdl.createMatchData();
+        program.setup(*match_data, queryEnv, overrides);
         return *this;
     }
     double get(uint32_t docid = default_docid) {
diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp
index 57fb19b7b23..01c72497246 100644
--- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp
+++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp
@@ -207,4 +207,39 @@ TEST("Access subqueries") {
     EXPECT_EQUAL(0ULL, state.f3->getSubqueries());
 }
 
+TEST("require that TermFieldMatchData can be tagged as needed or not") {
+    TermFieldMatchData tfmd;
+    tfmd.setFieldId(123);
+    EXPECT_EQUAL(tfmd.getFieldId(),123u);
+    EXPECT_TRUE(!tfmd.isNotNeeded());
+    tfmd.tagAsNotNeeded();
+    EXPECT_EQUAL(tfmd.getFieldId(),123u);
+    EXPECT_TRUE(tfmd.isNotNeeded());
+    tfmd.tagAsNeeded();
+    EXPECT_EQUAL(tfmd.getFieldId(),123u);
+    EXPECT_TRUE(!tfmd.isNotNeeded());
+}
+
+TEST("require that MatchData soft_reset retains appropriate state") {    
+    auto md = MatchData::makeTestInstance(10, 10);
+    md->set_termwise_limit(0.5);
+    auto *old_term = md->resolveTermField(7); 
+    old_term->tagAsNotNeeded();
+    old_term->populate_fixed()->setElementWeight(21);
+    old_term->resetOnlyDocId(42);
+    EXPECT_EQUAL(md->get_termwise_limit(), 0.5);
+    EXPECT_TRUE(old_term->isNotNeeded());
+    EXPECT_EQUAL(old_term->getFieldId(), 7u);
+    EXPECT_EQUAL(old_term->getWeight(), 21);
+    EXPECT_EQUAL(old_term->getDocId(), 42u);
+    md->soft_reset();
+    auto *new_term = md->resolveTermField(7);
+    EXPECT_EQUAL(new_term, old_term);
+    EXPECT_EQUAL(md->get_termwise_limit(), 1.0);
+    EXPECT_TRUE(!new_term->isNotNeeded());
+    EXPECT_EQUAL(new_term->getFieldId(), 7u);
+    EXPECT_EQUAL(new_term->getWeight(), 21);
+    EXPECT_EQUAL(new_term->getDocId(), TermFieldMatchData::invalidId());
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }