Refactor tagging of TermFieldMatchData regarding which features are

needed by feature executors.

4 files changed, 67 insertions, 19 deletions

diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp
index ca9e331bb62..9ed94c02287 100644
--- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp
+++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp
@@ -256,12 +256,33 @@ TEST("require that TermFieldMatchData can be tagged as needed or not") {
     tfmd.setFieldId(123);
     EXPECT_EQUAL(tfmd.getFieldId(),123u);
     EXPECT_TRUE(!tfmd.isNotNeeded());
+    EXPECT_TRUE(tfmd.needs_normal_features());
+    EXPECT_TRUE(tfmd.needs_cheap_features());
     tfmd.tagAsNotNeeded();
     EXPECT_EQUAL(tfmd.getFieldId(),123u);
     EXPECT_TRUE(tfmd.isNotNeeded());
-    tfmd.tagAsNeeded();
+    EXPECT_TRUE(!tfmd.needs_normal_features());
+    EXPECT_TRUE(!tfmd.needs_cheap_features());
+    tfmd.setNeedNormalFeatures(true);
     EXPECT_EQUAL(tfmd.getFieldId(),123u);
     EXPECT_TRUE(!tfmd.isNotNeeded());
+    EXPECT_TRUE(tfmd.needs_normal_features());
+    EXPECT_TRUE(!tfmd.needs_cheap_features());
+    tfmd.setNeedCheapFeatures(true);
+    EXPECT_EQUAL(tfmd.getFieldId(),123u);
+    EXPECT_TRUE(!tfmd.isNotNeeded());
+    EXPECT_TRUE(tfmd.needs_normal_features());
+    EXPECT_TRUE(tfmd.needs_cheap_features());
+    tfmd.setNeedNormalFeatures(false);
+    EXPECT_EQUAL(tfmd.getFieldId(),123u);
+    EXPECT_TRUE(!tfmd.isNotNeeded());
+    EXPECT_TRUE(!tfmd.needs_normal_features());
+    EXPECT_TRUE(tfmd.needs_cheap_features());
+    tfmd.setNeedCheapFeatures(false);
+    EXPECT_EQUAL(tfmd.getFieldId(),123u);
+    EXPECT_TRUE(tfmd.isNotNeeded());
+    EXPECT_TRUE(!tfmd.needs_normal_features());
+    EXPECT_TRUE(!tfmd.needs_cheap_features());
 }
 
 TEST("require that MatchData soft_reset retains appropriate state") {    
@@ -280,7 +301,7 @@ TEST("require that MatchData soft_reset retains appropriate state") {
     auto *new_term = md->resolveTermField(7);
     EXPECT_EQUAL(new_term, old_term);
     EXPECT_EQUAL(md->get_termwise_limit(), 1.0);
-    EXPECT_TRUE(!new_term->isNotNeeded());
+    EXPECT_TRUE(new_term->isNotNeeded());
     EXPECT_EQUAL(new_term->getFieldId(), 7u);
     EXPECT_EQUAL(new_term->getWeight(), 21);
     EXPECT_EQUAL(new_term->getDocId(), TermFieldMatchData::invalidId());
diff --git a/searchlib/src/vespa/searchlib/fef/matchdata.cpp b/searchlib/src/vespa/searchlib/fef/matchdata.cpp
index 0c589749112..be14b7686e9 100644
--- a/searchlib/src/vespa/searchlib/fef/matchdata.cpp
+++ b/searchlib/src/vespa/searchlib/fef/matchdata.cpp
@@ -15,7 +15,7 @@ void
 MatchData::soft_reset()
 {
     for (auto &tfmd: _termFields) {
-        tfmd.resetOnlyDocId(TermFieldMatchData::invalidId()).tagAsNeeded();
+        tfmd.resetOnlyDocId(TermFieldMatchData::invalidId());
     }
     _termwise_limit = 1.0;
 }
diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
index 1471021ddab..2b7b67e8365 100644
--- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
+++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
@@ -8,7 +8,8 @@ namespace search::fef {
 
 TermFieldMatchData::TermFieldMatchData() :
     _docId(invalidId()),
-    _fieldId(FIELDID_MASK),
+    _fieldId(ILLEGAL_FIELD_ID),
+    _flags(UNPACK_ALL_FEATURES_MASK),
     _sz(0),
     _numOccs(0),
     _fieldLength(0)
@@ -19,6 +20,7 @@ TermFieldMatchData::TermFieldMatchData() :
 TermFieldMatchData::TermFieldMatchData(const TermFieldMatchData & rhs) :
     _docId(rhs._docId),
     _fieldId(rhs._fieldId),
+    _flags(UNPACK_ALL_FEATURES_MASK),
     _sz(0),
     _numOccs(0),
     _fieldLength(0)
@@ -66,11 +68,11 @@ TermFieldMatchData::populate_fixed() {
 TermFieldMatchData &
 TermFieldMatchData::setFieldId(uint32_t fieldId) {
     if (fieldId == IllegalFieldId) {
-        fieldId = FIELDID_MASK;
+        fieldId = ILLEGAL_FIELD_ID;
     } else {
-        assert(fieldId < FIELDID_MASK);
+        assert(fieldId < ILLEGAL_FIELD_ID);
     }
-    _fieldId = (_fieldId & ~FIELDID_MASK) | fieldId;
+    _fieldId = fieldId;
     return *this;
 }
 
@@ -90,6 +92,7 @@ TermFieldMatchData::swap(TermFieldMatchData &rhs)
 {
     sswap(&_docId, &rhs._docId);
     sswap(&_fieldId, &rhs._fieldId);
+    sswap(&_flags, &rhs._flags);
     sswap(&_sz, &rhs._sz);
     sswap(&_numOccs, &rhs._numOccs);
     sswap(&_fieldLength, &rhs._fieldLength);
@@ -131,7 +134,7 @@ TermFieldMatchData::allocateVector()
         n[0] = *getFixed();
         _data._positions._maxElementLength = getFixed()->getElementLen();
     }
-    _fieldId = _fieldId | 0x4000; // set allocated() flag
+    _flags |= MULTIPOS_FLAG; // set allocated() flag
     _data._positions._allocated = newSize;
     _data._positions._positions = n;
 }
diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h
index 4d56a3c7ea7..5f1b2ae2ffe 100644
--- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h
+++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h
@@ -34,8 +34,8 @@ public:
         uint64_t      _subqueries;
     };
 private:
-    bool  isRawScore()  const { return _fieldId & 0x8000; }
-    bool  isMultiPos()  const { return _fieldId & 0x4000; }
+    bool  isRawScore()  const { return _flags & RAW_SCORE_FLAG; }
+    bool  isMultiPos()  const { return _flags & MULTIPOS_FLAG; }
     bool  empty() const { return _sz == 0; }
     void  clear() { _sz = 0; }
     bool  allocated() const { return isMultiPos(); }
@@ -49,11 +49,16 @@ private:
     void allocateVector();
     void resizePositionVector(size_t sz) __attribute__((noinline));
 
-    enum { FIELDID_MASK = 0x1fff};
+    static constexpr uint16_t ILLEGAL_FIELD_ID = std::numeric_limits<uint16_t>::max();
+    static constexpr uint16_t RAW_SCORE_FLAG = 1;
+    static constexpr uint16_t MULTIPOS_FLAG = 2;
+    static constexpr uint16_t UNPACK_NORMAL_FEATURES_FLAG = 4;
+    static constexpr uint16_t UNPACK_CHEAP_FEATURES_FLAG = 8;
+    static constexpr uint16_t UNPACK_ALL_FEATURES_MASK = UNPACK_NORMAL_FEATURES_FLAG | UNPACK_CHEAP_FEATURES_FLAG;
 
     uint32_t  _docId;
-    // 3 upper bits used to tell if it is use for RawScore, SinglePos or multiPos.
     uint16_t  _fieldId;
+    uint16_t  _flags;
     uint16_t  _sz;
 
     // Number of occurrences and field length used when unpacking "cheap" features.
@@ -113,7 +118,7 @@ public:
      * @return field id
      **/
     uint32_t getFieldId() const {
-        return __builtin_expect((_fieldId & FIELDID_MASK) != FIELDID_MASK, true) ? (_fieldId & FIELDID_MASK) : IllegalFieldId;
+        return __builtin_expect(_fieldId != ILLEGAL_FIELD_ID, true) ? _fieldId : IllegalFieldId;
     }
 
     /**
@@ -165,7 +170,7 @@ public:
         return *this;
     }
     TermFieldMatchData & enableRawScore() {
-        _fieldId = _fieldId | 0x8000;
+        _flags |= RAW_SCORE_FLAG;
         return *this;
     }
 
@@ -254,20 +259,39 @@ public:
      * This indicates if this instance is actually used for ranking or not.
      * @return true if it is not needed.
      */
-    bool  isNotNeeded() const { return _fieldId & 0x2000; }
+    bool  isNotNeeded() const { return ((_flags & (UNPACK_NORMAL_FEATURES_FLAG | UNPACK_CHEAP_FEATURES_FLAG)) == 0u); }
+
+    bool needs_normal_features() const { return ((_flags & UNPACK_NORMAL_FEATURES_FLAG) != 0u); }
+
+    bool needs_cheap_features() const { return ((_flags & UNPACK_CHEAP_FEATURES_FLAG) != 0u); }
 
     /**
      * Tag that this instance is not really used for ranking.
      */
     void tagAsNotNeeded() {
-        _fieldId = _fieldId | 0x2000;
+        _flags &=  ~(UNPACK_NORMAL_FEATURES_FLAG | UNPACK_CHEAP_FEATURES_FLAG);
+    }
+
+    /**
+     * Tag that this instance is used for ranking (normal features)
+     */
+    void setNeedNormalFeatures(bool needed) {
+        if (needed) {
+            _flags |= UNPACK_NORMAL_FEATURES_FLAG;
+        } else {
+            _flags &= ~UNPACK_NORMAL_FEATURES_FLAG;
+        }
     }
 
     /**
-     * Tag that this instance is used for ranking.
+     * Tag that this instance is used for ranking (cheap features)
      */
-    void tagAsNeeded() {
-        _fieldId = _fieldId & ~0x2000;
+    void setNeedCheapFeatures(bool needed) {
+        if (needed) {
+            _flags |= UNPACK_CHEAP_FEATURES_FLAG;
+        } else {
+            _flags &= ~UNPACK_CHEAP_FEATURES_FLAG;
+        }
     }
 
     /**