Aggregate elementId

11 files changed, 88 insertions, 71 deletions

diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
index 194c9b422da..f3239553fa9 100644
--- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
@@ -37,9 +37,7 @@ ArrayFieldValue::ArrayFieldValue(const ArrayFieldValue& other)
 {
 }
 
-ArrayFieldValue::~ArrayFieldValue()
-{
-}
+ArrayFieldValue::~ArrayFieldValue() = default;
 
 ArrayFieldValue&
 ArrayFieldValue::operator=(const ArrayFieldValue& other)
@@ -194,6 +192,7 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos,
     std::vector<int> indicesToRemove;
 
     for (int i = startPos; i <= endPos && i < static_cast<int>(_array->size()); ++i) {
+        handler.setArrayIndex(i);
         if (!variable.empty()) {
             handler.getVariables()[variable] = IndexValue(i);
         }
@@ -212,9 +211,7 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos,
         handler.getVariables().erase(variable);
     }
 
-    for (std::vector<int>::reverse_iterator i = indicesToRemove.rbegin();
-         i != indicesToRemove.rend(); ++i)
-    {
+    for (auto i = indicesToRemove.rbegin(); i != indicesToRemove.rend(); ++i) {
         const_cast<ArrayFieldValue&>(*this).remove(*i);
     }
 
diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.cpp b/document/src/vespa/document/fieldvalue/iteratorhandler.cpp
index 2e9a525d9e6..97529e09578 100644
--- a/document/src/vespa/document/fieldvalue/iteratorhandler.cpp
+++ b/document/src/vespa/document/fieldvalue/iteratorhandler.cpp
@@ -5,7 +5,14 @@
 
 namespace document::fieldvalue {
 
-IteratorHandler::~IteratorHandler() { }
+IteratorHandler::IteratorHandler()
+    : _weight(1),
+      _arrayIndexStack(1, 0),
+      _variables()
+{
+}
+
+IteratorHandler::~IteratorHandler() = default;
 
 
 void
@@ -18,11 +25,13 @@ IteratorHandler::handleComplex(const FieldValue & fv) {
 }
 void
 IteratorHandler::handleCollectionStart(const FieldValue & fv) {
+    _arrayIndexStack.push_back(0);
     onCollectionStart(Content(fv, getWeight()));
 }
 void
 IteratorHandler::handleCollectionEnd(const FieldValue & fv) {
     onCollectionEnd(Content(fv, getWeight()));
+    _arrayIndexStack.resize(_arrayIndexStack.size() - 1);
 }
 void
 IteratorHandler::handleStructStart(const FieldValue & fv) {
diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.h b/document/src/vespa/document/fieldvalue/iteratorhandler.h
index 757cea54ef9..cf71504e7a1 100644
--- a/document/src/vespa/document/fieldvalue/iteratorhandler.h
+++ b/document/src/vespa/document/fieldvalue/iteratorhandler.h
@@ -4,6 +4,7 @@
 
 #include "variablemap.h"
 #include "modificationstatus.h"
+#include <vector>
 
 namespace document::fieldvalue {
 
@@ -55,7 +56,7 @@ protected:
         int _weight;
     };
 
-    IteratorHandler() : _weight(1) {}
+    IteratorHandler();
 
 public:
     virtual ~IteratorHandler();
@@ -72,6 +73,8 @@ public:
     void handleStructStart(const FieldValue &fv);
     void handleStructEnd(const FieldValue &fv);
     void setWeight(int weight) { _weight = weight; }
+    uint32_t getArrayIndex() const { return _arrayIndexStack.back(); }
+    void setArrayIndex(uint32_t index) { _arrayIndexStack.back() = index; }
     ModificationStatus modify(FieldValue &fv) { return doModify(fv); }
     fieldvalue::VariableMap &getVariables() { return _variables; }
     void setVariables(const fieldvalue::VariableMap &vars) { _variables = vars; }
@@ -93,6 +96,7 @@ private:
     int getWeight() const { return _weight; }
 
     int _weight;
+    std::vector<uint32_t>   _arrayIndexStack;
     fieldvalue::VariableMap _variables;
 };
 
diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp
index 8e5a641ab6e..ebd51c82794 100644
--- a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp
@@ -78,9 +78,7 @@ MapFieldValue::MapFieldValue(const DataType &mapType)
 {
 }
 
-MapFieldValue::~MapFieldValue()
-{
-}
+MapFieldValue::~MapFieldValue() = default;
 
 MapFieldValue::MapFieldValue(const MapFieldValue & rhs) :
     FieldValue(rhs),
@@ -425,6 +423,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
     bool wasModified = false;
     const bool isWSet(complexFieldValue.inherits(WeightedSetFieldValue::classId));
 
+    uint32_t index(0);
     if ( ! nested.atEnd() ) {
         LOG(spam, "not yet at end of field path");
         const FieldPathEntry & fpe = nested.cur();
@@ -451,6 +450,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
         case FieldPathEntry::MAP_ALL_KEYS:
             LOG(spam, "MAP_ALL_KEYS");
             for (const auto & entry : *this) {
+                handler.setArrayIndex(index++);
                 if (isWSet) {
                     handler.setWeight(static_cast<const IntFieldValue &>(*entry.second).getValue());
                 }
@@ -462,6 +462,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
         case FieldPathEntry::MAP_ALL_VALUES:
             LOG(spam, "MAP_ALL_VALUES");
             for (const auto & entry : *this) {
+                handler.setArrayIndex(index++);
                 wasModified = checkAndRemove(*entry.second,
                                              entry.second->iterateNested(nested.next(), handler),
                                              wasModified, keysToRemove);
@@ -482,6 +483,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
             } else {
                 PathRange next = nested.next();
                 for (const auto & entry : *this) {
+                    handler.setArrayIndex(index++);
                     LOG(spam, "key is '%s'", entry.first->toString().c_str());
                     handler.getVariables()[fpe.getVariableName()] = IndexValue(*entry.first);
                     LOG(spam, "vars at this time = %s", handler.getVariables().toString().c_str());
@@ -495,6 +497,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
         default:
             LOG(spam, "default");
             for (const auto & entry : *this) {
+                handler.setArrayIndex(index++);
                 if (isWSet) {
                     handler.setWeight(static_cast<const IntFieldValue &>(*entry.second).getValue());
                 }
@@ -522,6 +525,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
         if (handler.handleComplex(complexFieldValue)) {
             LOG(spam, "calling handler.handleComplex for all map keys");
             for (const auto & entry : *this) {
+                handler.setArrayIndex(index++);
                 if (isWSet) {
                     handler.setWeight(static_cast<const IntFieldValue &>(*entry.second).getValue());
                 }
diff --git a/searchlib/src/tests/query/query-old.cpp b/searchlib/src/tests/query/query-old.cpp
index 77fe813dda5..22e3bf79f9f 100644
--- a/searchlib/src/tests/query/query-old.cpp
+++ b/searchlib/src/tests/query/query-old.cpp
@@ -364,31 +364,31 @@ TEST("testPhraseEvaluate") {
     }
 
     // field 0
-    terms[0]->add(0, 0, 1);
-    terms[1]->add(1, 0, 1);
-    terms[2]->add(2, 0, 1);
-    terms[0]->add(7, 0, 1);
-    terms[1]->add(8, 0, 1);
-    terms[2]->add(9, 0, 1);
+    terms[0]->add(0, 0, 0, 1);
+    terms[1]->add(1, 0, 0, 1);
+    terms[2]->add(2, 0, 0, 1);
+    terms[0]->add(7, 0, 0, 1);
+    terms[1]->add(8, 0, 0, 1);
+    terms[2]->add(9, 0, 0, 1);
     // field 1
-    terms[0]->add(4, 1, 1);
-    terms[1]->add(5, 1, 1);
-    terms[2]->add(6, 1, 1);
+    terms[0]->add(4, 1, 0, 1);
+    terms[1]->add(5, 1, 0, 1);
+    terms[2]->add(6, 1, 0, 1);
     // field 2 (not complete match)
-    terms[0]->add(1, 2, 1);
-    terms[1]->add(2, 2, 1);
-    terms[2]->add(4, 2, 1);
+    terms[0]->add(1, 2, 0, 1);
+    terms[1]->add(2, 2, 0, 1);
+    terms[2]->add(4, 2, 0, 1);
     // field 3
-    terms[0]->add(0, 3, 1);
-    terms[1]->add(1, 3, 1);
-    terms[2]->add(2, 3, 1);
+    terms[0]->add(0, 3, 0, 1);
+    terms[1]->add(1, 3, 0, 1);
+    terms[2]->add(2, 3, 0, 1);
     // field 4 (not complete match)
-    terms[0]->add(1, 4, 1);
-    terms[1]->add(2, 4, 1);
+    terms[0]->add(1, 4, 0, 1);
+    terms[1]->add(2, 4, 0, 1);
     // field 5 (not complete match)
-    terms[0]->add(2, 5, 1);
-    terms[1]->add(1, 5, 1);
-    terms[2]->add(0, 5, 1);
+    terms[0]->add(2, 5, 0, 1);
+    terms[1]->add(1, 5, 0, 1);
+    terms[2]->add(0, 5, 0, 1);
     HitList hits;
     PhraseQueryNode * p = static_cast<PhraseQueryNode *>(phrases[0]);
     p->evaluateHits(hits);
@@ -414,15 +414,15 @@ TEST("testPhraseEvaluate") {
 
 TEST("testHit") {
     // positions (0 - (2^24-1))
-    assertHit(Hit(0,        0, 0),        0, 0, 0);
-    assertHit(Hit(256,      0, 1),      256, 0, 1);
-    assertHit(Hit(16777215, 0, -1), 16777215, 0, -1);
-    assertHit(Hit(16777216, 0, 1),        0, 1, 1); // overflow
+    assertHit(Hit(0,        0, 0, 0),        0, 0, 0);
+    assertHit(Hit(256,      0, 0, 1),      256, 0, 1);
+    assertHit(Hit(16777215, 0, 0, -1), 16777215, 0, -1);
+    assertHit(Hit(16777216, 0, 0, 1),        0, 1, 1); // overflow
 
     // contexts (0 - 255)
-    assertHit(Hit(0,   1, 1), 0,   1, 1);
-    assertHit(Hit(0, 255, 1), 0, 255, 1);
-    assertHit(Hit(0, 256, 1), 0,   0, 1); // overflow
+    assertHit(Hit(0,   1, 0, 1), 0,   1, 1);
+    assertHit(Hit(0, 255, 0, 1), 0, 255, 1);
+    assertHit(Hit(0, 256, 0, 1), 0,   0, 1); // overflow
 }
 
 void assertInt8Range(const std::string &term, bool expAdjusted, int64_t expLow, int64_t expHigh) {
diff --git a/searchlib/src/vespa/searchlib/query/posocc.h b/searchlib/src/vespa/searchlib/query/posocc.h
index 1b5b6283f88..fcabef0297f 100644
--- a/searchlib/src/vespa/searchlib/query/posocc.h
+++ b/searchlib/src/vespa/searchlib/query/posocc.h
@@ -4,28 +4,29 @@
 #include <cstdint>
 #include <vector>
 
-namespace search
-{
+namespace search {
 
 class Hit
 {
- public:
-  Hit(uint32_t pos_, uint32_t context_, int32_t weight_) :
-    _position(pos_ | (context_<<24)),
-    _weight(weight_)
-  { }
-  int32_t weight() const { return _weight; }
-  uint32_t pos()     const { return _position; }
-  uint32_t wordpos() const { return _position & 0xffffff; }
-  uint32_t context() const { return _position >> 24; }
-  bool operator < (const Hit & b) const { return cmp(b) < 0; }
- private:
-  int cmp(const Hit & b) const { return _position - b._position; }
-  uint32_t _position;
-  int32_t  _weight;
+public:
+    Hit(uint32_t pos_, uint32_t context_, uint32_t elemId_, int32_t weight_)
+        : _position(pos_ | (context_<<24)),
+          _elemId(elemId_),
+          _weight(weight_)
+    { }
+    int32_t weight() const { return _weight; }
+    uint32_t pos()     const { return _position; }
+    uint32_t wordpos() const { return _position & 0xffffff; }
+    uint32_t context() const { return _position >> 24; }
+    uint32_t elemId() const { return _elemId; }
+    bool operator < (const Hit & b) const { return cmp(b) < 0; }
+private:
+    int cmp(const Hit & b) const { return _position - b._position; }
+    uint32_t _position;
+    uint32_t _elemId;
+    int32_t  _weight;
 };
 
 typedef std::vector<Hit> HitList;
 
 }
-
diff --git a/searchlib/src/vespa/searchlib/query/query.cpp b/searchlib/src/vespa/searchlib/query/query.cpp
index 984337f40ba..2be4e505f67 100644
--- a/searchlib/src/vespa/searchlib/query/query.cpp
+++ b/searchlib/src/vespa/searchlib/query/query.cpp
@@ -16,22 +16,22 @@ QueryConnector::QueryConnector(const char * opName) :
 {
 }
 
-QueryConnector::~QueryConnector() { }
+QueryConnector::~QueryConnector() = default;
 
 const HitList & QueryConnector::evaluateHits(HitList & hl) const
 {
-  if (evaluate()) {
-    hl.push_back(Hit(1, 0, 1));
-  }
-  return hl;
+    if (evaluate()) {
+        hl.push_back(Hit(1, 0, 0, 1));
+    }
+    return hl;
 }
 
 void QueryConnector::reset()
 {
-  for(iterator it=begin(), mt=end(); it != mt; it++) {
-    QueryNode & qn = **it;
-    qn.reset();
-  }
+    for(iterator it=begin(), mt=end(); it != mt; it++) {
+        QueryNode & qn = **it;
+        qn.reset();
+    }
 }
 
 void QueryConnector::getLeafs(QueryTermList & tl)
diff --git a/searchlib/src/vespa/searchlib/query/queryterm.cpp b/searchlib/src/vespa/searchlib/query/queryterm.cpp
index d47d942dc10..4128629bcf0 100644
--- a/searchlib/src/vespa/searchlib/query/queryterm.cpp
+++ b/searchlib/src/vespa/searchlib/query/queryterm.cpp
@@ -301,9 +301,9 @@ void QueryTerm::resizeFieldId(size_t fieldNo)
     }
 }
 
-void QueryTerm::add(unsigned pos, unsigned context, int32_t weight_)
+void QueryTerm::add(unsigned pos, unsigned context, uint32_t elemId, int32_t weight_)
 {
-    _hitList.emplace_back(pos, context, weight_);
+    _hitList.emplace_back(pos, context, elemId, weight_);
 }
 
 template <int B>
diff --git a/searchlib/src/vespa/searchlib/query/queryterm.h b/searchlib/src/vespa/searchlib/query/queryterm.h
index 423b8d28efb..457a2feeeed 100644
--- a/searchlib/src/vespa/searchlib/query/queryterm.h
+++ b/searchlib/src/vespa/searchlib/query/queryterm.h
@@ -173,7 +173,7 @@ public:
     /// Gives you all phrases of this tree. Indicating that they are all const.
     void getPhrases(ConstQueryNodeRefList & tl) const override;
 
-    void                add(unsigned pos, unsigned context, int32_t weight);
+    void                add(unsigned pos, unsigned context, uint32_t elemId, int32_t weight);
     EncodingBitMap      encoding()                 const { return _encoding; }
     size_t              termLen()                  const { return getTermLen(); }
     const string      & index()                    const { return _index; }
diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp
index 31f53fee300..78531f41cf8 100644
--- a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp
+++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp
@@ -73,6 +73,7 @@ FieldSearcher::FieldSearcher(const FieldIdT & fId, bool defaultPrefix) :
     _field(fId),
     _matchType(defaultPrefix ? PREFIX : REGULAR),
     _maxFieldLength(0x100000),
+    _currentElementId(0),
     _currentElementWeight(1),
     _pureUsAsciiCount(0),
     _pureUsAsciiFieldCount(0),
@@ -85,9 +86,7 @@ FieldSearcher::FieldSearcher(const FieldIdT & fId, bool defaultPrefix) :
     zeroStat();
 }
 
-FieldSearcher::~FieldSearcher()
-{
-}
+FieldSearcher::~FieldSearcher() = default;
 
 bool FieldSearcher::search(const StorageDocument & doc)
 {
@@ -275,6 +274,7 @@ FieldSearcher::IteratorHandler::onPrimitive(uint32_t, const Content & c)
 {
     LOG(spam, "onPrimitive: field value '%s'", c.getValue().toString().c_str());
     _searcher.setCurrentWeight(c.getWeight());
+    _searcher.setCurrentElementId(getArrayIndex());
     _searcher.onValue(c.getValue());
 }
 
diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.h b/vsm/src/vespa/vsm/searcher/fieldsearcher.h
index 236382be5f9..2bf976de017 100644
--- a/vsm/src/vespa/vsm/searcher/fieldsearcher.h
+++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.h
@@ -95,11 +95,13 @@ private:
 
     void prepareFieldId();
     void setCurrentWeight(int32_t weight) { _currentElementWeight = weight; }
+    void setCurrentElementId(int32_t weight) { _currentElementId = weight; }
     bool onSearch(const StorageDocument & doc);
     virtual void onValue(const document::FieldValue & fv) = 0;
     FieldIdT      _field;
     MatchType     _matchType;
     unsigned      _maxFieldLength;
+    uint32_t      _currentElementId;
     int32_t       _currentElementWeight; // Contains the weight of the current item being evaluated.
     /// Number of bytes in blocks containing pure us-ascii
     unsigned _pureUsAsciiCount;
@@ -124,7 +126,7 @@ protected:
      * For each call to onValue() a batch of words are processed, and the position is local to this batch.
      **/
     void addHit(search::QueryTerm & qt, uint32_t pos) const {
-        qt.add(_words + pos, field(), getCurrentWeight());
+        qt.add(_words + pos, field(), _currentElementId, getCurrentWeight());
     }
 public:
     static search::byte _foldLowCase[256];