Revert "Revert "When we pull in a cacheline, we should use it too.""

5 files changed, 73 insertions, 60 deletions

diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp
index 96234e373dc..0a33e23de72 100644
--- a/searchlib/src/vespa/searchlib/common/bitvector.cpp
+++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp
@@ -167,7 +167,7 @@ BitVector::countInterval(Range range_in) const
         ++endw;
     }
     if (startw < endw) {
-        res += IAccelrated::getAccelrator().populationCount(bitValues + startw, endw - startw);
+        res += IAccelrated::getAccelerator().populationCount(bitValues + startw, endw - startw);
     }
     if (partialEnd) {
         res += Optimized::popCount(bitValues[endw] & ~endBits(last));
@@ -185,13 +185,13 @@ BitVector::orWith(const BitVector & right)
         if (right.size() > 0) {
             ssize_t commonBytes = numActiveBytes(getStartIndex(), right.size()) - sizeof(Word);
             if (commonBytes > 0) {
-                IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
+                IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
             }
             Index last(right.size() - 1);
             getWordIndex(last)[0] |= (right.getWordIndex(last)[0] & ~endBits(last));
         }
     } else {
-        IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
+        IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
     }
     repairEnds();
     invalidateCachedCount();
@@ -216,7 +216,7 @@ BitVector::andWith(const BitVector & right)
     verifyInclusiveStart(*this, right);
 
     uint32_t commonBytes = std::min(getActiveBytes(), numActiveBytes(getStartIndex(), right.size()));
-    IAccelrated::getAccelrator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
+    IAccelrated::getAccelerator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
     if (right.size() < size()) {
         clearInterval(right.size(), size());
     }
@@ -235,13 +235,13 @@ BitVector::andNotWith(const BitVector& right)
         if (right.size() > 0) {
             ssize_t commonBytes = numActiveBytes(getStartIndex(), right.size()) - sizeof(Word);
             if (commonBytes > 0) {
-                IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
+                IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
             }
             Index last(right.size() - 1);
             getWordIndex(last)[0] &= ~(right.getWordIndex(last)[0] & ~endBits(last));
         }
     } else {
-        IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
+        IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
     }
 
     repairEnds();
@@ -250,7 +250,7 @@ BitVector::andNotWith(const BitVector& right)
 
 void
 BitVector::notSelf() {
-    IAccelrated::getAccelrator().notBit(getActiveStart(), getActiveBytes());
+    IAccelrated::getAccelerator().notBit(getActiveStart(), getActiveBytes());
     setGuardBit();
     invalidateCachedCount();
 }
diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
index a8737a19eec..37fd98c9f20 100644
--- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
@@ -256,7 +256,7 @@ namespace dotproduct::array {
 template <typename BaseType>
 DotProductExecutorBase<BaseType>::DotProductExecutorBase(const V & queryVector)
     : FeatureExecutor(),
-      _multiplier(IAccelrated::getAccelrator()),
+      _multiplier(IAccelrated::getAccelerator()),
       _queryVector(queryVector)
 {
 }
diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp
index 105d57b22b1..d36d16a679a 100644
--- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp
@@ -1,19 +1,19 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/searchlib/queryeval/multibitvectoriterator.h>
-#include <vespa/searchlib/queryeval/andsearch.h>
-#include <vespa/searchlib/queryeval/andnotsearch.h>
-#include <vespa/searchlib/queryeval/sourceblendersearch.h>
-#include <vespa/searchlib/queryeval/orsearch.h>
+#include "multibitvectoriterator.h"
+#include "andsearch.h"
+#include "andnotsearch.h"
+#include "sourceblendersearch.h"
 #include <vespa/searchlib/common/bitvectoriterator.h>
-#include <vespa/searchlib/attribute/attributeiterators.h>
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include <vespa/searchlib/fef/termfieldmatchdataarray.h>
 #include <vespa/vespalib/util/optimized.h>
+#include <vespa/vespalib/hwaccelrated/iaccelrated.h>
 
 namespace search::queryeval {
 
 using vespalib::Trinary;
+using vespalib::hwaccelrated::IAccelrated;
 
 namespace {
 
@@ -21,7 +21,15 @@ template<typename Update>
 class MultiBitVectorIterator : public MultiBitVectorIteratorBase
 {
 public:
-    MultiBitVectorIterator(Children children) : MultiBitVectorIteratorBase(std::move(children)) { }
+    explicit MultiBitVectorIterator(Children children)
+        : MultiBitVectorIteratorBase(std::move(children)),
+          _update(),
+          _accel(IAccelrated::getAccelerator()),
+          _lastWords()
+    {
+        static_assert(sizeof(_lastWords) == 64, "Latswords should have 64 byte size");
+        memset(&_lastWords, 0, sizeof(_lastWords));
+    }
 protected:
     void updateLastValue(uint32_t docId);
     void strictSeek(uint32_t docId);
@@ -29,33 +37,55 @@ private:
     void doSeek(uint32_t docId) override;
     Trinary is_strict() const override { return Trinary::False; }
     bool acceptExtraFilter() const override { return Update::isAnd(); }
-    Update                  _update;
+    Update              _update;
+    const IAccelrated & _accel;
+    alignas(64) Word    _lastWords[8];
 };
 
 template<typename Update>
 class MultiBitVectorIteratorStrict : public MultiBitVectorIterator<Update>
 {
 public:
-    MultiBitVectorIteratorStrict(MultiSearch::Children children) : MultiBitVectorIterator<Update>(std::move(children)) { }
+    explicit MultiBitVectorIteratorStrict(MultiSearch::Children  children)
+        : MultiBitVectorIterator<Update>(std::move(children))
+    { }
 private:
     void doSeek(uint32_t docId) override { this->strictSeek(docId); }
     Trinary is_strict() const override { return Trinary::True; }
 };
 
+struct And {
+    using Word = BitWord::Word;
+    void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, Word *dest) {
+        accel.and64(offset*sizeof(uint64_t), src, dest);
+    }
+    static bool isAnd() { return true; }
+};
+
+struct Or {
+    using Word = BitWord::Word;
+    void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, Word *dest) {
+        accel.or64(offset*sizeof(uint64_t), src, dest);
+    }
+    static bool isAnd() { return false; }
+};
+
 template<typename Update>
 void MultiBitVectorIterator<Update>::updateLastValue(uint32_t docId)
 {
     if (docId >= _lastMaxDocIdLimit) {
-        if (__builtin_expect(docId < _numDocs, true)) {
-            const uint32_t index(wordNum(docId));
-            _lastValue = _bvs[0][index];
-            for(uint32_t i(1); i < _bvs.size(); i++) {
-                _lastValue = _update(_lastValue, _bvs[i][index]);
-            }
-            _lastMaxDocIdLimit = (index + 1) * WordLen;
-        } else {
+        if (__builtin_expect(docId >= _numDocs, false)) {
             setAtEnd();
+            return;
+        }
+        const uint32_t index(wordNum(docId));
+        if (docId >= _lastMaxDocIdLimitRequireFetch) {
+            uint32_t baseIndex = index & ~(sizeof(_lastWords)/sizeof(Word) - 1);
+            _update(_accel, baseIndex, _bvs, _lastWords);
+            _lastMaxDocIdLimitRequireFetch = (baseIndex + (sizeof(_lastWords)/sizeof(Word))) * WordLen;
         }
+        _lastValue = _lastWords[index % (sizeof(_lastWords)/sizeof(Word))];
+        _lastMaxDocIdLimit = (index + 1) * WordLen;
     }
 }
 
@@ -75,7 +105,7 @@ template<typename Update>
 void
 MultiBitVectorIterator<Update>::strictSeek(uint32_t docId)
 {
-    for (updateLastValue(docId), _lastValue=_lastValue & checkTab(docId);
+    for (updateLastValue(docId), _lastValue = _lastValue & checkTab(docId);
          (_lastValue == 0) && __builtin_expect(! isAtEnd(), true);
          updateLastValue(_lastMaxDocIdLimit));
     if (__builtin_expect(!isAtEnd(), true)) {
@@ -88,21 +118,6 @@ MultiBitVectorIterator<Update>::strictSeek(uint32_t docId)
     }
 }
 
-struct And {
-    typedef BitWord::Word Word;
-    Word operator () (const Word a, const Word b) {
-        return a & b;
-    }
-    static bool isAnd() { return true; }
-};
-
-struct Or {
-    typedef BitWord::Word Word;
-    Word operator () (const Word a, const Word b) {
-        return a | b;
-    }
-    static bool isAnd() { return false; }
-};
 
 typedef MultiBitVectorIterator<And> AndBVIterator;
 typedef MultiBitVectorIteratorStrict<And> AndBVIteratorStrict;
@@ -136,14 +151,15 @@ bool canOptimize(const MultiSearch & s) {
 MultiBitVectorIteratorBase::MultiBitVectorIteratorBase(Children children) :
     MultiSearch(std::move(children)),
     _numDocs(std::numeric_limits<unsigned int>::max()),
-    _lastValue(0),
     _lastMaxDocIdLimit(0),
+    _lastMaxDocIdLimitRequireFetch(0),
+    _lastValue(0),
     _bvs()
 {
     _bvs.reserve(getChildren().size());
-    for (size_t i(0); i < getChildren().size(); i++) {
-        const auto * bv = static_cast<const BitVectorIterator *>(getChildren()[i].get());
-        _bvs.emplace_back(reinterpret_cast<const Word *>(bv->getBitValues()), bv->isInverted());
+    for (const auto & child : getChildren()) {
+        const auto * bv = static_cast<const BitVectorIterator *>(child.get());
+        _bvs.emplace_back(bv->getBitValues(), bv->isInverted());
         _numDocs = std::min(_numDocs, bv->getDocIdLimit());
     }
 }
@@ -155,6 +171,7 @@ MultiBitVectorIteratorBase::initRange(uint32_t beginId, uint32_t endId)
 {
     MultiSearch::initRange(beginId, endId);
     _lastMaxDocIdLimit = 0;
+    _lastMaxDocIdLimitRequireFetch = 0;
 }
 
 SearchIterator::UP
@@ -163,9 +180,10 @@ MultiBitVectorIteratorBase::andWith(UP filter, uint32_t estimate)
     (void) estimate;
     if (filter->isBitVector() && acceptExtraFilter()) {
         const auto & bv = static_cast<const BitVectorIterator &>(*filter);
-        _bvs.emplace_back(reinterpret_cast<const Word *>(bv.getBitValues()), bv.isInverted());
+        _bvs.emplace_back(bv.getBitValues(), bv.isInverted());
         insert(getChildren().size(), std::move(filter));
         _lastMaxDocIdLimit = 0;  // force reload
+        _lastMaxDocIdLimitRequireFetch = 0;
     }
     return filter;
 }
diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h
index cde9ffcbfe5..29e92584ffe 100644
--- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h
+++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h
@@ -11,7 +11,7 @@ namespace search::queryeval {
 class MultiBitVectorIteratorBase : public MultiSearch, protected BitWord
 {
 public:
-    ~MultiBitVectorIteratorBase();
+    ~MultiBitVectorIteratorBase() override;
     void initRange(uint32_t beginId, uint32_t endId) override;
     void addUnpackIndex(size_t index) { _unpackInfo.add(index); }
     /**
@@ -20,26 +20,21 @@ public:
      */
     static SearchIterator::UP optimize(SearchIterator::UP parent);
 protected:
-    MultiBitVectorIteratorBase(Children children);
-    class MetaWord {
-    public:
-        MetaWord(const Word * words, bool inverted) : _words(words), _inverted(inverted) { }
-        Word operator [] (uint32_t index) const { return _inverted ? ~_words[index] : _words[index]; }
-    private:
-        const Word * _words;
-        bool         _inverted;
-    };
+    MultiBitVectorIteratorBase(Children hildren);
+    using MetaWord = std::pair<const void *, bool>;
 
     uint32_t                _numDocs;
-    Word                    _lastValue; // Last value computed
     uint32_t                _lastMaxDocIdLimit; // next documentid requiring recomputation.
+    uint32_t                _lastMaxDocIdLimitRequireFetch;
+    Word                    _lastValue; // Last value computed
     std::vector<MetaWord>   _bvs;
 private:
     virtual bool acceptExtraFilter() const = 0;
     UP andWith(UP filter, uint32_t estimate) override;
     void doUnpack(uint32_t docid) override;
-    UnpackInfo _unpackInfo;
     static SearchIterator::UP optimizeMultiSearch(SearchIterator::UP parent);
+
+    UnpackInfo  _unpackInfo;
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h
index 79f987c740c..d37495e85da 100644
--- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h
+++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h
@@ -17,7 +17,7 @@ template <typename FloatType>
 class SquaredEuclideanDistance : public DistanceFunction {
 public:
     SquaredEuclideanDistance()
-        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator())
+        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator())
     {}
     double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override {
         auto lhs_vector = lhs.typify<FloatType>();
@@ -60,7 +60,7 @@ template <typename FloatType>
 class AngularDistance : public DistanceFunction {
 public:
     AngularDistance()
-        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator())
+        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator())
     {}
     double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override {
         auto lhs_vector = lhs.typify<FloatType>();