Revert "When we pull in a cacheline, we should use it too."

5 files changed, 60 insertions, 73 deletions

diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp
index 0a33e23de72..96234e373dc 100644
--- a/searchlib/src/vespa/searchlib/common/bitvector.cpp
+++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp
@@ -167,7 +167,7 @@ BitVector::countInterval(Range range_in) const
         ++endw;
     }
     if (startw < endw) {
-        res += IAccelrated::getAccelerator().populationCount(bitValues + startw, endw - startw);
+        res += IAccelrated::getAccelrator().populationCount(bitValues + startw, endw - startw);
     }
     if (partialEnd) {
         res += Optimized::popCount(bitValues[endw] & ~endBits(last));
@@ -185,13 +185,13 @@ BitVector::orWith(const BitVector & right)
         if (right.size() > 0) {
             ssize_t commonBytes = numActiveBytes(getStartIndex(), right.size()) - sizeof(Word);
             if (commonBytes > 0) {
-                IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
+                IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
             }
             Index last(right.size() - 1);
             getWordIndex(last)[0] |= (right.getWordIndex(last)[0] & ~endBits(last));
         }
     } else {
-        IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
+        IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
     }
     repairEnds();
     invalidateCachedCount();
@@ -216,7 +216,7 @@ BitVector::andWith(const BitVector & right)
     verifyInclusiveStart(*this, right);
 
     uint32_t commonBytes = std::min(getActiveBytes(), numActiveBytes(getStartIndex(), right.size()));
-    IAccelrated::getAccelerator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
+    IAccelrated::getAccelrator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
     if (right.size() < size()) {
         clearInterval(right.size(), size());
     }
@@ -235,13 +235,13 @@ BitVector::andNotWith(const BitVector& right)
         if (right.size() > 0) {
             ssize_t commonBytes = numActiveBytes(getStartIndex(), right.size()) - sizeof(Word);
             if (commonBytes > 0) {
-                IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
+                IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes);
             }
             Index last(right.size() - 1);
             getWordIndex(last)[0] &= ~(right.getWordIndex(last)[0] & ~endBits(last));
         }
     } else {
-        IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
+        IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
     }
 
     repairEnds();
@@ -250,7 +250,7 @@ BitVector::andNotWith(const BitVector& right)
 
 void
 BitVector::notSelf() {
-    IAccelrated::getAccelerator().notBit(getActiveStart(), getActiveBytes());
+    IAccelrated::getAccelrator().notBit(getActiveStart(), getActiveBytes());
     setGuardBit();
     invalidateCachedCount();
 }
diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
index 37fd98c9f20..a8737a19eec 100644
--- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
@@ -256,7 +256,7 @@ namespace dotproduct::array {
 template <typename BaseType>
 DotProductExecutorBase<BaseType>::DotProductExecutorBase(const V & queryVector)
     : FeatureExecutor(),
-      _multiplier(IAccelrated::getAccelerator()),
+      _multiplier(IAccelrated::getAccelrator()),
       _queryVector(queryVector)
 {
 }
diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp
index d36d16a679a..105d57b22b1 100644
--- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp
@@ -1,19 +1,19 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include "multibitvectoriterator.h"
-#include "andsearch.h"
-#include "andnotsearch.h"
-#include "sourceblendersearch.h"
+#include <vespa/searchlib/queryeval/multibitvectoriterator.h>
+#include <vespa/searchlib/queryeval/andsearch.h>
+#include <vespa/searchlib/queryeval/andnotsearch.h>
+#include <vespa/searchlib/queryeval/sourceblendersearch.h>
+#include <vespa/searchlib/queryeval/orsearch.h>
 #include <vespa/searchlib/common/bitvectoriterator.h>
+#include <vespa/searchlib/attribute/attributeiterators.h>
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include <vespa/searchlib/fef/termfieldmatchdataarray.h>
 #include <vespa/vespalib/util/optimized.h>
-#include <vespa/vespalib/hwaccelrated/iaccelrated.h>
 
 namespace search::queryeval {
 
 using vespalib::Trinary;
-using vespalib::hwaccelrated::IAccelrated;
 
 namespace {
 
@@ -21,15 +21,7 @@ template<typename Update>
 class MultiBitVectorIterator : public MultiBitVectorIteratorBase
 {
 public:
-    explicit MultiBitVectorIterator(Children children)
-        : MultiBitVectorIteratorBase(std::move(children)),
-          _update(),
-          _accel(IAccelrated::getAccelerator()),
-          _lastWords()
-    {
-        static_assert(sizeof(_lastWords) == 64, "Latswords should have 64 byte size");
-        memset(&_lastWords, 0, sizeof(_lastWords));
-    }
+    MultiBitVectorIterator(Children children) : MultiBitVectorIteratorBase(std::move(children)) { }
 protected:
     void updateLastValue(uint32_t docId);
     void strictSeek(uint32_t docId);
@@ -37,55 +29,33 @@ private:
     void doSeek(uint32_t docId) override;
     Trinary is_strict() const override { return Trinary::False; }
     bool acceptExtraFilter() const override { return Update::isAnd(); }
-    Update              _update;
-    const IAccelrated & _accel;
-    alignas(64) Word    _lastWords[8];
+    Update                  _update;
 };
 
 template<typename Update>
 class MultiBitVectorIteratorStrict : public MultiBitVectorIterator<Update>
 {
 public:
-    explicit MultiBitVectorIteratorStrict(MultiSearch::Children  children)
-        : MultiBitVectorIterator<Update>(std::move(children))
-    { }
+    MultiBitVectorIteratorStrict(MultiSearch::Children children) : MultiBitVectorIterator<Update>(std::move(children)) { }
 private:
     void doSeek(uint32_t docId) override { this->strictSeek(docId); }
     Trinary is_strict() const override { return Trinary::True; }
 };
 
-struct And {
-    using Word = BitWord::Word;
-    void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, Word *dest) {
-        accel.and64(offset*sizeof(uint64_t), src, dest);
-    }
-    static bool isAnd() { return true; }
-};
-
-struct Or {
-    using Word = BitWord::Word;
-    void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, Word *dest) {
-        accel.or64(offset*sizeof(uint64_t), src, dest);
-    }
-    static bool isAnd() { return false; }
-};
-
 template<typename Update>
 void MultiBitVectorIterator<Update>::updateLastValue(uint32_t docId)
 {
     if (docId >= _lastMaxDocIdLimit) {
-        if (__builtin_expect(docId >= _numDocs, false)) {
+        if (__builtin_expect(docId < _numDocs, true)) {
+            const uint32_t index(wordNum(docId));
+            _lastValue = _bvs[0][index];
+            for(uint32_t i(1); i < _bvs.size(); i++) {
+                _lastValue = _update(_lastValue, _bvs[i][index]);
+            }
+            _lastMaxDocIdLimit = (index + 1) * WordLen;
+        } else {
             setAtEnd();
-            return;
-        }
-        const uint32_t index(wordNum(docId));
-        if (docId >= _lastMaxDocIdLimitRequireFetch) {
-            uint32_t baseIndex = index & ~(sizeof(_lastWords)/sizeof(Word) - 1);
-            _update(_accel, baseIndex, _bvs, _lastWords);
-            _lastMaxDocIdLimitRequireFetch = (baseIndex + (sizeof(_lastWords)/sizeof(Word))) * WordLen;
         }
-        _lastValue = _lastWords[index % (sizeof(_lastWords)/sizeof(Word))];
-        _lastMaxDocIdLimit = (index + 1) * WordLen;
     }
 }
 
@@ -105,7 +75,7 @@ template<typename Update>
 void
 MultiBitVectorIterator<Update>::strictSeek(uint32_t docId)
 {
-    for (updateLastValue(docId), _lastValue = _lastValue & checkTab(docId);
+    for (updateLastValue(docId), _lastValue=_lastValue & checkTab(docId);
          (_lastValue == 0) && __builtin_expect(! isAtEnd(), true);
          updateLastValue(_lastMaxDocIdLimit));
     if (__builtin_expect(!isAtEnd(), true)) {
@@ -118,6 +88,21 @@ MultiBitVectorIterator<Update>::strictSeek(uint32_t docId)
     }
 }
 
+struct And {
+    typedef BitWord::Word Word;
+    Word operator () (const Word a, const Word b) {
+        return a & b;
+    }
+    static bool isAnd() { return true; }
+};
+
+struct Or {
+    typedef BitWord::Word Word;
+    Word operator () (const Word a, const Word b) {
+        return a | b;
+    }
+    static bool isAnd() { return false; }
+};
 
 typedef MultiBitVectorIterator<And> AndBVIterator;
 typedef MultiBitVectorIteratorStrict<And> AndBVIteratorStrict;
@@ -151,15 +136,14 @@ bool canOptimize(const MultiSearch & s) {
 MultiBitVectorIteratorBase::MultiBitVectorIteratorBase(Children children) :
     MultiSearch(std::move(children)),
     _numDocs(std::numeric_limits<unsigned int>::max()),
-    _lastMaxDocIdLimit(0),
-    _lastMaxDocIdLimitRequireFetch(0),
     _lastValue(0),
+    _lastMaxDocIdLimit(0),
     _bvs()
 {
     _bvs.reserve(getChildren().size());
-    for (const auto & child : getChildren()) {
-        const auto * bv = static_cast<const BitVectorIterator *>(child.get());
-        _bvs.emplace_back(bv->getBitValues(), bv->isInverted());
+    for (size_t i(0); i < getChildren().size(); i++) {
+        const auto * bv = static_cast<const BitVectorIterator *>(getChildren()[i].get());
+        _bvs.emplace_back(reinterpret_cast<const Word *>(bv->getBitValues()), bv->isInverted());
         _numDocs = std::min(_numDocs, bv->getDocIdLimit());
     }
 }
@@ -171,7 +155,6 @@ MultiBitVectorIteratorBase::initRange(uint32_t beginId, uint32_t endId)
 {
     MultiSearch::initRange(beginId, endId);
     _lastMaxDocIdLimit = 0;
-    _lastMaxDocIdLimitRequireFetch = 0;
 }
 
 SearchIterator::UP
@@ -180,10 +163,9 @@ MultiBitVectorIteratorBase::andWith(UP filter, uint32_t estimate)
     (void) estimate;
     if (filter->isBitVector() && acceptExtraFilter()) {
         const auto & bv = static_cast<const BitVectorIterator &>(*filter);
-        _bvs.emplace_back(bv.getBitValues(), bv.isInverted());
+        _bvs.emplace_back(reinterpret_cast<const Word *>(bv.getBitValues()), bv.isInverted());
         insert(getChildren().size(), std::move(filter));
         _lastMaxDocIdLimit = 0;  // force reload
-        _lastMaxDocIdLimitRequireFetch = 0;
     }
     return filter;
 }
diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h
index 29e92584ffe..cde9ffcbfe5 100644
--- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h
+++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h
@@ -11,7 +11,7 @@ namespace search::queryeval {
 class MultiBitVectorIteratorBase : public MultiSearch, protected BitWord
 {
 public:
-    ~MultiBitVectorIteratorBase() override;
+    ~MultiBitVectorIteratorBase();
     void initRange(uint32_t beginId, uint32_t endId) override;
     void addUnpackIndex(size_t index) { _unpackInfo.add(index); }
     /**
@@ -20,21 +20,26 @@ public:
      */
     static SearchIterator::UP optimize(SearchIterator::UP parent);
 protected:
-    MultiBitVectorIteratorBase(Children hildren);
-    using MetaWord = std::pair<const void *, bool>;
+    MultiBitVectorIteratorBase(Children children);
+    class MetaWord {
+    public:
+        MetaWord(const Word * words, bool inverted) : _words(words), _inverted(inverted) { }
+        Word operator [] (uint32_t index) const { return _inverted ? ~_words[index] : _words[index]; }
+    private:
+        const Word * _words;
+        bool         _inverted;
+    };
 
     uint32_t                _numDocs;
-    uint32_t                _lastMaxDocIdLimit; // next documentid requiring recomputation.
-    uint32_t                _lastMaxDocIdLimitRequireFetch;
     Word                    _lastValue; // Last value computed
+    uint32_t                _lastMaxDocIdLimit; // next documentid requiring recomputation.
     std::vector<MetaWord>   _bvs;
 private:
     virtual bool acceptExtraFilter() const = 0;
     UP andWith(UP filter, uint32_t estimate) override;
     void doUnpack(uint32_t docid) override;
+    UnpackInfo _unpackInfo;
     static SearchIterator::UP optimizeMultiSearch(SearchIterator::UP parent);
-
-    UnpackInfo  _unpackInfo;
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h
index d37495e85da..79f987c740c 100644
--- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h
+++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h
@@ -17,7 +17,7 @@ template <typename FloatType>
 class SquaredEuclideanDistance : public DistanceFunction {
 public:
     SquaredEuclideanDistance()
-        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator())
+        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator())
     {}
     double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override {
         auto lhs_vector = lhs.typify<FloatType>();
@@ -60,7 +60,7 @@ template <typename FloatType>
 class AngularDistance : public DistanceFunction {
 public:
     AngularDistance()
-        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator())
+        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator())
     {}
     double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override {
         auto lhs_vector = lhs.typify<FloatType>();