diff options
Diffstat (limited to 'searchlib')
5 files changed, 42 insertions, 27 deletions
diff --git a/searchlib/src/tests/sortspec/multilevelsort.cpp b/searchlib/src/tests/sortspec/multilevelsort.cpp index c65683a1754..5f35cf65711 100644 --- a/searchlib/src/tests/sortspec/multilevelsort.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort.cpp @@ -1,7 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("multilevelsort_test"); #include <vespa/searchlib/common/sortresults.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributeguard.h> @@ -14,6 +12,8 @@ LOG_SETUP("multilevelsort_test"); #include <vespa/vespalib/testkit/testapp.h> #include <map> #include <sstream> +#include <vespa/log/log.h> +LOG_SETUP("multilevelsort_test"); using namespace search; @@ -253,7 +253,7 @@ MultilevelSortTest::sortAndCheck(const std::vector<Spec> &spec, uint32_t num, vespalib::Clock clock; vespalib::Doom doom(clock, std::numeric_limits<long>::max()); search::uca::UcaConverterFactory ucaFactory; - FastS_SortSpec sorter(doom, ucaFactory, _sortMethod); + FastS_SortSpec sorter(nullptr, doom, ucaFactory, _sortMethod); // init sorter with sort data for(uint32_t i = 0; i < spec.size(); ++i) { AttributeGuard ag; diff --git a/searchlib/src/vespa/searchlib/common/idocumentmetastore.h b/searchlib/src/vespa/searchlib/common/idocumentmetastore.h index 56c7baeb2ec..9b59c6b36eb 100644 --- a/searchlib/src/vespa/searchlib/common/idocumentmetastore.h +++ b/searchlib/src/vespa/searchlib/common/idocumentmetastore.h @@ -28,8 +28,7 @@ struct DocumentMetaData { bucketId(), gid(), removed(false) - { - } + { } DocumentMetaData(DocId lid_, storage::spi::Timestamp timestamp_, @@ -40,8 +39,7 @@ struct DocumentMetaData { bucketId(bucketId_), gid(gid_), removed(false) - { - } + { } DocumentMetaData(DocId lid_, storage::spi::Timestamp timestamp_, @@ -53,8 +51,7 @@ struct DocumentMetaData { bucketId(bucketId_), gid(gid_), removed(removed_) - { - } + { } bool valid() const { return lid != 0 && timestamp != 0 && bucketId.isSet(); diff --git a/searchlib/src/vespa/searchlib/common/lid_usage_stats.h b/searchlib/src/vespa/searchlib/common/lid_usage_stats.h index ced4bc36c8d..133f31ad55b 100644 --- a/searchlib/src/vespa/searchlib/common/lid_usage_stats.h +++ b/searchlib/src/vespa/searchlib/common/lid_usage_stats.h @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/bucket/bucketid.h> -#include <persistence/spi/types.h> +#include <stdint.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index 0b430369e96..155929b661b 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -8,6 +8,8 @@ #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/vespalib/util/array.h> +#include <vespa/document/base/globalid.h> +#include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/log/log.h> LOG_SETUP(".search.attribute.sortresults"); @@ -17,6 +19,8 @@ using search::common::SortInfo; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; using vespalib::alloc::Alloc; +using namespace vespalib; + namespace { constexpr size_t MMAP_LIMIT = 0x2000000; @@ -25,7 +29,7 @@ template<typename T> class RadixHelper { public: - typedef vespalib::convertForSort<T, true> C; + typedef convertForSort<T, true> C; inline typename C::UIntType operator()(typename C::InputType v) const { return C::convert(v); @@ -225,7 +229,7 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) size_t variableWidth = 0; for (auto iter = _vectors.begin(); iter != _vectors.end(); ++iter) { if (iter->_type >= ASC_DOCID) { // doc id - fixedWidth += 4; + fixedWidth += (_metaStore != nullptr) ? 8 : 4; }else if (iter->_type >= ASC_RANK) { // rank value fixedWidth += sizeof(search::HitRank); } else { @@ -244,6 +248,7 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) _sortDataArray.resize(n); + document::GlobalId gid; for (uint32_t i(0), idx(0); (i < n) && !_doom.doom(); ++i) { uint32_t len = 0; for (auto iter = _vectors.begin(); iter != _vectors.end(); ++iter) { @@ -254,19 +259,31 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) do { switch (iter->_type) { case ASC_DOCID: - vespalib::serializeForSort<vespalib::convertForSort<uint32_t, true> >(hits[i].getDocId(), mySortData); - written = sizeof(hits->_docId); + if (_metaStore) { + _metaStore->getGidEvenIfMoved(hits[i].getDocId(), gid); + serializeForSort<convertForSort<uint64_t, true> >(*reinterpret_cast<const uint64_t *>(gid.get()), mySortData); + written = sizeof(uint64_t); + } else { + serializeForSort<convertForSort<uint32_t, true> >(hits[i].getDocId(), mySortData); + written = sizeof(hits->_docId); + } break; case DESC_DOCID: - vespalib::serializeForSort<vespalib::convertForSort<uint32_t, false> >(hits[i].getDocId(), mySortData); - written = sizeof(hits->_docId); + if (_metaStore) { + _metaStore->getGidEvenIfMoved(hits[i].getDocId(), gid); + serializeForSort<convertForSort<uint64_t, false> >(*reinterpret_cast<const uint64_t *>(gid.get()), mySortData); + written = sizeof(uint64_t); + } else { + serializeForSort<convertForSort<uint32_t, false> >(hits[i].getDocId(), mySortData); + written = sizeof(hits->_docId); + } break; case ASC_RANK: - vespalib::serializeForSort<vespalib::convertForSort<search::HitRank, true> >(hits[i]._rankValue, mySortData); + serializeForSort<convertForSort<search::HitRank, true> >(hits[i]._rankValue, mySortData); written = sizeof(hits->_rankValue); break; case DESC_RANK: - vespalib::serializeForSort<vespalib::convertForSort<search::HitRank, false> >(hits[i]._rankValue, mySortData); + serializeForSort<convertForSort<search::HitRank, false> >(hits[i]._rankValue, mySortData); written = sizeof(hits->_rankValue); break; case ASC_VECTOR: @@ -295,14 +312,14 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) } -FastS_SortSpec::FastS_SortSpec(const vespalib::Doom & doom, const ConverterFactory & ucaFactory, int method) : +FastS_SortSpec::FastS_SortSpec(const search::IDocumentMetaStore * metaStore, const Doom & doom, const ConverterFactory & ucaFactory, int method) : + _metaStore(metaStore), _doom(doom), _ucaFactory(ucaFactory), _method(method), _sortSpec(), _vectors() -{ -} +{ } FastS_SortSpec::~FastS_SortSpec() @@ -312,7 +329,7 @@ FastS_SortSpec::~FastS_SortSpec() bool -FastS_SortSpec::Init(const vespalib::string & sortStr, IAttributeContext & vecMan) +FastS_SortSpec::Init(const string & sortStr, IAttributeContext & vecMan) { LOG(spam, "sortStr = %s", sortStr.c_str()); bool retval(true); @@ -489,7 +506,7 @@ FastS_SortSpec::sortResults(RankedHit a[], uint32_t n, uint32_t topn) } else if (_method == 1) { std::sort(sortData, sortData + n, StdSortDataCompare(&_binarySortData[0])); } else { - vespalib::Array<uint32_t> radixScratchPad(n, Alloc::alloc(0, MMAP_LIMIT)); + Array<uint32_t> radixScratchPad(n, Alloc::alloc(0, MMAP_LIMIT)); search::radix_sort(SortDataRadix(&_binarySortData[0]), StdSortDataCompare(&_binarySortData[0]), SortDataEof(), 1, sortData, n, &radixScratchPad[0], 0, 96, topn); } for (uint32_t i(0), m(_sortDataArray.size()); i < m; ++i) { diff --git a/searchlib/src/vespa/searchlib/common/sortresults.h b/searchlib/src/vespa/searchlib/common/sortresults.h index 53b0c4b0e8d..e4d8a4d6510 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.h +++ b/searchlib/src/vespa/searchlib/common/sortresults.h @@ -15,6 +15,7 @@ #define PREFETCH 64 #define INSERT_SORT_LEVEL 80 +namespace search { class IDocumentMetaStore; } /** * Sort the given array of results. * @@ -122,6 +123,7 @@ private: typedef vespalib::Array<uint8_t> BinarySortData; typedef vespalib::Array<SortData> SortDataArray; using ConverterFactory = search::common::ConverterFactory; + const search::IDocumentMetaStore * _metaStore; vespalib::Doom _doom; const ConverterFactory & _ucaFactory; int _method; @@ -137,7 +139,7 @@ private: public: FastS_SortSpec(const FastS_SortSpec &) = delete; FastS_SortSpec & operator = (const FastS_SortSpec &) = delete; - FastS_SortSpec(const vespalib::Doom & doom, const ConverterFactory & ucaFactory, int method=2); + FastS_SortSpec(const search::IDocumentMetaStore * metaStore, const vespalib::Doom & doom, const ConverterFactory & ucaFactory, int method=2); virtual ~FastS_SortSpec(); std::pair<const char *, size_t> getSortRef(size_t i) const { @@ -145,8 +147,8 @@ public: _sortDataArray[i]._len); } bool Init(const vespalib::string & sortSpec, search::attribute::IAttributeContext & vecMan); - virtual bool completeSort() const { return true; } - virtual void sortResults(search::RankedHit a[], uint32_t n, uint32_t topn); + bool completeSort() const override { return true; } + void sortResults(search::RankedHit a[], uint32_t n, uint32_t topn) override; uint32_t getSortDataSize(uint32_t offset, uint32_t n); void copySortData(uint32_t offset, uint32_t n, uint32_t *idx, char *buf); void freeSortData(); |