diff options
25 files changed, 222 insertions, 278 deletions
diff --git a/config/src/tests/configparser/configparser.cpp b/config/src/tests/configparser/configparser.cpp index 7a66d831ee5..521df96af3a 100644 --- a/config/src/tests/configparser/configparser.cpp +++ b/config/src/tests/configparser/configparser.cpp @@ -108,14 +108,6 @@ TEST("require that escaped values are properly unescaped") { ASSERT_EQUAL("a\nb\rc\\d\"eBg", value); } -IGNORE_TEST("verify that locale affects double parsing") { // Failing on some CentOS based environments - std::vector<vespalib::string> payload; - setlocale(LC_NUMERIC, "nb_NO.UTF-8"); - payload.push_back("foo 3.14"); - ASSERT_EXCEPTION(ConfigParser::parse<double>("foo", payload), InvalidConfigException, "Value 3.14 is not a legal double"); - setlocale(LC_NUMERIC, "C"); -} - TEST("require that maps can be parsed") { writeFile("foo.cfg", "\nfooValue \"a\"\nfooMap{\"foo\"} 1336\nfooMap{\"bar\"} 1337\n"); diff --git a/configserver/src/main/sh/start-filedistribution b/configserver/src/main/sh/start-filedistribution index ecf04e9e035..df8dce34f13 100755 --- a/configserver/src/main/sh/start-filedistribution +++ b/configserver/src/main/sh/start-filedistribution @@ -78,5 +78,5 @@ if [ "$multitenant" = "true" ]; then export VESPA_LOG_CONTROL_DIR export VESPA_LOG_CONTROL_FILE cd ${ROOT} - vespa-runserver -r 30 -s filedistributor -p $PIDFILE_FILEDISTRIBUTOR -- ${ROOT}/sbin/filedistributor --configid $VESPA_CONFIG_ID + vespa-runserver -r 30 -s filedistributor -p $PIDFILE_FILEDISTRIBUTOR -- ${ROOT}/sbin/vespa-filedistributor --configid $VESPA_CONFIG_ID fi diff --git a/dist/post_install.sh b/dist/post_install.sh index 95f07b76f1c..abbfd55e13f 100755 --- a/dist/post_install.sh +++ b/dist/post_install.sh @@ -69,6 +69,10 @@ ln -s $PREFIX/lib/jars/zkfacade-jar-with-dependencies.jar $INSTALLPATH/conf/conf ln -s $PREFIX/conf/configserver-app/components $INSTALLPATH/lib/jars/config-models ln -s vespa-storaged-bin $INSTALLPATH/sbin/vespa-distributord-bin +# Temporary when renaming programs in filedistribution +ln -s vespa-filedistributor $INSTALLPATH/sbin/filedistributor +ln -s vespa-filedistributor-bin $INSTALLPATH/sbin/filedistributor-bin + # Temporary when renaming binaries in fnet ln -s vespa-rpc-info $INSTALLPATH/bin/rpc_info ln -s vespa-rpc-invoke $INSTALLPATH/bin/rpc_invoke diff --git a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h index 4a176120bc5..08b3e60a538 100644 --- a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h +++ b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h @@ -98,10 +98,12 @@ class WordFoldersTest : public Test } bool AccentRemovalTest() { + // Note last encoded characters encoded as octets to avoid interpreting following letters after xNN as part of the encoding of the character + // See http://en.cppreference.com/w/cpp/language/escape auto freefunction = [] (char * ptr) { free(ptr); }; auto input = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\x70\xFE\x21"), freefunction); - auto yelloutput = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBFAAAAAEAAAECEEEEIIIIDNOOOOOE\xD7OEUUUUEYTHssaaaaaeaaaeceeeeiiiidnoooooe\xF7oeuuuueythpth!"), + auto yelloutput = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\277AAAAAEAAAECEEEEIIIIDNOOOOOE\327OEUUUUEYTHssaaaaaeaaaeceeeeiiiidnoooooe\367oeuuuueythpth!"), freefunction); Fast_NormalizeWordFolder wordfolder; int len = wordfolder.FoldedSizeAsUTF8(input.get()); diff --git a/filedistribution/src/apps/filedistributor/.gitignore b/filedistribution/src/apps/filedistributor/.gitignore index a41b1963b70..89aaa70637e 100644 --- a/filedistribution/src/apps/filedistributor/.gitignore +++ b/filedistribution/src/apps/filedistributor/.gitignore @@ -1,2 +1,2 @@ /filedistributor -filedistributor-bin +vespa-filedistributor-bin diff --git a/filedistribution/src/apps/filedistributor/CMakeLists.txt b/filedistribution/src/apps/filedistributor/CMakeLists.txt index 1a7ef87bbac..42e47856208 100644 --- a/filedistribution/src/apps/filedistributor/CMakeLists.txt +++ b/filedistribution/src/apps/filedistributor/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_executable(filedistribution_filedistributor_app SOURCES filedistributor.cpp - OUTPUT_NAME filedistributor-bin + OUTPUT_NAME vespa-filedistributor-bin INSTALL sbin DEPENDS filedistribution_distributor diff --git a/filedistribution/src/apps/status/.gitignore b/filedistribution/src/apps/status/.gitignore index 6dc1c1fff5d..2105a3c7051 100644 --- a/filedistribution/src/apps/status/.gitignore +++ b/filedistribution/src/apps/status/.gitignore @@ -1,2 +1,2 @@ -/status-filedistribution +/vespa-status-filedistribution-bin filedistribution_status-filedistribution_app diff --git a/filedistribution/src/apps/status/CMakeLists.txt b/filedistribution/src/apps/status/CMakeLists.txt index 63e9752496e..e46e0c58d86 100644 --- a/filedistribution/src/apps/status/CMakeLists.txt +++ b/filedistribution/src/apps/status/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(filedistribution_status-filedistribution_app SOURCES status-filedistribution.cpp INSTALL bin - OUTPUT_NAME status-filedistribution + OUTPUT_NAME vespa-status-filedistribution-bin DEPENDS filedistribution_filedistributionmodel filedistribution_common diff --git a/filedistribution/src/apps/status/vespa-status-filedistribution.sh b/filedistribution/src/apps/status/vespa-status-filedistribution.sh index 104688a4883..1ba56d6c798 100644 --- a/filedistribution/src/apps/status/vespa-status-filedistribution.sh +++ b/filedistribution/src/apps/status/vespa-status-filedistribution.sh @@ -65,4 +65,4 @@ ROOT=${VESPA_HOME%/} ZKSTRING=$($ROOT/libexec/vespa/vespa-config.pl -zkstring) test -z "$VESPA_LOG_LEVEL" && VESPA_LOG_LEVEL=warning export VESPA_LOG_LEVEL -exec $ROOT/bin/status-filedistribution --zkstring "$ZKSTRING" $@ +exec $ROOT/bin/vespa-status-filedistribution-bin --zkstring "$ZKSTRING" $@ diff --git a/sample-apps/http-api-using-request-handlers-and-processors/pom.xml b/sample-apps/http-api-using-request-handlers-and-processors/pom.xml index 7ae8549c385..8141a0b91b1 100644 --- a/sample-apps/http-api-using-request-handlers-and-processors/pom.xml +++ b/sample-apps/http-api-using-request-handlers-and-processors/pom.xml @@ -6,7 +6,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.yahoo.demo</groupId> - <artifactId>sample-app</artifactId> + <artifactId>sample-app-http-api-processor</artifactId> <version>1.0.1</version> <packaging>container-plugin</packaging> <!-- Use Vespa packaging --> diff --git a/sample-apps/http-api-using-searcher/pom.xml b/sample-apps/http-api-using-searcher/pom.xml index 7ae8549c385..6dfee6026ee 100644 --- a/sample-apps/http-api-using-searcher/pom.xml +++ b/sample-apps/http-api-using-searcher/pom.xml @@ -6,7 +6,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.yahoo.demo</groupId> - <artifactId>sample-app</artifactId> + <artifactId>sample-app-http-api-searcher</artifactId> <version>1.0.1</version> <packaging>container-plugin</packaging> <!-- Use Vespa packaging --> diff --git a/sample-apps/pom.xml b/sample-apps/pom.xml index 91d1b7c7580..d39e3d1e13f 100644 --- a/sample-apps/pom.xml +++ b/sample-apps/pom.xml @@ -13,9 +13,7 @@ <module>basic-search-java</module> <module>blog-recommendation</module> <module>boolean-search</module> -<!-- <module>http-api-using-request-handlers-and-processors</module> <module>http-api-using-searcher</module> ---> </modules> </project> diff --git a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h index 96c3d7f3470..bd88c5e5901 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h +++ b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h @@ -2,10 +2,9 @@ #pragma once -#include <stddef.h> +#include <cstddef> -namespace search { -namespace attribute { +namespace search::attribute { class IAttributeVector; @@ -45,4 +44,3 @@ public: }; } -} diff --git a/searchcommon/src/vespa/searchcommon/common/range.h b/searchcommon/src/vespa/searchcommon/common/range.h index 5bcf2355eb9..f3e3156491a 100644 --- a/searchcommon/src/vespa/searchcommon/common/range.h +++ b/searchcommon/src/vespa/searchcommon/common/range.h @@ -5,10 +5,9 @@ #pragma once #include <limits> -#include <stdint.h> +#include <cstdint> -namespace search -{ +namespace search { template <typename T> class Range { @@ -30,4 +29,3 @@ private: using Int64Range = Range<int64_t>; } // namespace search - diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 6abf0ac5f27..1908d22d7b9 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -579,10 +579,10 @@ AttributeVector::SearchContext:: createIterator(fef::TermFieldMatchData *matchData, bool strict) { if (_plsc != NULL) { - SearchIterator::UP res = - _plsc->createPostingIterator(matchData, strict); - if (res.get() != NULL) + SearchIterator::UP res = _plsc->createPostingIterator(matchData, strict); + if (res) { return res; + } } return createFilterIterator(matchData, strict); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h index 2960d573098..fe96928dccd 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h @@ -7,8 +7,8 @@ #include <vespa/searchlib/datastore/datastore.h> #include <vespa/searchlib/util/memoryusage.h> #include <vespa/vespalib/util/array.h> -#include <set> #include <vespa/searchlib/btree/btree.h> +#include <set> #include <atomic> namespace vespalib { class asciistream; } diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp index c972e25a7cf..0b7430599b9 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp @@ -4,6 +4,7 @@ #include "multivalue.h" #include "multi_value_mapping.h" #include "postinglistattribute.h" +#include <vespa/searchlib/common/growablebitvector.h> #include <vespa/vespalib/util/array.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 9665872f8de..cb3dd844041 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -6,13 +6,11 @@ #include "postinglisttraits.h" #include "postingstore.h" #include "ipostinglistsearchcontext.h" -#include "attributevector.h" +#include <vespa/searchcommon/attribute/search_context_params.h> +#include <vespa/searchcommon/common/range.h> #include <vespa/vespalib/util/regexp.h> -#include <cstdlib> -namespace search { - -namespace attribute { +namespace search::attribute { /** * Search context helper for posting list attributes, used to instantiate @@ -22,11 +20,11 @@ namespace attribute { class PostingListSearchContext : public IPostingListSearchContext { protected: - typedef EnumPostingTree Dictionary; - typedef Dictionary::ConstIterator DictionaryConstIterator; - typedef Dictionary::FrozenView FrozenDictionary; - typedef EnumStoreBase::Index EnumIndex; - + using Dictionary = EnumPostingTree; + using DictionaryConstIterator = Dictionary::ConstIterator; + using FrozenDictionary = Dictionary::FrozenView; + using EnumIndex = EnumStoreBase::Index; + const FrozenDictionary _frozenDictionary; DictionaryConstIterator _lowerDictItr; DictionaryConstIterator _upperDictItr; @@ -36,22 +34,17 @@ protected: uint64_t _numValues; // attr.getStatus().getNumValues(); bool _hasWeight; bool _useBitVector; - search::datastore::EntryRef _pidx; - search::datastore::EntryRef _frozenRoot; // Posting list in tree form + datastore::EntryRef _pidx; + datastore::EntryRef _frozenRoot; // Posting list in tree form float _FSTC; // Filtering Search Time Constant float _PLSTC; // Posting List Search Time Constant const EnumStoreBase &_esb; uint32_t _minBvDocFreq; const GrowableBitVector *_gbv; // bitvector if _useBitVector has been set - - PostingListSearchContext(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const EnumStoreBase &esb, - uint32_t minBvDocFreq, - bool useBitVector); + + PostingListSearchContext(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, + const EnumStoreBase &esb, uint32_t minBvDocFreq, bool useBitVector); ~PostingListSearchContext(); @@ -96,13 +89,13 @@ template <class DataT> class PostingListSearchContextT : public PostingListSearchContext { protected: - typedef DataT DataType; - typedef PostingListTraits<DataType> Traits; - typedef typename Traits::PostingList PostingList; - typedef typename Traits::Posting Posting; - typedef std::vector<Posting> PostingVector; - typedef datastore::EntryRef EntryRef; - typedef typename PostingList::ConstIterator PostingConstIterator; + using DataType = DataT; + using Traits = PostingListTraits<DataType>; + using PostingList = typename Traits::PostingList; + using Posting = typename Traits::Posting; + using PostingVector = std::vector<Posting>; + using EntryRef = datastore::EntryRef; + using FrozenView = typename PostingList::BTreeType::FrozenView; const PostingList &_postingList; /* @@ -112,19 +105,14 @@ protected: std::unique_ptr<BitVector> _bitVector; bool _fetchPostingsDone; bool _arrayValid; - + static const long MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION = 100; static const long MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 20; static const long MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 10; - PostingListSearchContextT(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const PostingList &postingList, - const EnumStoreBase &esb, - uint32_t minBvCocFreq, - bool useBitVector); + PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, + bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + uint32_t minBvCocFreq, bool useBitVector); ~PostingListSearchContextT(); void lookupSingle(); @@ -133,16 +121,14 @@ protected: void fillBitVector(); PostingVector & - merge(PostingVector &v, PostingVector &temp, - const std::vector<size_t> & startPos) __attribute__((noinline)); + merge(PostingVector &v, PostingVector &temp, const std::vector<size_t> & startPos) __attribute__((noinline)); void fetchPostings(bool strict) override; // this will be called instead of the fetchPostings function in some cases - void diversify(bool forward, size_t wanted_hits, - const IAttributeVector &diversity_attr, size_t max_per_group, - size_t cutoff_groups, bool cutoff_strict); + void diversify(bool forward, size_t wanted_hits, const IAttributeVector &diversity_attr, + size_t max_per_group, size_t cutoff_groups, bool cutoff_strict); - queryeval::SearchIterator::UP + std::unique_ptr<queryeval::SearchIterator> createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) override; unsigned int singleHits() const; @@ -155,9 +141,9 @@ template <class DataT> class PostingListFoldedSearchContextT : public PostingListSearchContextT<DataT> { protected: - typedef PostingListSearchContextT<DataT> Parent; - typedef typename Parent::Dictionary Dictionary; - typedef typename Parent::PostingList PostingList; + using Parent = PostingListSearchContextT<DataT>; + using Dictionary = typename Parent::Dictionary; + using PostingList = typename Parent::PostingList; using Parent::_lowerDictItr; using Parent::_uniqueValues; using Parent::_postingList; @@ -165,14 +151,9 @@ protected: using Parent::countHits; using Parent::singleHits; - PostingListFoldedSearchContextT(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const PostingList &postingList, - const EnumStoreBase &esb, - uint32_t minBvCocFreq, - bool useBitVector); + PostingListFoldedSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, + bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + uint32_t minBvCocFreq, bool useBitVector); unsigned int approximateHits() const override; }; @@ -183,12 +164,12 @@ class PostingSearchContext: public BaseSC, public BaseSC2 { public: - typedef typename AttrT::EnumStore EnumStore; + using EnumStore = typename AttrT::EnumStore; using QueryTermSimpleUP = std::unique_ptr<QueryTermSimple>; protected: const AttrT &_toBeSearched; const EnumStore &_enumStore; - + PostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const AttrT &toBeSearched); ~PostingSearchContext(); }; @@ -198,14 +179,11 @@ class StringPostingSearchContext : public PostingSearchContext<BaseSC, PostingListFoldedSearchContextT<DataT>, AttrT> { private: - typedef PostingListTraits<DataT> AggregationTraits; - typedef typename AggregationTraits::PostingList PostingList; - typedef typename PostingList::Iterator PostingIterator; - typedef typename PostingList::ConstIterator PostingConstIterator; - typedef PostingSearchContext<BaseSC, PostingListFoldedSearchContextT<DataT>, AttrT> Parent; - typedef typename Parent::EnumStore EnumStore; - typedef typename EnumStore::FoldedComparatorType FoldedComparatorType; - typedef vespalib::Regexp Regexp; + using AggregationTraits = PostingListTraits<DataT>; + using PostingList = typename AggregationTraits::PostingList; + using Parent = PostingSearchContext<BaseSC, PostingListFoldedSearchContextT<DataT>, AttrT>; + using FoldedComparatorType = typename Parent::EnumStore::FoldedComparatorType; + using Regexp = vespalib::Regexp; using QueryTermSimpleUP = typename Parent::QueryTermSimpleUP; using Parent::_toBeSearched; using Parent::_enumStore; @@ -225,10 +203,7 @@ private: typedef PostingSearchContext<BaseSC, PostingListSearchContextT<DataT>, AttrT> Parent; typedef PostingListTraits<DataT> AggregationTraits; typedef typename AggregationTraits::PostingList PostingList; - typedef typename PostingList::Iterator PostingIterator; - typedef typename PostingList::ConstIterator PostingConstIterator; - typedef typename Parent::EnumStore EnumStore; - typedef typename EnumStore::ComparatorType ComparatorType; + typedef typename Parent::EnumStore::ComparatorType ComparatorType; typedef typename AttrT::T BaseType; using Params = attribute::SearchContextParams; using QueryTermSimpleUP = typename Parent::QueryTermSimpleUP; @@ -237,7 +212,7 @@ private: using Parent::_toBeSearched; using Parent::_enumStore; Params _params; - + void getIterators(bool shouldApplyRangeLimit); bool valid() const override { return this->isValid(); } @@ -269,8 +244,8 @@ public: NumericPostingSearchContext(QueryTermSimpleUP qTerm, const Params & params, const AttrT &toBeSearched); const Params ¶ms() const { return _params; } }; - - + + template <typename BaseSC, typename BaseSC2, typename AttrT> PostingSearchContext<BaseSC, BaseSC2, AttrT>:: PostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const AttrT &toBeSearched) @@ -305,7 +280,7 @@ StringPostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const Att // after benchmarking prefix search performance on single, array, and weighted set fast-search string attributes // with 1M values the following constant has been derived: this->_PLSTC = 0.000000; - + if (this->valid()) { if (this->isPrefix()) { FoldedComparatorType comp(_enumStore, this->queryTerm().getTerm(), true); @@ -386,7 +361,4 @@ extern template class PostingListSearchContextT<int32_t>; extern template class PostingListFoldedSearchContextT<btree::BTreeNoLeafData>; extern template class PostingListFoldedSearchContextT<int32_t>; -} // namespace attribute - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 270ae161f4c..573af5f40bd 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -5,28 +5,23 @@ #include "dociditerator.h" #include "attributeiterators.h" #include "diversity.h" +#include "postingstore.hpp" #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/common/bitvectoriterator.h> +#include <vespa/searchlib/common/growablebitvector.h> -namespace search { -using queryeval::EmptySearch; -using queryeval::SearchIterator; +using search::queryeval::EmptySearch; +using search::queryeval::SearchIterator; -namespace attribute { +namespace search::attribute { template <typename DataT> PostingListSearchContextT<DataT>:: -PostingListSearchContextT(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const PostingList &postingList, - const EnumStoreBase &esb, - uint32_t minBvDocFreq, - bool useBitVector) - : PostingListSearchContext(dictionary, docIdLimit, numValues, hasWeight, - esb, minBvDocFreq, useBitVector), +PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, + const PostingList &postingList, const EnumStoreBase &esb, + uint32_t minBvDocFreq, bool useBitVector) + : PostingListSearchContext(dictionary, docIdLimit, numValues, hasWeight, esb, minBvDocFreq, useBitVector), _postingList(postingList), _array(), _bitVector(), @@ -56,9 +51,7 @@ PostingListSearchContextT<DataT>::lookupSingle() } else { _pidx = bve->_tree; if (_pidx.valid()) { - typename PostingList::BTreeType::FrozenView - frozenView(_postingList.getTreeEntry(_pidx)-> - getFrozenView(_postingList.getAllocator())); + auto frozenView = _postingList.getTreeEntry(_pidx)->getFrozenView(_postingList.getAllocator()); _frozenRoot = frozenView.getRoot(); if (!_frozenRoot.valid()) { _pidx = datastore::EntryRef(); @@ -68,9 +61,7 @@ PostingListSearchContextT<DataT>::lookupSingle() } } } else { - typename PostingList::BTreeType::FrozenView - frozenView(_postingList.getTreeEntry(_pidx)-> - getFrozenView(_postingList.getAllocator())); + auto frozenView = _postingList.getTreeEntry(_pidx)->getFrozenView(_postingList.getAllocator()); _frozenRoot = frozenView.getRoot(); if (!_frozenRoot.valid()) { _pidx = datastore::EntryRef(); @@ -106,8 +97,8 @@ PostingListSearchContextT<DataT>::fillArray(size_t numDocs) for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { if (useThis(it)) { _postingList.foreach_frozen(it.getData(), - [&](uint32_t key, const DataT &data) - { _array.push_back(Posting(key, data)); + [&](uint32_t key, const DataT &data) { + _array.push_back(Posting(key, data)); }); startPos.push_back(_array.size()); } @@ -130,8 +121,8 @@ PostingListSearchContextT<DataT>::fillBitVector() for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { if (useThis(it)) { _postingList.foreach_frozen_key(it.getData(), - [&](uint32_t key) - { if (key < limit) { + [&](uint32_t key) { + if (key < limit) { bv.setBit(key); } }); @@ -193,9 +184,8 @@ PostingListSearchContextT<DataT>::fetchPostings(bool strict) template <typename DataT> void -PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, - const IAttributeVector &diversity_attr, size_t max_per_group, - size_t cutoff_groups, bool cutoff_strict) +PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, const IAttributeVector &diversity_attr, + size_t max_per_group, size_t cutoff_groups, bool cutoff_strict) { assert(!_fetchPostingsDone); _fetchPostingsDone = true; @@ -203,9 +193,8 @@ PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, _array.reserve(wanted_hits); std::vector<size_t> fragments; fragments.push_back(0); - diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, - diversity_attr, max_per_group, cutoff_groups, cutoff_strict, - _array, fragments); + diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, diversity_attr, + max_per_group, cutoff_groups, cutoff_strict, _array, fragments); if (fragments.size() > 2) { PostingVector temp(_array.size()); _array.swap(merge(_array, temp, fragments)); @@ -226,12 +215,14 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) if (_arrayValid || (_bitVector.get() != nullptr)) { // synthetic results are available if (!_array.empty()) { assert(_arrayValid); - typedef DocIdIterator<Posting> DocIt; + using DocIt = DocIdIterator<Posting>; DocIt postings; postings.set(&_array[0], &_array[_array.size()]); - return (_postingList._isFilter) - ? SearchIterator::UP(new FilterAttributePostingListIteratorT<DocIt>(matchData, postings)) - : SearchIterator::UP(new AttributePostingListIteratorT<DocIt>(_hasWeight, matchData, postings)); + if (_postingList._isFilter) { + return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(matchData, postings); + } else { + return std::make_unique<AttributePostingListIteratorT<DocIt>>(_hasWeight, matchData, postings); + } } if (_arrayValid) { return SearchIterator::UP(new EmptySearch()); @@ -251,19 +242,24 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) if (!_frozenRoot.valid()) { uint32_t clusterSize = _postingList.getClusterSize(_pidx); assert(clusterSize != 0); - typedef DocIdMinMaxIterator<Posting> DocIt; + using DocIt = DocIdMinMaxIterator<Posting>; DocIt postings; const Posting *array = postingList.getKeyDataEntry(_pidx, clusterSize); postings.set(array, array + clusterSize); - return (postingList._isFilter) - ? SearchIterator::UP(new FilterAttributePostingListIteratorT<DocIt>(matchData, postings)) - : SearchIterator::UP(new AttributePostingListIteratorT<DocIt>(_hasWeight, matchData, postings)); + if (postingList._isFilter) { + return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(matchData, postings); + } else { + return std::make_unique<AttributePostingListIteratorT<DocIt>>(_hasWeight, matchData, postings); + } } typename PostingList::BTreeType::FrozenView frozen(_frozenRoot, postingList.getAllocator()); - return (_postingList._isFilter) - ? SearchIterator::UP(new FilterAttributePostingListIteratorT<PostingConstIterator> (matchData, frozen.getRoot(), frozen.getAllocator())) - : SearchIterator::UP(new AttributePostingListIteratorT<PostingConstIterator> (_hasWeight, matchData, frozen.getRoot(), frozen.getAllocator())); + using DocIt = typename PostingList::ConstIterator; + if (_postingList._isFilter) { + return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(matchData, frozen.getRoot(), frozen.getAllocator()); + } else { + return std::make_unique<AttributePostingListIteratorT<DocIt>> (_hasWeight, matchData, frozen.getRoot(), frozen.getAllocator()); + } } // returning nullptr will trigger fallback to filter iterator return SearchIterator::UP(); @@ -300,12 +296,10 @@ PostingListSearchContextT<DataT>::approximateHits() const if (this->fallbackToFiltering()) { numHits = _docIdLimit; } else if (_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION) { - if ((_uniqueValues * - MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > - static_cast<int>(_docIdLimit)) || - (this->calculateApproxNumHits() * - MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > - _docIdLimit)) { + if ((_uniqueValues * MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > static_cast<int>(_docIdLimit)) || + (this->calculateApproxNumHits() * MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > _docIdLimit) || + (_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION*10)) + { numHits = this->calculateApproxNumHits(); } else { // XXX: Unsafe @@ -346,16 +340,10 @@ PostingListSearchContextT<DataT>::applyRangeLimit(int rangeLimit) template <typename DataT> PostingListFoldedSearchContextT<DataT>:: -PostingListFoldedSearchContextT(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const PostingList &postingList, - const EnumStoreBase &esb, - uint32_t minBvDocFreq, - bool useBitVector) - : Parent(dictionary, docIdLimit, numValues, hasWeight, postingList, - esb, minBvDocFreq, useBitVector) +PostingListFoldedSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, + bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + uint32_t minBvDocFreq, bool useBitVector) + : Parent(dictionary, docIdLimit, numValues, hasWeight, postingList, esb, minBvDocFreq, useBitVector) { } @@ -379,7 +367,4 @@ PostingListFoldedSearchContextT<DataT>::approximateHits() const return numHits; } -} // namespace attribute - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/attribute/postinglisttraits.h b/searchlib/src/vespa/searchlib/attribute/postinglisttraits.h index c2504f779a1..d619751d451 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglisttraits.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglisttraits.h @@ -4,11 +4,9 @@ #include <vespa/searchlib/btree/btreestore.h> -namespace search -{ +namespace search { -namespace attribute -{ +namespace attribute { template <typename DataT> class PostingListTraits; template <typename DataT> class PostingStore; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index a21884bca4a..e1b0cf24305 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -3,12 +3,11 @@ #include "postingstore.h" #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> +#include <vespa/searchlib/common/growablebitvector.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcommon/attribute/status.h> -namespace search { - -namespace attribute { +namespace search::attribute { using btree::BTreeNoLeafData; @@ -635,6 +634,4 @@ template class PostingStore<BTreeNoLeafData>; template class PostingStore<int32_t>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index 9aae723db73..9c34344451a 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -5,12 +5,13 @@ #include "postinglisttraits.h" #include "enumstorebase.h" #include <set> -#include <vespa/searchlib/common/bitvector.h> -#include <vespa/searchlib/common/growablebitvector.h> namespace search { + class BitVector; + class GrowableBitVector; +} -namespace attribute { +namespace search::attribute { class Status; class Config; @@ -43,9 +44,9 @@ public: uint32_t _maxBvDocFreq; // Greater than or equal to this ==> create bv protected: std::set<uint32_t> _bvs; // Current bitvectors - EnumPostingTree &_dict; - Status &_status; - uint64_t _bvExtraBytes; + EnumPostingTree &_dict; + Status &_status; + uint64_t _bvExtraBytes; static constexpr uint32_t BUFFERTYPE_BITVECTOR = 9u; @@ -197,90 +198,4 @@ PostingStore<int32_t>::bitVectorWeight() return 1; } -template <typename DataT> -template <typename FunctionType> -void -PostingStore<DataT>::foreach_frozen_key(EntryRef ref, FunctionType func) const -{ - if (!ref.valid()) - return; - RefType iRef(ref); - uint32_t typeId = getTypeId(iRef); - uint32_t clusterSize = getClusterSize(typeId); - if (clusterSize == 0) { - if (isBitVector(typeId)) { - const BitVectorEntry *bve = getBitVectorEntry(iRef); - EntryRef ref2(bve->_tree); - RefType iRef2(ref2); - if (iRef2.valid()) { - assert(isBTree(iRef2)); - const BTreeType *tree = getTreeEntry(iRef2); - _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); - } else { - const BitVector *bv = bve->_bv.get(); - uint32_t docIdLimit = bv->size(); - uint32_t docId = bv->getFirstTrueBit(1); - while (docId < docIdLimit) { - func(docId); - docId = bv->getNextTrueBit(docId + 1); - } - } - } else { - assert(isBTree(typeId)); - const BTreeType *tree = getTreeEntry(iRef); - _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); - } - } else { - const KeyDataType *p = getKeyDataEntry(iRef, clusterSize); - const KeyDataType *pe = p + clusterSize; - for (; p != pe; ++p) { - func(p->_key); - } - } -} - - -template <typename DataT> -template <typename FunctionType> -void -PostingStore<DataT>::foreach_frozen(EntryRef ref, FunctionType func) const -{ - if (!ref.valid()) - return; - RefType iRef(ref); - uint32_t typeId = getTypeId(iRef); - uint32_t clusterSize = getClusterSize(typeId); - if (clusterSize == 0) { - if (isBitVector(typeId)) { - const BitVectorEntry *bve = getBitVectorEntry(iRef); - EntryRef ref2(bve->_tree); - RefType iRef2(ref2); - if (iRef2.valid()) { - assert(isBTree(iRef2)); - const BTreeType *tree = getTreeEntry(iRef2); - _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); - } else { - const BitVector *bv = bve->_bv.get(); - uint32_t docIdLimit = bv->size(); - uint32_t docId = bv->getFirstTrueBit(1); - while (docId < docIdLimit) { - func(docId, bitVectorWeight()); - docId = bv->getNextTrueBit(docId + 1); - } - } - } else { - const BTreeType *tree = getTreeEntry(iRef); - _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); - } - } else { - const KeyDataType *p = getKeyDataEntry(iRef, clusterSize); - const KeyDataType *pe = p + clusterSize; - for (; p != pe; ++p) { - func(p->_key, p->getData()); - } - } } - -} // namespace attribute - -} // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.hpp b/searchlib/src/vespa/searchlib/attribute/postingstore.hpp new file mode 100644 index 00000000000..6680fc86f71 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.hpp @@ -0,0 +1,90 @@ +#pragma once + +#include "postingstore.h" +#include <vespa/searchlib/common/growablebitvector.h> + +namespace search::attribute { + +template<typename DataT> +template<typename FunctionType> +void +PostingStore<DataT>::foreach_frozen_key(EntryRef ref, FunctionType func) const { + if (!ref.valid()) + return; + RefType iRef(ref); + uint32_t typeId = getTypeId(iRef); + uint32_t clusterSize = getClusterSize(typeId); + if (clusterSize == 0) { + if (isBitVector(typeId)) { + const BitVectorEntry *bve = getBitVectorEntry(iRef); + EntryRef ref2(bve->_tree); + RefType iRef2(ref2); + if (iRef2.valid()) { + assert(isBTree(iRef2)); + const BTreeType *tree = getTreeEntry(iRef2); + _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); + } else { + const BitVector *bv = bve->_bv.get(); + uint32_t docIdLimit = bv->size(); + uint32_t docId = bv->getFirstTrueBit(1); + while (docId < docIdLimit) { + func(docId); + docId = bv->getNextTrueBit(docId + 1); + } + } + } else { + assert(isBTree(typeId)); + const BTreeType *tree = getTreeEntry(iRef); + _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); + } + } else { + const KeyDataType *p = getKeyDataEntry(iRef, clusterSize); + const KeyDataType *pe = p + clusterSize; + for (; p != pe; ++p) { + func(p->_key); + } + } +} + + +template<typename DataT> +template<typename FunctionType> +void +PostingStore<DataT>::foreach_frozen(EntryRef ref, FunctionType func) const { + if (!ref.valid()) + return; + RefType iRef(ref); + uint32_t typeId = getTypeId(iRef); + uint32_t clusterSize = getClusterSize(typeId); + if (clusterSize == 0) { + if (isBitVector(typeId)) { + const BitVectorEntry *bve = getBitVectorEntry(iRef); + EntryRef ref2(bve->_tree); + RefType iRef2(ref2); + if (iRef2.valid()) { + assert(isBTree(iRef2)); + const BTreeType *tree = getTreeEntry(iRef2); + _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); + } else { + const BitVector *bv = bve->_bv.get(); + uint32_t docIdLimit = bv->size(); + uint32_t docId = bv->getFirstTrueBit(1); + while (docId < docIdLimit) { + func(docId, bitVectorWeight()); + docId = bv->getNextTrueBit(docId + 1); + } + } + } else { + const BTreeType *tree = getTreeEntry(iRef); + _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); + } + } else { + const KeyDataType *p = getKeyDataEntry(iRef, clusterSize); + const KeyDataType *pe = p + clusterSize; + for (; p != pe; ++p) { + func(p->_key, p->getData()); + } + } +} + +} diff --git a/searchlib/src/vespa/searchlib/btree/btreestore.h b/searchlib/src/vespa/searchlib/btree/btreestore.h index e8153f94363..c22d2dd3f55 100644 --- a/searchlib/src/vespa/searchlib/btree/btreestore.h +++ b/searchlib/src/vespa/searchlib/btree/btreestore.h @@ -10,11 +10,7 @@ #include <vespa/searchlib/datastore/datastore.h> #include <vespa/searchlib/datastore/handle.h> -namespace search -{ - -namespace btree -{ +namespace search::btree { template <typename KeyT, typename DataT, @@ -508,8 +504,6 @@ extern template class BTreeStore<uint32_t, int32_t, BTreeDefaultTraits, MinMaxAggrCalc>; -} // namespace btree - -} // namespace search +} diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index cbd7bd3d334..9c0aced2319 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -11,7 +11,7 @@ vespa_install_script(src/start-cbinaries.sh vespa-transactionlog-inspect bin) vespa_install_script(src/start-cbinaries.sh vespa-vds-disktool bin) vespa_install_script(src/start-cbinaries.sh vespa-distributord sbin) vespa_install_script(src/start-cbinaries.sh vespa-fdispatch sbin) -vespa_install_script(src/start-cbinaries.sh filedistributor sbin) +vespa_install_script(src/start-cbinaries.sh vespa-filedistributor sbin) vespa_install_script(src/start-cbinaries.sh vespa-proton sbin) vespa_install_script(src/start-cbinaries.sh vespa-storaged sbin) |