diff options
Diffstat (limited to 'searchlib')
70 files changed, 456 insertions, 640 deletions
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 6d1695155a8..66408b1d7d7 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -173,14 +173,14 @@ vespa_define_module( src/tests/index/doctypebuilder src/tests/indexmetainfo src/tests/ld-library-path - src/tests/memoryindex/compact_document_words_store + src/tests/memoryindex/compact_words_store src/tests/memoryindex/datastore - src/tests/memoryindex/document_remover - src/tests/memoryindex/documentinverter + src/tests/memoryindex/document_inverter src/tests/memoryindex/field_index - src/tests/memoryindex/fieldinverter - src/tests/memoryindex/memoryindex - src/tests/memoryindex/urlfieldinverter + src/tests/memoryindex/field_index_remover + src/tests/memoryindex/field_inverter + src/tests/memoryindex/memory_index + src/tests/memoryindex/url_field_inverter src/tests/nativerank src/tests/nearsearch src/tests/postinglistbm diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index b911284a1b4..2ef8448db8b 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index fb6535c4a70..24146e516a0 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -12,9 +12,9 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore b/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore deleted file mode 100644 index 3ad290f1731..00000000000 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_compact_document_words_store_test_app diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt deleted file mode 100644 index a5c8e0b2d14..00000000000 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_compact_document_words_store_test_app TEST - SOURCES - compact_document_words_store_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_compact_document_words_store_test_app COMMAND searchlib_compact_document_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_words_store/.gitignore b/searchlib/src/tests/memoryindex/compact_words_store/.gitignore new file mode 100644 index 00000000000..9f9acb50adc --- /dev/null +++ b/searchlib/src/tests/memoryindex/compact_words_store/.gitignore @@ -0,0 +1 @@ +searchlib_compact_words_store_test_app diff --git a/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt new file mode 100644 index 00000000000..ee31ef7c7aa --- /dev/null +++ b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_compact_words_store_test_app TEST + SOURCES + compact_words_store_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_compact_words_store_test_app COMMAND searchlib_compact_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/compact_document_words_store_test.cpp b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp index 6e22a4e5ff6..04d66396f90 100644 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/compact_document_words_store_test.cpp +++ b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp @@ -2,7 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/datastore/entryref.h> -#include <vespa/searchlib/memoryindex/compact_document_words_store.h> +#include <vespa/searchlib/memoryindex/compact_words_store.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <iostream> @@ -12,8 +12,8 @@ using namespace search; using namespace search::datastore; using namespace search::memoryindex; -typedef CompactDocumentWordsStore::Builder Builder; -typedef CompactDocumentWordsStore::Iterator Iterator; +typedef CompactWordsStore::Builder Builder; +typedef CompactWordsStore::Iterator Iterator; typedef Builder::WordRefVector WordRefVector; const EntryRef w1(1); @@ -53,7 +53,7 @@ toStr(Iterator itr) struct SingleFixture { - CompactDocumentWordsStore _store; + CompactWordsStore _store; SingleFixture() : _store() { _store.insert(Builder(d1).insert(w1).insert(w2).insert(w3)); } @@ -61,7 +61,7 @@ struct SingleFixture struct MultiFixture { - CompactDocumentWordsStore _store; + CompactWordsStore _store; MultiFixture() : _store() { _store.insert(Builder(d1).insert(w1)); _store.insert(Builder(d2).insert(w2)); @@ -100,7 +100,7 @@ TEST_F("require that documents can be removed and re-inserted", MultiFixture) TEST("require that a lot of words can be inserted, retrieved and removed") { - CompactDocumentWordsStore store; + CompactWordsStore store; for (uint32_t docId = 0; docId < 50; ++docId) { Builder b(docId); for (uint32_t wordRef = 0; wordRef < 20000; ++wordRef) { @@ -125,9 +125,9 @@ TEST("require that a lot of words can be inserted, retrieved and removed") TEST("require that initial memory usage is reported") { - CompactDocumentWordsStore store; - CompactDocumentWordsStore::DocumentWordsMap docs; - CompactDocumentWordsStore::Store internalStore; + CompactWordsStore store; + CompactWordsStore::DocumentWordsMap docs; + CompactWordsStore::Store internalStore; MemoryUsage initExp; initExp.incAllocatedBytes(docs.getMemoryConsumption()); initExp.incUsedBytes(docs.getMemoryUsed()); @@ -142,7 +142,7 @@ TEST("require that initial memory usage is reported") TEST("require that memory usage is updated after insert") { - CompactDocumentWordsStore store; + CompactWordsStore store; MemoryUsage init = store.getMemoryUsage(); store.insert(Builder(d1).insert(w1)); diff --git a/searchlib/src/tests/memoryindex/datastore/.gitignore b/searchlib/src/tests/memoryindex/datastore/.gitignore index 98f4acc70a8..a5842a0fd69 100644 --- a/searchlib/src/tests/memoryindex/datastore/.gitignore +++ b/searchlib/src/tests/memoryindex/datastore/.gitignore @@ -1,8 +1,4 @@ .depend Makefile -datastore_test -featurestore_test -wordstore_test -searchlib_datastore_test_app -searchlib_featurestore_test_app -searchlib_wordstore_test_app +searchlib_feature_store_test_app +searchlib_word_store_test_app diff --git a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt index 2ba0f2eac63..45507f3b0ae 100644 --- a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt @@ -1,15 +1,15 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_featurestore_test_app TEST +vespa_add_executable(searchlib_feature_store_test_app TEST SOURCES - featurestore_test.cpp + feature_store_test.cpp DEPENDS searchlib ) -vespa_add_test(NAME searchlib_featurestore_test_app COMMAND searchlib_featurestore_test_app) -vespa_add_executable(searchlib_wordstore_test_app TEST +vespa_add_test(NAME searchlib_feature_store_test_app COMMAND searchlib_feature_store_test_app) +vespa_add_executable(searchlib_word_store_test_app TEST SOURCES - wordstore_test.cpp + word_store_test.cpp DEPENDS searchlib ) -vespa_add_test(NAME searchlib_wordstore_test_app COMMAND searchlib_wordstore_test_app) +vespa_add_test(NAME searchlib_word_store_test_app COMMAND searchlib_word_store_test_app) diff --git a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp index dc061f55732..49e9d613861 100644 --- a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("featurestore_test"); +LOG_SETUP("feature_store_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/featurestore.h> +#include <vespa/searchlib/memoryindex/feature_store.h> using namespace search::btree; using namespace search::datastore; @@ -213,7 +213,7 @@ Test::Test() int Test::Main() { - TEST_INIT("featurestore_test"); + TEST_INIT("feature_store_test"); requireThatFeaturesCanBeAddedAndRetrieved(); requireThatNextWordsAreWorking(); diff --git a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp index c1baff72514..b7f454bfdf7 100644 --- a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("wordstore_test"); +LOG_SETUP("word_store_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/wordstore.h> +#include <vespa/searchlib/memoryindex/word_store.h> using namespace search::datastore; @@ -75,7 +75,7 @@ Test::requireThatAddWordTriggersChangeOfBuffer() int Test::Main() { - TEST_INIT("wordstore_test"); + TEST_INIT("word_store_test"); requireThatWordsCanBeAddedAndRetrieved(); requireThatAddWordTriggersChangeOfBuffer(); diff --git a/searchlib/src/tests/memoryindex/document_inverter/.gitignore b/searchlib/src/tests/memoryindex/document_inverter/.gitignore new file mode 100644 index 00000000000..6245bb146e7 --- /dev/null +++ b/searchlib/src/tests/memoryindex/document_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_document_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt new file mode 100644 index 00000000000..1058a19d0ce --- /dev/null +++ b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_document_inverter_test_app TEST + SOURCES + document_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_document_inverter_test_app COMMAND searchlib_document_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp index 36cd15c8ada..91c1ccba706 100644 --- a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp @@ -3,9 +3,9 @@ #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/vespalib/testkit/testapp.h> @@ -97,7 +97,7 @@ struct Fixture SequencedTaskExecutor _invertThreads; SequencedTaskExecutor _pushThreads; DocumentInverter _inv; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; static Schema makeSchema() diff --git a/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt deleted file mode 100644 index 04e5b0ec126..00000000000 --- a/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_document_remover_test_app TEST - SOURCES - document_remover_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_document_remover_test_app COMMAND searchlib_document_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/documentinverter/.gitignore b/searchlib/src/tests/memoryindex/documentinverter/.gitignore deleted file mode 100644 index 1e9666b2d63..00000000000 --- a/searchlib/src/tests/memoryindex/documentinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_documentinverter_test_app diff --git a/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt deleted file mode 100644 index c7760e9f12f..00000000000 --- a/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_documentinverter_test_app TEST - SOURCES - documentinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_documentinverter_test_app COMMAND searchlib_documentinverter_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 373ed7fd311..3a635756ec7 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -10,11 +10,11 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/ordereddocumentinserter.h> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/memoryindex/ordered_field_index_inserter.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/vespalib/testkit/testapp.h> @@ -300,7 +300,7 @@ MockFieldIndex::~MockFieldIndex() = default; /** * MockWordStoreScan is a helper class to ensure that previous word is - * still stored safely in memory, to satisfy OrderedDocumentInserter + * still stored safely in memory, to satisfy OrderedFieldIndexInserter * needs. */ class MockWordStoreScan @@ -347,7 +347,7 @@ class MyInserter MockFieldIndex _mock; FieldIndexCollection _fieldIndexes; DocIdAndPosOccFeatures _features; - IOrderedDocumentInserter *_documentInserter; + IOrderedFieldIndexInserter *_inserter; public: MyInserter(const Schema &schema) @@ -355,7 +355,7 @@ public: _mock(), _fieldIndexes(schema), _features(), - _documentInserter(nullptr) + _inserter(nullptr) { _features.addNextOcc(0, 0, 1, 1); } @@ -365,32 +365,32 @@ public: setNextWord(const vespalib::string &word) { const vespalib::string &w = _wordStoreScan.setWord(word); - _documentInserter->setNextWord(w); + _inserter->setNextWord(w); _mock.setNextWord(w); } void setNextField(uint32_t fieldId) { - if (_documentInserter != nullptr) { - _documentInserter->flush(); + if (_inserter != nullptr) { + _inserter->flush(); } - _documentInserter = &_fieldIndexes.getFieldIndex(fieldId)->getInserter(); - _documentInserter->rewind(); + _inserter = &_fieldIndexes.getFieldIndex(fieldId)->getInserter(); + _inserter->rewind(); _mock.setNextField(fieldId); } void add(uint32_t docId) { - _documentInserter->add(docId, _features); + _inserter->add(docId, _features); _mock.add(docId); } void remove(uint32_t docId) { - _documentInserter->remove(docId); + _inserter->remove(docId); _mock.remove(docId); } @@ -406,8 +406,8 @@ public: bool assertPostings() { - if (_documentInserter != nullptr) { - _documentInserter->flush(); + if (_inserter != nullptr) { + _inserter->flush(); } for (auto wfp : _mock) { auto &wf = wfp.first; @@ -423,9 +423,9 @@ public: void rewind() { - if (_documentInserter != nullptr) { - _documentInserter->flush(); - _documentInserter = nullptr; + if (_inserter != nullptr) { + _inserter->flush(); + _inserter = nullptr; } } @@ -451,7 +451,7 @@ myremove(uint32_t docId, DocumentInverter &inv, FieldIndexCollection &fieldIndex class WrapInserter { - OrderedDocumentInserter &_inserter; + OrderedFieldIndexInserter &_inserter; public: WrapInserter(FieldIndexCollection &fieldIndexes, uint32_t fieldId) : _inserter(fieldIndexes.getFieldIndex(fieldId)->getInserter()) @@ -503,9 +503,9 @@ public: }; -class MyDrainRemoves : IDocumentRemoveListener +class MyDrainRemoves : IFieldIndexRemoveListener { - DocumentRemover &_remover; + FieldIndexRemover &_remover; public: virtual void remove(const vespalib::stringref, uint32_t) override { } diff --git a/searchlib/src/tests/memoryindex/document_remover/.gitignore b/searchlib/src/tests/memoryindex/field_index_remover/.gitignore index 2126f9147bd..2126f9147bd 100644 --- a/searchlib/src/tests/memoryindex/document_remover/.gitignore +++ b/searchlib/src/tests/memoryindex/field_index_remover/.gitignore diff --git a/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt new file mode 100644 index 00000000000..ef75337c6b6 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_field_index_remover_test_app TEST + SOURCES + field_index_remover_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_field_index_remover_test_app COMMAND searchlib_field_index_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp index af7a9422e49..fed6d963b70 100644 --- a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp @@ -2,9 +2,9 @@ #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/document_remover.h> -#include <vespa/searchlib/memoryindex/wordstore.h> -#include <vespa/searchlib/memoryindex/i_document_remove_listener.h> +#include <vespa/searchlib/memoryindex/field_index_remover.h> +#include <vespa/searchlib/memoryindex/i_field_index_remove_listener.h> +#include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/test/insertion_operators.h> #include <algorithm> @@ -38,7 +38,7 @@ operator<<(std::ostream &os, const WordFieldPair &val) return os; } -struct MockRemoveListener : public IDocumentRemoveListener +struct MockRemoveListener : public IFieldIndexRemoveListener { WordFieldVector _words; uint32_t _expDocId; @@ -65,7 +65,7 @@ struct Fixture MockRemoveListener _listener; std::vector<std::unique_ptr<WordStore>> _wordStores; std::vector<std::map<vespalib::string, datastore::EntryRef>> _wordToRefMaps; - std::vector<std::unique_ptr<DocumentRemover>> _removers; + std::vector<std::unique_ptr<FieldIndexRemover>> _removers; Fixture() : _listener(), _wordStores(), @@ -75,7 +75,7 @@ struct Fixture uint32_t numFields = 4; for (uint32_t fieldId = 0; fieldId < numFields; ++fieldId) { _wordStores.push_back(std::make_unique<WordStore>()); - _removers.push_back(std::make_unique<DocumentRemover> + _removers.push_back(std::make_unique<FieldIndexRemover> (*_wordStores.back())); } _wordToRefMaps.resize(numFields); diff --git a/searchlib/src/tests/memoryindex/field_inverter/.gitignore b/searchlib/src/tests/memoryindex/field_inverter/.gitignore new file mode 100644 index 00000000000..58579d09421 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_field_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt new file mode 100644 index 00000000000..f39e05d6823 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_field_inverter_test_app TEST + SOURCES + field_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_field_inverter_test_app COMMAND searchlib_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp index 1d066747ef8..f08e61b0da2 100644 --- a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/fixedtyperepo.h> namespace search { @@ -105,7 +105,7 @@ struct Fixture Schema _schema; DocBuilder _b; std::vector<std::unique_ptr<FieldInverter> > _inverters; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; static Schema makeSchema() diff --git a/searchlib/src/tests/memoryindex/fieldinverter/.gitignore b/searchlib/src/tests/memoryindex/fieldinverter/.gitignore deleted file mode 100644 index 482663dd92e..00000000000 --- a/searchlib/src/tests/memoryindex/fieldinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_fieldinverter_test_app diff --git a/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt deleted file mode 100644 index b6b223dceed..00000000000 --- a/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_fieldinverter_test_app TEST - SOURCES - fieldinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_fieldinverter_test_app COMMAND searchlib_fieldinverter_test_app) diff --git a/searchlib/src/tests/memoryindex/memory_index/.gitignore b/searchlib/src/tests/memoryindex/memory_index/.gitignore new file mode 100644 index 00000000000..6ec9ccf5015 --- /dev/null +++ b/searchlib/src/tests/memoryindex/memory_index/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +searchlib_memory_index_test_app diff --git a/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt new file mode 100644 index 00000000000..d6b6691f05d --- /dev/null +++ b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_memory_index_test_app TEST + SOURCES + memory_index_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_memory_index_test_app COMMAND searchlib_memory_index_test_app) diff --git a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 9de6ac9f310..50f44074683 100644 --- a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -2,24 +2,24 @@ #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/common/scheduletaskcallback.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> +#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> -#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> -#include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> -LOG_SETUP("memoryindex_test"); +LOG_SETUP("memory_index_test"); using document::Document; using document::FieldValue; diff --git a/searchlib/src/tests/memoryindex/memoryindex/.gitignore b/searchlib/src/tests/memoryindex/memoryindex/.gitignore deleted file mode 100644 index 174d0a494e2..00000000000 --- a/searchlib/src/tests/memoryindex/memoryindex/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.depend -Makefile -memoryindex_test -sourceselectorwriter_test -searchlib_memoryindex_test_app diff --git a/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt b/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt deleted file mode 100644 index 20f526e5c99..00000000000 --- a/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_memoryindex_test_app TEST - SOURCES - memoryindex_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_memoryindex_test_app COMMAND searchlib_memoryindex_test_app) diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore b/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore new file mode 100644 index 00000000000..694dc947042 --- /dev/null +++ b/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_url_field_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt new file mode 100644 index 00000000000..28efc8a861e --- /dev/null +++ b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_url_field_inverter_test_app TEST + SOURCES + url_field_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_url_field_inverter_test_app COMMAND searchlib_url_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp index daec09828f6..76fbf662b77 100644 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /* -*- mode: C++; coding: utf-8; -*- */ - +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/urlfieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/memoryindex/url_field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/fixedtyperepo.h> namespace search { @@ -183,7 +182,7 @@ struct Fixture DocBuilder _b; std::vector<std::unique_ptr<FieldInverter> > _inverters; std::unique_ptr<UrlFieldInverter> _urlInverter; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; index::SchemaIndexFields _schemaIndexFields; static Schema diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore b/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore deleted file mode 100644 index b2636fe5e81..00000000000 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_urlfieldinverter_test_app diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt deleted file mode 100644 index 16fa8f5952e..00000000000 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_urlfieldinverter_test_app TEST - SOURCES - urlfieldinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_urlfieldinverter_test_app COMMAND searchlib_urlfieldinverter_test_app) diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt index ffcd7ebd975..441fe12c383 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt @@ -1,17 +1,17 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_memoryindex OBJECT SOURCES - compact_document_words_store.cpp - documentinverter.cpp - document_remover.cpp - featurestore.cpp + compact_words_store.cpp + document_inverter.cpp + feature_store.cpp field_index.cpp field_index_collection.cpp - fieldinverter.cpp - memoryindex.cpp - ordereddocumentinserter.cpp - postingiterator.cpp - urlfieldinverter.cpp - wordstore.cpp + field_index_remover.cpp + field_inverter.cpp + memory_index.cpp + ordered_field_index_inserter.cpp + posting_iterator.cpp + url_field_inverter.cpp + word_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp index e2d089626b1..27282282c11 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp @@ -1,15 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "compact_document_words_store.h" +#include "compact_words_store.h" #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.compact_document_words_store"); +LOG_SETUP(".memoryindex.compact_words_store"); namespace search::memoryindex { -using Builder = CompactDocumentWordsStore::Builder; +using Builder = CompactWordsStore::Builder; namespace { @@ -36,28 +36,28 @@ serialize(const Builder &builder, uint32_t *begin) } -CompactDocumentWordsStore::Builder::Builder(uint32_t docId_) +CompactWordsStore::Builder::Builder(uint32_t docId_) : _docId(docId_), _words() { } -CompactDocumentWordsStore::Builder::~Builder() { } +CompactWordsStore::Builder::~Builder() { } -CompactDocumentWordsStore::Builder & -CompactDocumentWordsStore::Builder::insert(datastore::EntryRef wordRef) +CompactWordsStore::Builder & +CompactWordsStore::Builder::insert(datastore::EntryRef wordRef) { _words.push_back(wordRef); return *this; } inline void -CompactDocumentWordsStore::Iterator::nextWord() +CompactWordsStore::Iterator::nextWord() { _wordRef = *_buf++; _remainingWords--; } -CompactDocumentWordsStore::Iterator::Iterator() +CompactWordsStore::Iterator::Iterator() : _buf(nullptr), _remainingWords(0), _wordRef(0), @@ -65,7 +65,7 @@ CompactDocumentWordsStore::Iterator::Iterator() { } -CompactDocumentWordsStore::Iterator::Iterator(const uint32_t *buf) +CompactWordsStore::Iterator::Iterator(const uint32_t *buf) : _buf(buf), _remainingWords(0), _wordRef(0), @@ -79,8 +79,8 @@ CompactDocumentWordsStore::Iterator::Iterator(const uint32_t *buf) } } -CompactDocumentWordsStore::Iterator & -CompactDocumentWordsStore::Iterator::operator++() +CompactWordsStore::Iterator & +CompactWordsStore::Iterator::operator++() { if (_remainingWords > 0) { nextWord(); @@ -90,7 +90,7 @@ CompactDocumentWordsStore::Iterator::operator++() return *this; } -CompactDocumentWordsStore::Store::Store() +CompactWordsStore::Store::Store() : _store(), _type(1, MIN_BUFFER_ARRAYS, @@ -101,13 +101,13 @@ CompactDocumentWordsStore::Store::Store() _store.initActiveBuffers(); } -CompactDocumentWordsStore::Store::~Store() +CompactWordsStore::Store::~Store() { _store.dropBuffers(); } datastore::EntryRef -CompactDocumentWordsStore::Store::insert(const Builder &builder) +CompactWordsStore::Store::insert(const Builder &builder) { size_t serializedSize = getSerializedSize(builder); auto result = _store.rawAllocator<uint32_t>(_typeId).alloc(serializedSize); @@ -118,26 +118,26 @@ CompactDocumentWordsStore::Store::insert(const Builder &builder) return result.ref; } -CompactDocumentWordsStore::Iterator -CompactDocumentWordsStore::Store::get(datastore::EntryRef ref) const +CompactWordsStore::Iterator +CompactWordsStore::Store::get(datastore::EntryRef wordRef) const { - RefType internalRef(ref); + RefType internalRef(wordRef); const uint32_t *buf = _store.getEntry<uint32_t>(internalRef); return Iterator(buf); } -CompactDocumentWordsStore::CompactDocumentWordsStore() +CompactWordsStore::CompactWordsStore() : _docs(), _wordsStore() { } -CompactDocumentWordsStore::~CompactDocumentWordsStore() { } +CompactWordsStore::~CompactWordsStore() { } void -CompactDocumentWordsStore::insert(const Builder &builder) +CompactWordsStore::insert(const Builder &builder) { - datastore::EntryRef ref = _wordsStore.insert(builder); - auto insres = _docs.insert(std::make_pair(builder.docId(), ref)); + datastore::EntryRef wordRef = _wordsStore.insert(builder); + auto insres = _docs.insert(std::make_pair(builder.docId(), wordRef)); if (!insres.second) { LOG(error, "Failed inserting remove info for docid %u", builder.docId()); @@ -146,13 +146,13 @@ CompactDocumentWordsStore::insert(const Builder &builder) } void -CompactDocumentWordsStore::remove(uint32_t docId) +CompactWordsStore::remove(uint32_t docId) { _docs.erase(docId); } -CompactDocumentWordsStore::Iterator -CompactDocumentWordsStore::get(uint32_t docId) const +CompactWordsStore::Iterator +CompactWordsStore::get(uint32_t docId) const { auto itr = _docs.find(docId); if (itr != _docs.end()) { @@ -162,7 +162,7 @@ CompactDocumentWordsStore::get(uint32_t docId) const } MemoryUsage -CompactDocumentWordsStore::getMemoryUsage() const +CompactWordsStore::getMemoryUsage() const { MemoryUsage usage; usage.incAllocatedBytes(_docs.getMemoryConsumption()); diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h index ced7ec241bd..2fc6ec8d5bb 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h @@ -10,18 +10,16 @@ namespace search::memoryindex { /** - * Class used to store the {wordRef, fieldId, docId} tuples that are inserted - * into the memory index dictionary. These tuples are later used when removing - * all remains of a document from the posting lists of the dictionary. + * Class used to store the {wordRef, docId} tuples that are inserted into a FieldIndex and its posting lists. + * + * These tuples are later used when removing all remains of a document from the posting lists in that index. */ -class CompactDocumentWordsStore -{ +class CompactWordsStore { public: /** - * Builder used to collect all wordRefs for a field. + * Builder used to collect all words (as wordRefs) for a docId in a field. */ - class Builder - { + class Builder { public: using UP = std::unique_ptr<Builder>; using WordRefVector = vespalib::Array<datastore::EntryRef>; @@ -39,10 +37,9 @@ public: }; /** - * Iterator over all {wordRef, fieldId} pairs for a document. + * Iterator over all words (as wordRefs) for a docId in a field. */ - class Iterator - { + class Iterator { private: const uint32_t *_buf; uint32_t _remainingWords; @@ -61,10 +58,9 @@ public: }; /** - * Store for all {wordRef, fieldId} pairs among all documents. + * Store for all unique words (as wordRefs) among all documents. */ - class Store - { + class Store { public: using DataStoreType = datastore::DataStoreT<datastore::EntryRefT<22>>; using RefType = DataStoreType::RefType; @@ -78,7 +74,7 @@ public: Store(); ~Store(); datastore::EntryRef insert(const Builder &builder); - Iterator get(datastore::EntryRef ref) const; + Iterator get(datastore::EntryRef wordRef) const; MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } }; @@ -89,8 +85,8 @@ private: Store _wordsStore; public: - CompactDocumentWordsStore(); - ~CompactDocumentWordsStore(); + CompactWordsStore(); + ~CompactWordsStore(); void insert(const Builder &builder); void remove(uint32_t docId); Iterator get(uint32_t docId) const; diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index 1501ff7d2fc..a468428e21f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentinverter.h" +#include "document_inverter.h" #include "field_index_collection.h" -#include "fieldinverter.h" -#include "ordereddocumentinserter.h" -#include "urlfieldinverter.h" +#include "field_inverter.h" +#include "ordered_field_index_inserter.h" +#include "url_field_inverter.h" #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/repo/fixedtyperepo.h> @@ -16,7 +16,7 @@ #include <stdexcept> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.documentinverter"); +LOG_SETUP(".memoryindex.document_inverter"); namespace search::memoryindex { @@ -39,7 +39,6 @@ using index::DocIdAndPosOccFeatures; using index::Schema; using search::util::URL; - DocumentInverter::DocumentInverter(const Schema &schema, ISequencedTaskExecutor &invertThreads, ISequencedTaskExecutor &pushThreads) @@ -74,14 +73,12 @@ DocumentInverter::DocumentInverter(const Schema &schema, } } - DocumentInverter::~DocumentInverter() { _invertThreads.sync(); _pushThreads.sync(); } - void DocumentInverter::addFieldPath(const document::DocumentType &docType, uint32_t fieldId) @@ -100,9 +97,9 @@ DocumentInverter::addFieldPath(const document::DocumentType &docType, _indexedFieldPaths[fieldId] = std::move(fp); } - -void DocumentInverter::buildFieldPath(const document::DocumentType &docType, - const document::DataType *dataType) +void +DocumentInverter::buildFieldPath(const document::DocumentType &docType, + const document::DataType *dataType) { _indexedFieldPaths.clear(); _indexedFieldPaths.resize(_schema.getNumIndexFields()); @@ -115,7 +112,6 @@ void DocumentInverter::buildFieldPath(const document::DocumentType &docType, _dataType = dataType; } - void DocumentInverter::invertDocument(uint32_t docId, const Document &doc) { @@ -154,7 +150,6 @@ DocumentInverter::invertDocument(uint32_t docId, const Document &doc) } } - void DocumentInverter::removeDocument(uint32_t docId) { @@ -175,7 +170,6 @@ DocumentInverter::removeDocument(uint32_t docId) } } - void DocumentInverter::pushDocuments(FieldIndexCollection &fieldIndexes, const std::shared_ptr<IDestructorCallback> &onWriteDone) @@ -184,8 +178,8 @@ DocumentInverter::pushDocuments(FieldIndexCollection &fieldIndexes, uint32_t fieldId = 0; for (auto &inverter : _inverters) { FieldIndex &fieldIndex(**indexFieldIterator); - DocumentRemover &remover(fieldIndex.getDocumentRemover()); - OrderedDocumentInserter &inserter(fieldIndex.getInserter()); + FieldIndexRemover &remover(fieldIndex.getDocumentRemover()); + OrderedFieldIndexInserter &inserter(fieldIndex.getInserter()); _pushThreads.execute(fieldId, [inverter(inverter.get()), &remover, &inserter, &fieldIndex, onWriteDone]() diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index fa8d13d98fc..5c2d9cc84ed 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -2,7 +2,7 @@ #pragma once -#include "i_document_remove_listener.h" +#include "i_field_index_remove_listener.h" #include <vespa/searchlib/index/schema_index_fields.h> namespace document { @@ -24,8 +24,7 @@ class FieldInverter; class UrlFieldInverter; class FieldIndexCollection; -class DocumentInverter -{ +class DocumentInverter { private: DocumentInverter(const DocumentInverter &) = delete; DocumentInverter &operator=(const DocumentInverter &) = delete; diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp index c032bb33217..974fcc01c36 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "featurestore.h" +#include "feature_store.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/datastore/datastore.hpp> @@ -26,7 +26,6 @@ FeatureStore::writeFeatures(uint32_t packedIndex, const DocIdAndFeatures &featur return oldOffset; } - datastore::EntryRef FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) { @@ -43,7 +42,6 @@ FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) return result.ref; } - std::pair<datastore::EntryRef, uint64_t> FeatureStore::addFeatures(uint64_t beginOffset, uint64_t endOffset) { @@ -58,7 +56,6 @@ FeatureStore::addFeatures(uint64_t beginOffset, uint64_t endOffset) return std::make_pair(ref, bitLen); } - datastore::EntryRef FeatureStore::moveFeatures(datastore::EntryRef ref, uint64_t bitLen) { @@ -70,7 +67,6 @@ FeatureStore::moveFeatures(datastore::EntryRef ref, uint64_t bitLen) return newRef; } - FeatureStore::FeatureStore(const Schema &schema) : _store(), _f(nullptr), @@ -95,13 +91,11 @@ FeatureStore::FeatureStore(const Schema &schema) _store.initActiveBuffers(); } - FeatureStore::~FeatureStore() { _store.dropBuffers(); } - std::pair<datastore::EntryRef, uint64_t> FeatureStore::addFeatures(uint32_t packedIndex, const DocIdAndFeatures &features) { @@ -111,8 +105,6 @@ FeatureStore::addFeatures(uint32_t packedIndex, const DocIdAndFeatures &features return addFeatures(oldOffset, newOffset); } - - void FeatureStore::getFeatures(uint32_t packedIndex, datastore::EntryRef ref, DocIdAndFeatures &features) { @@ -121,7 +113,6 @@ FeatureStore::getFeatures(uint32_t packedIndex, datastore::EntryRef ref, DocIdAn _d.readFeatures(features); } - size_t FeatureStore::bitSize(uint32_t packedIndex, datastore::EntryRef ref) { @@ -135,7 +126,6 @@ FeatureStore::bitSize(uint32_t packedIndex, datastore::EntryRef ref) return bitLen; } - datastore::EntryRef FeatureStore::moveFeatures(uint32_t packedIndex, datastore::EntryRef ref) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h index ef75b9f6d31..94d44eaf44d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h @@ -9,8 +9,7 @@ namespace search::memoryindex { -class FeatureStore -{ +class FeatureStore { public: using DataStoreType = datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2>>; using RefType = DataStoreType::RefType; @@ -122,9 +121,7 @@ public: * @param packedIndex The field or field collection owning features * @param decoder The feature decoder */ - void - setupForField(uint32_t packedIndex, DecodeContextCooked &decoder) const - { + void setupForField(uint32_t packedIndex, DecodeContextCooked &decoder) const { decoder._fieldsParams = &_fieldsParams[packedIndex]; } @@ -135,9 +132,7 @@ public: * @param ref Reference to stored features * @param decoder The feature decoder */ - void - setupForReadFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const - { + void setupForReadFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const { const uint8_t * bits = getBits(ref); decoder.setByteCompr(bits); uint32_t bufferId = RefType(ref).bufferId(); @@ -155,9 +150,7 @@ public: * @param ref Reference to stored features * @param decoder The feature decoder */ - void - setupForUnpackFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const - { + void setupForUnpackFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const { decoder.setByteCompr(getBits(ref)); } @@ -169,8 +162,7 @@ public: * @param ref Reference to stored features * @return size of features in bits */ - size_t - bitSize(uint32_t packedIndex, datastore::EntryRef ref); + size_t bitSize(uint32_t packedIndex, datastore::EntryRef ref); /** * Get byte address of stored features diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 4d42b9ae493..7d10895c32f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "field_index.h" -#include "ordereddocumentinserter.h" +#include "ordered_field_index_inserter.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/searchlib/bitcompression/posocccompression.h> @@ -38,7 +38,7 @@ FieldIndex::FieldIndex(const Schema & schema, uint32_t fieldId) _featureStore(schema), _fieldId(fieldId), _remover(_wordStore), - _inserter(std::make_unique<OrderedDocumentInserter>(*this)) + _inserter(std::make_unique<OrderedFieldIndexInserter>(*this)) { } FieldIndex::~FieldIndex() @@ -88,7 +88,6 @@ FieldIndex::findFrozen(const vespalib::stringref word) const return PostingList::Iterator(); } - void FieldIndex::compactFeatures() { @@ -218,7 +217,6 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) } } - MemoryUsage FieldIndex::getMemoryUsage() const { @@ -231,7 +229,7 @@ FieldIndex::getMemoryUsage() const return usage; } -} // namespace search::memoryindex +} namespace search::btree { diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h index 4a27e30b47a..3b0675b5fdf 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h @@ -2,12 +2,12 @@ #pragma once -#include "featurestore.h" -#include "wordstore.h" -#include "document_remover.h" -#include <vespa/searchlib/btree/btreeroot.h> +#include "feature_store.h" +#include "field_index_remover.h" +#include "word_store.h" #include <vespa/searchlib/btree/btree.h> #include <vespa/searchlib/btree/btreenodeallocator.h> +#include <vespa/searchlib/btree/btreeroot.h> #include <vespa/searchlib/btree/btreestore.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/indexbuilder.h> @@ -16,7 +16,7 @@ namespace search::memoryindex { -class OrderedDocumentInserter; +class OrderedFieldIndexInserter; /** * Memory index for a single field. @@ -56,9 +56,7 @@ public: const WordStore &_wordStore; const vespalib::stringref _word; - const char * - getWord(datastore::EntryRef wordRef) const - { + const char *getWord(datastore::EntryRef wordRef) const { if (wordRef.valid()) { return _wordStore.getWord(wordRef); } @@ -71,9 +69,7 @@ public: _word(word) { } - bool - operator()(const WordKey & lhs, const WordKey & rhs) const - { + bool operator()(const WordKey & lhs, const WordKey & rhs) const { int cmpres = strcmp(getWord(lhs._wordRef), getWord(rhs._wordRef)); return cmpres < 0; } @@ -93,8 +89,8 @@ private: PostingListStore _postingListStore; FeatureStore _featureStore; uint32_t _fieldId; - DocumentRemover _remover; - std::unique_ptr<OrderedDocumentInserter> _inserter; + FieldIndexRemover _remover; + std::unique_ptr<OrderedFieldIndexInserter> _inserter; public: datastore::EntryRef addWord(const vespalib::stringref word) { @@ -102,9 +98,7 @@ public: return _wordStore.addWord(word); } - datastore::EntryRef - addFeatures(const index::DocIdAndFeatures &features) - { + datastore::EntryRef addFeatures(const index::DocIdAndFeatures &features) { return _featureStore.addFeatures(_fieldId, features).first; } @@ -118,7 +112,7 @@ public: uint64_t getNumUniqueWords() const { return _numUniqueWords; } const FeatureStore & getFeatureStore() const { return _featureStore; } const WordStore &getWordStore() const { return _wordStore; } - OrderedDocumentInserter &getInserter() const { return *_inserter; } + OrderedFieldIndexInserter &getInserter() const { return *_inserter; } private: void freeze() { @@ -126,9 +120,7 @@ private: _dict.getAllocator().freeze(); } - void - trimHoldLists() - { + void trimHoldLists() { GenerationHandler::generation_t usedGen = _generationHandler.getFirstUsedGeneration(); _postingListStore.trimHoldLists(usedGen); @@ -136,9 +128,7 @@ private: _featureStore.trimHoldLists(usedGen); } - void - transferHoldLists() - { + void transferHoldLists() { GenerationHandler::generation_t generation = _generationHandler.getCurrentGeneration(); _postingListStore.transferHoldLists(generation); @@ -146,9 +136,7 @@ private: _featureStore.transferHoldLists(generation); } - void - incGeneration() - { + void incGeneration() { _generationHandler.incGeneration(); } @@ -163,27 +151,11 @@ public: void dump(search::index::IndexBuilder & indexBuilder); MemoryUsage getMemoryUsage() const; + DictionaryTree &getDictionaryTree() { return _dict; } + PostingListStore &getPostingListStore() { return _postingListStore; } + FieldIndexRemover &getDocumentRemover() { return _remover; } - DictionaryTree & - getDictionaryTree() - { - return _dict; - } - - PostingListStore & - getPostingListStore() - { - return _postingListStore; - } - - DocumentRemover & - getDocumentRemover() - { - return _remover; - } - - void commit() - { + void commit() { _remover.flush(); freeze(); transferHoldLists(); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp index 45431f0e8ef..27944b5fe89 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "field_index_collection.h" -#include "fieldinverter.h" +#include "field_inverter.h" #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/btree/btreenode.hpp> @@ -40,7 +40,6 @@ FieldIndexCollection::~FieldIndexCollection() { } - void FieldIndexCollection::dump(search::index::IndexBuilder &indexBuilder) { @@ -61,6 +60,5 @@ FieldIndexCollection::getMemoryUsage() const return usage; } - } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h index 3b8e63626bf..5c2aa6f9b2c 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h @@ -6,7 +6,7 @@ namespace search::memoryindex { -class IDocumentRemoveListener; +class IFieldIndexRemoveListener; class FieldInverter; /** @@ -29,14 +29,11 @@ public: FieldIndexCollection(const index::Schema &schema); ~FieldIndexCollection(); PostingList::Iterator find(const vespalib::stringref word, - uint32_t fieldId) const - { + uint32_t fieldId) const { return _fieldIndexes[fieldId]->find(word); } - PostingList::ConstIterator - findFrozen(const vespalib::stringref word, uint32_t fieldId) const - { + PostingList::ConstIterator findFrozen(const vespalib::stringref word, uint32_t fieldId) const { return _fieldIndexes[fieldId]->findFrozen(word); } @@ -56,8 +53,7 @@ public: return _fieldIndexes[fieldId].get(); } - const std::vector<std::unique_ptr<FieldIndex>> & - getFieldIndexes() const { return _fieldIndexes; } + const std::vector<std::unique_ptr<FieldIndex>> &getFieldIndexes() const { return _fieldIndexes; } uint32_t getNumFields() const { return _numFields; } }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp index 67b519bbadc..2afddf072f2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "document_remover.h" -#include "i_document_remove_listener.h" -#include "wordstore.h" + +#include "field_index_remover.h" +#include "i_field_index_remove_listener.h" +#include "word_store.h" #include <vespa/searchlib/common/sort.h> namespace search::memoryindex { -using Builder = CompactDocumentWordsStore::Builder; -using Iterator = CompactDocumentWordsStore::Iterator; +using Builder = CompactWordsStore::Builder; +using Iterator = CompactWordsStore::Iterator; -DocumentRemover::DocumentRemover(const WordStore &wordStore) +FieldIndexRemover::FieldIndexRemover(const WordStore &wordStore) : _store(), _builder(), _wordFieldDocTuples(), @@ -17,11 +18,10 @@ DocumentRemover::DocumentRemover(const WordStore &wordStore) { } -DocumentRemover::~DocumentRemover() { -} +FieldIndexRemover::~FieldIndexRemover() = default; void -DocumentRemover::remove(uint32_t docId, IDocumentRemoveListener &listener) +FieldIndexRemover::remove(uint32_t docId, IFieldIndexRemoveListener &listener) { Iterator itr = _store.get(docId); if (itr.valid()) { @@ -34,14 +34,13 @@ DocumentRemover::remove(uint32_t docId, IDocumentRemoveListener &listener) } void -DocumentRemover::insert(datastore::EntryRef wordRef, uint32_t docId) +FieldIndexRemover::insert(datastore::EntryRef wordRef, uint32_t docId) { _wordFieldDocTuples.emplace_back(wordRef, docId); } - void -DocumentRemover::flush() +FieldIndexRemover::flush() { if (_wordFieldDocTuples.empty()) { return; @@ -60,5 +59,4 @@ DocumentRemover::flush() _wordFieldDocTuples.clear(); } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h index 5d44a666ff9..19b3353a27a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h @@ -1,22 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "compact_document_words_store.h" -#include "i_document_insert_listener.h" +#include "compact_words_store.h" +#include "i_field_index_insert_listener.h" namespace search::memoryindex { -class IDocumentRemoveListener; +class IFieldIndexRemoveListener; class WordStore; /** - * Class used to remove documents from the memory index dictionary. + * Class used to handle removal of documents from a FieldIndex. + * + * It tracks all {word, docId} tuples that are inserted into the index, + * and when removing a document, all these {word, docId} tuples are sent to the component + * that is doing the actual removal (IFieldIndexRemoveListener). */ -class DocumentRemover : public IDocumentInsertListener -{ +class FieldIndexRemover : public IFieldIndexInsertListener { private: - struct WordFieldDocTuple - { + struct WordFieldDocTuple { datastore::EntryRef _wordRef; uint32_t _docId; WordFieldDocTuple() : @@ -38,22 +40,21 @@ private: return wft._docId; } }; - }; - CompactDocumentWordsStore _store; - CompactDocumentWordsStore::Builder::UP _builder; + CompactWordsStore _store; + CompactWordsStore::Builder::UP _builder; std::vector<WordFieldDocTuple> _wordFieldDocTuples; const WordStore &_wordStore; public: - DocumentRemover(const WordStore &wordStore); - ~DocumentRemover(); - void remove(uint32_t docId, IDocumentRemoveListener &inverter); - CompactDocumentWordsStore &getStore() { return _store; } - const CompactDocumentWordsStore &getStore() const { return _store; } + FieldIndexRemover(const WordStore &wordStore); + ~FieldIndexRemover(); + void remove(uint32_t docId, IFieldIndexRemoveListener &inverter); + CompactWordsStore &getStore() { return _store; } + const CompactWordsStore &getStore() const { return _store; } - // Implements IDocumentInsertListener + // Implements IFieldIndexInsertListener void insert(datastore::EntryRef wordRef, uint32_t docId) override; void flush() override; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index fa261a4e90a..d19f05a98ee 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -1,25 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "fieldinverter.h" -#include "ordereddocumentinserter.h" +#include "field_inverter.h" +#include "ordered_field_index_inserter.h" +#include <vespa/document/annotation/alternatespanlist.h> +#include <vespa/document/annotation/annotation.h> +#include <vespa/document/annotation/span.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/annotation/spantreevisitor.h> #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchlib/bitcompression/compression.h> +#include <vespa/searchlib/bitcompression/posocccompression.h> +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sort.h> -#include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/bitcompression/posocccompression.h> -#include <vespa/document/annotation/annotation.h> -#include <vespa/document/annotation/span.h> -#include <vespa/document/annotation/spanlist.h> -#include <vespa/document/annotation/alternatespanlist.h> -#include <vespa/document/annotation/spantree.h> -#include <vespa/document/annotation/spantreevisitor.h> +#include <stdexcept> namespace search::memoryindex { @@ -48,23 +48,17 @@ using search::index::schema::CollectionType; using search::util::URL; using vespalib::make_string; -namespace documentinverterkludge { - -namespace linguistics { +namespace documentinverterkludge::linguistics { const vespalib::string SPANTREE_NAME("linguistics"); } -} - using namespace documentinverterkludge; -namespace -{ +namespace { -class SpanFinder : public SpanTreeVisitor -{ +class SpanFinder : public SpanTreeVisitor { public: int32_t begin_pos; int32_t end_pos; @@ -165,7 +159,6 @@ FieldInverter::processAnnotations(const StringFieldValue &value) } } - void FieldInverter::reset() { @@ -228,14 +221,12 @@ FieldInverter::sortWords() } } - void FieldInverter::startElement(int32_t weight) { _elems.push_back(ElemInfo(weight)); // Fill in length later } - void FieldInverter::endElement() { @@ -270,7 +261,6 @@ FieldInverter::saveWord(const vespalib::stringref word) return wordRef; } - uint32_t FieldInverter::saveWord(const document::FieldValue &fv) { @@ -280,7 +270,6 @@ FieldInverter::saveWord(const document::FieldValue &fv) return saveWord(vespalib::stringref(sRef.first, sRef.second)); } - void FieldInverter::remove(const vespalib::stringref word, uint32_t docId) { @@ -289,7 +278,6 @@ FieldInverter::remove(const vespalib::stringref word, uint32_t docId) _positions.emplace_back(wordRef, docId); } - void FieldInverter::processNormalDocTextField(const StringFieldValue &field) { @@ -298,7 +286,6 @@ FieldInverter::processNormalDocTextField(const StringFieldValue &field) endElement(); } - void FieldInverter::processNormalDocArrayTextField(const ArrayFieldValue &field) { @@ -314,7 +301,6 @@ FieldInverter::processNormalDocArrayTextField(const ArrayFieldValue &field) } } - void FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue &field) { @@ -331,7 +317,6 @@ FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue } } - FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId) : _fieldId(fieldId), _elem(0u), @@ -352,7 +337,6 @@ FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId) { } - void FieldInverter::abortPendingDoc(uint32_t docId) { @@ -365,7 +349,6 @@ FieldInverter::abortPendingDoc(uint32_t docId) } } - void FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, @@ -390,7 +373,6 @@ FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, dstIdx += size; } - void FieldInverter::trimAbortedDocs() { @@ -413,7 +395,6 @@ FieldInverter::trimAbortedDocs() _abortedDocs.clear(); } - void FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) { @@ -424,7 +405,6 @@ FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) endDoc(); } - void FieldInverter::invertNormalDocTextField(const FieldValue &val) { @@ -467,7 +447,6 @@ FieldInverter::invertNormalDocTextField(const FieldValue &val) } } - namespace { struct FullRadix { @@ -479,9 +458,8 @@ struct FullRadix { } - void -FieldInverter::applyRemoves(DocumentRemover &remover) +FieldInverter::applyRemoves(FieldIndexRemover &remover) { for (auto docId : _removeDocs) { remover.remove(docId, *this); @@ -489,9 +467,8 @@ FieldInverter::applyRemoves(DocumentRemover &remover) _removeDocs.clear(); } - void -FieldInverter::pushDocuments(IOrderedDocumentInserter &inserter) +FieldInverter::pushDocuments(IOrderedFieldIndexInserter &inserter) { trimAbortedDocs(); @@ -568,6 +545,5 @@ FieldInverter::pushDocuments(IOrderedDocumentInserter &inserter) reset(); } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 69cfd370041..ecf2f8d8979 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -2,27 +2,25 @@ #pragma once -#include <map> -#include <set> -#include <vespa/document/fieldvalue/document.h> +#include "i_field_index_remove_listener.h" +#include <vespa/document/annotation/span.h> #include <vespa/document/datatype/datatypes.h> -#include <limits> -#include "i_document_remove_listener.h" -#include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -#include <vespa/document/annotation/span.h> +#include <vespa/searchlib/index/docidandfeatures.h> +#include <limits> +#include <map> +#include <set> namespace search::memoryindex { -class IOrderedDocumentInserter; -class DocumentRemover; +class IOrderedFieldIndexInserter; +class FieldIndexRemover; -class FieldInverter : public IDocumentRemoveListener -{ +class FieldInverter : public IFieldIndexRemoveListener { public: - class PosInfo - { + class PosInfo { public: uint32_t _wordNum; // XXX: Initially word reference uint32_t _docId; @@ -54,7 +52,6 @@ public: { } - PosInfo(uint32_t wordRef, uint32_t docId) : _wordNum(wordRef), @@ -65,22 +62,19 @@ public: { } - bool - removed() const - { - return _elemId == _elemRemoved; - } + bool removed() const { return _elemId == _elemRemoved; } - bool - operator<(const PosInfo &rhs) const - { - if (_wordNum != rhs._wordNum) + bool operator<(const PosInfo &rhs) const { + if (_wordNum != rhs._wordNum) { return _wordNum < rhs._wordNum; - if (_docId != rhs._docId) + } + if (_docId != rhs._docId) { return _docId < rhs._docId; + } if (_elemId != rhs._elemId) { - if (removed() != rhs.removed()) + if (removed() != rhs.removed()) { return removed() && !rhs.removed(); + } return _elemId < rhs._elemId; } return _wordPos < rhs._wordPos; @@ -95,8 +89,7 @@ private: using WordBuffer = vespalib::Array<char>; - class ElemInfo - { + class ElemInfo { public: int32_t _weight; uint32_t _len; @@ -107,18 +100,13 @@ private: { } - void - setLen(uint32_t len) - { - _len = len; - } + void setLen(uint32_t len) { _len = len; } }; using ElemInfoVec = std::vector<ElemInfo>; using PosInfoVec = std::vector<PosInfo>; - class CompareWordRef - { + class CompareWordRef { const char *const _wordBuffer; public: @@ -127,15 +115,11 @@ private: { } - const char * - getWord(uint32_t wordRef) const - { + const char *getWord(uint32_t wordRef) const { return &_wordBuffer[static_cast<size_t>(wordRef) << 2]; } - bool - operator()(const uint32_t lhs, const uint32_t rhs) const - { + bool operator()(const uint32_t lhs, const uint32_t rhs) const { return strcmp(getWord(lhs), getWord(rhs)) < 0; } }; @@ -143,8 +127,7 @@ private: /* * Range in _positions vector used to represent a document put. */ - class PositionRange - { + class PositionRange { uint32_t _start; uint32_t _len; @@ -155,9 +138,7 @@ private: { } - bool - operator<(const PositionRange &rhs) const - { + bool operator<(const PositionRange &rhs) const { if (_start != rhs._start) { return _start < rhs._start; } @@ -202,14 +183,12 @@ public: * * @param weight element weight */ - void - startElement(int32_t weight); + void startElement(int32_t weight); /** * End an element. */ - void - endElement(); + void endElement(); private: /** @@ -220,8 +199,7 @@ private: * * @return word reference */ - VESPA_DLL_LOCAL uint32_t - saveWord(const vespalib::stringref word); + VESPA_DLL_LOCAL uint32_t saveWord(const vespalib::stringref word); /** * Save field value as word in word buffer. @@ -230,8 +208,7 @@ private: * * @return word reference */ - VESPA_DLL_LOCAL uint32_t - saveWord(const document::FieldValue &fv); + VESPA_DLL_LOCAL uint32_t saveWord(const document::FieldValue &fv); /** * Get pointer to saved word from a word reference. @@ -240,9 +217,7 @@ private: * * @return saved word */ - const char * - getWordFromRef(uint32_t wordRef) const - { + const char *getWordFromRef(uint32_t wordRef) const { return &_words[static_cast<size_t>(wordRef) << 2]; } @@ -253,9 +228,7 @@ private: * * @return saved word */ - const char * - getWordFromNum(uint32_t wordNum) const - { + const char *getWordFromNum(uint32_t wordNum) const { return getWordFromRef(_wordRefs[wordNum]); } @@ -266,9 +239,7 @@ private: * * @return word number */ - uint32_t - getWordNum(uint32_t wordRef) const - { + uint32_t getWordNum(uint32_t wordRef) const { const char *p = &_words[static_cast<size_t>(wordRef - 1) << 2]; return *reinterpret_cast<const uint32_t *>(p); } @@ -279,9 +250,7 @@ private: * @param wordRef word reference * @param wordNum word number */ - void - updateWordNum(uint32_t wordRef, uint32_t wordNum) - { + void updateWordNum(uint32_t wordRef, uint32_t wordNum) { char *p = &_words[static_cast<size_t>(wordRef - 1) << 2]; *reinterpret_cast<uint32_t *>(p) = wordNum; } @@ -292,17 +261,12 @@ private: * * @param wordRef word reference */ - void - add(uint32_t wordRef) { + void add(uint32_t wordRef) { _positions.emplace_back(wordRef, _docId, _elem, _wpos, _elems.size() - 1); } - void - stepWordPos() - { - ++_wpos; - } + void stepWordPos() { ++_wpos; } public: VESPA_DLL_LOCAL void @@ -323,30 +287,22 @@ private: * * @return schema used by this index */ - const index::Schema & - getSchema() const - { - return _schema; - } + const index::Schema &getSchema() const { return _schema; } /** * Clear internal memory structures. */ - void - reset(); + void reset(); /** * Calculate word numbers and replace word references with word * numbers in internal memory structures. */ - void - sortWords(); + void sortWords(); - void - moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); + void moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); - void - trimAbortedDocs(); + void trimAbortedDocs(); /* * Abort a pending document that has already been inverted. @@ -354,8 +310,7 @@ private: * @param docId local id for document * */ - void - abortPendingDoc(uint32_t docId); + void abortPendingDoc(uint32_t docId); public: /** @@ -371,42 +326,31 @@ public: * * @param remover document remover */ - void - applyRemoves(DocumentRemover &remover); + void applyRemoves(FieldIndexRemover &remover); /** - * Push inverted documents to memory index structure. + * Push inverted documents to field index structure using the given inserter. * - * Temporary restriction: Currently only one document at a time is - * supported. - * - * @param inserter ordered document inserter + * Temporary restriction: Currently only one document at a time is supported. */ - void - pushDocuments(IOrderedDocumentInserter &inserter); + void pushDocuments(IOrderedFieldIndexInserter &inserter); /* * Invert a normal text field, based on annotations. */ - void - invertField(uint32_t docId, const document::FieldValue::UP &val); + void invertField(uint32_t docId, const document::FieldValue::UP &val); /* * Setup remove of word in old version of document. */ - virtual void - remove(const vespalib::stringref word, uint32_t docId) override; + virtual void remove(const vespalib::stringref word, uint32_t docId) override; - void - removeDocument(uint32_t docId) - { + void removeDocument(uint32_t docId) { abortPendingDoc(docId); _removeDocs.push_back(docId); } - void - startDoc(uint32_t docId) - { + void startDoc(uint32_t docId) { assert(_docId == 0); assert(docId != 0); abortPendingDoc(docId); @@ -416,9 +360,7 @@ public: _wpos = 0; } - void - endDoc() - { + void endDoc() { uint32_t newPosSize = static_cast<uint32_t>(_positions.size()); _pendingDocs.insert({ _docId, { _oldPosSize, newPosSize - _oldPosSize } }); @@ -426,9 +368,7 @@ public: _oldPosSize = newPosSize; } - void - addWord(const vespalib::stringref word) - { + void addWord(const vespalib::stringref word) { uint32_t wordRef = saveWord(word); if (wordRef != 0u) { add(wordRef); diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h deleted file mode 100644 index 194a98ef8ba..00000000000 --- a/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once -#include <vespa/searchlib/datastore/entryref.h> - -namespace search::memoryindex { - -/** - * Interface used to track which {wordRef, fieldId} pairs that are - * inserted into the memory index dictionary for a document. - */ -class IDocumentInsertListener -{ -public: - virtual ~IDocumentInsertListener() {} - virtual void insert(datastore::EntryRef wordRef, uint32_t docId) = 0; - virtual void flush() = 0; -}; - -} - diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h new file mode 100644 index 00000000000..0aacfa53c34 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <vespa/searchlib/datastore/entryref.h> + +namespace search::memoryindex { + +/** + * Interface used to track which {wordRef, docId} pairs that are inserted into a FieldIndex. + */ +class IFieldIndexInsertListener { +public: + virtual ~IFieldIndexInsertListener() {} + + /** + * Called when a {wordRef, docId} tuple is inserted into the field index. + */ + virtual void insert(datastore::EntryRef wordRef, uint32_t docId) = 0; + + /** + * Called to process the set of {wordRef, docId} tuples inserted since last flush(). + */ + virtual void flush() = 0; +}; + +} + diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_document_remove_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_remove_listener.h index 436ee0a49e3..4419303a654 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/i_document_remove_listener.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_remove_listener.h @@ -7,13 +7,15 @@ namespace search::memoryindex { /** - * Interface used to track which {wordRef, fieldId} pairs that are - * removed from the memory index dictionary for a document. + * Interface used to track which {word, docId} pairs that are removed from a FieldIndex. */ -class IDocumentRemoveListener -{ +class IFieldIndexRemoveListener { public: - virtual ~IDocumentRemoveListener() {} + virtual ~IFieldIndexRemoveListener() {} + + /** + * Called when a {word, docId} tuple is removed from the field index. + */ virtual void remove(const vespalib::stringref word, uint32_t docId) = 0; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h index 9edd1eb4d3b..a1eee2e10ee 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h @@ -10,13 +10,14 @@ namespace search::index { class DocIdAndFeatures; } namespace search::memoryindex { /** - * Interface class for ordered document inserter. + * Interface used to insert inverted documents into a FieldIndex, + * updating the underlying posting lists in that index. * - * Insert order must be properly sorted, by (word, docId) + * Insert order must be properly sorted, first by word, then by docId. */ -class IOrderedDocumentInserter { +class IOrderedFieldIndexInserter { public: - virtual ~IOrderedDocumentInserter() {} + virtual ~IOrderedFieldIndexInserter() {} /** * Set next word to operate on. @@ -24,7 +25,7 @@ public: virtual void setNextWord(const vespalib::stringref word) = 0; /** - * Add (word, docId) tuple with given features. + * Add (word, docId) tuple with the given features. */ virtual void add(uint32_t docId, const index::DocIdAndFeatures &features) = 0; @@ -33,15 +34,13 @@ public: */ virtual void remove(uint32_t docId) = 0; - /* - * Flush pending changes to postinglist for (_word). - * - * _dItr is located at correct position. + /** + * Flush pending changes for the current word (into the underlying posting list). */ virtual void flush() = 0; - /* - * Rewind iterator, to start new pass. + /** + * Rewind to prepare for another set of (word, docId) tuples. */ virtual void rewind() = 0; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 90036c83efb..3ff2d553a96 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentinverter.h" +#include "document_inverter.h" #include "field_index_collection.h" -#include "memoryindex.h" -#include "postingiterator.h" +#include "memory_index.h" +#include "posting_iterator.h" #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/btree/btreenodeallocator.hpp> @@ -15,7 +15,7 @@ #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.memoryindex"); +LOG_SETUP(".searchlib.memoryindex.memory_index"); using document::ArrayFieldValue; using document::WeightedSetFieldValue; @@ -118,7 +118,6 @@ MemoryIndex::commit(const std::shared_ptr<IDestructorCallback> &onWriteDone) flipInverter(); } - void MemoryIndex::flipInverter() { @@ -139,8 +138,7 @@ MemoryIndex::dump(IndexBuilder &indexBuilder) namespace { -class MemTermBlueprint : public queryeval::SimpleLeafBlueprint -{ +class MemTermBlueprint : public queryeval::SimpleLeafBlueprint { private: GenerationHandler::Guard _genGuard; FieldIndex::PostingList::ConstIterator _pitr; @@ -167,8 +165,7 @@ public: setEstimate(estimate); } - SearchIterator::UP - createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { auto search = std::make_unique<PostingIterator>(_pitr, _featureStore, _fieldId, tfmda); if (_useBitVector) { LOG(debug, "Return BooleanMatchIteratorWrapper: fieldId(%u), docCount(%zu)", @@ -185,8 +182,7 @@ public: /** * Determines the correct Blueprint to use. **/ -class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper -{ +class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper { private: const FieldSpec &_field; const uint32_t _fieldId; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index 621c72d56a3..0b74e05c619 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -23,8 +23,7 @@ class FieldIndexCollection; * Lock-free implementation of a memory-based index * using the document inverter and dictionary classes from searchlib. **/ -class MemoryIndex : public queryeval::Searchable -{ +class MemoryIndex : public queryeval::Searchable { private: index::Schema _schema; ISequencedTaskExecutor &_invertThreads; @@ -136,15 +135,13 @@ public: void dump(index::IndexBuilder &indexBuilder); // implements Searchable - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpec &field, - const query::Node &term) override; - - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpecList &fields, - const query::Node &term) override { + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpec &field, + const query::Node &term) override; + + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpecList &fields, + const query::Node &term) override { return queryeval::Searchable::createBlueprint(requestContext, fields, term); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp index 3c4fca5b044..9b127a8b096 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "ordereddocumentinserter.h" -#include "i_document_insert_listener.h" +#include "i_field_index_insert_listener.h" +#include "ordered_field_index_inserter.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> @@ -27,7 +27,7 @@ const vespalib::string emptyWord = ""; } -OrderedDocumentInserter::OrderedDocumentInserter(FieldIndex &fieldIndex) +OrderedFieldIndexInserter::OrderedFieldIndexInserter(FieldIndex &fieldIndex) : _word(), _prevDocId(noDocId), _prevAdd(false), @@ -39,14 +39,13 @@ OrderedDocumentInserter::OrderedDocumentInserter(FieldIndex &fieldIndex) { } -OrderedDocumentInserter::~OrderedDocumentInserter() +OrderedFieldIndexInserter::~OrderedFieldIndexInserter() { flush(); } - void -OrderedDocumentInserter::flushWord() +OrderedFieldIndexInserter::flushWord() { if (_removes.empty() && _adds.empty()) { return; @@ -68,17 +67,15 @@ OrderedDocumentInserter::flushWord() _adds.clear(); } - void -OrderedDocumentInserter::flush() +OrderedFieldIndexInserter::flush() { flushWord(); _listener.flush(); } - void -OrderedDocumentInserter::setNextWord(const vespalib::stringref word) +OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word) { // TODO: Adjust here if zero length words should be legal. assert(_word < word); @@ -103,10 +100,9 @@ OrderedDocumentInserter::setNextWord(const vespalib::stringref word) assert(_word == wordStore.getWord(_dItr.getKey()._wordRef)); } - void -OrderedDocumentInserter::add(uint32_t docId, - const index::DocIdAndFeatures &features) +OrderedFieldIndexInserter::add(uint32_t docId, + const index::DocIdAndFeatures &features) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId || @@ -118,9 +114,8 @@ OrderedDocumentInserter::add(uint32_t docId, _prevAdd = true; } - void -OrderedDocumentInserter::remove(uint32_t docId) +OrderedFieldIndexInserter::remove(uint32_t docId) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId); @@ -129,9 +124,8 @@ OrderedDocumentInserter::remove(uint32_t docId) _prevAdd = false; } - void -OrderedDocumentInserter::rewind() +OrderedFieldIndexInserter::rewind() { assert(_removes.empty() && _adds.empty()); _word = ""; @@ -140,9 +134,8 @@ OrderedDocumentInserter::rewind() _dItr.begin(); } - datastore::EntryRef -OrderedDocumentInserter::getWordRef() const +OrderedFieldIndexInserter::getWordRef() const { return _dItr.getKey()._wordRef; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h index 328346e9eee..03cf3723f01 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h @@ -2,23 +2,25 @@ #pragma once -#include "iordereddocumentinserter.h" +#include "i_ordered_field_index_inserter.h" #include "field_index.h" #include <limits> namespace search::memoryindex { -class IDocumentInsertListener; - +class IFieldIndexInsertListener; /** - * Class for inserting updates to FieldIndex in an ordered manner - * (single pass scan of dictionary tree) + * Class used to insert inverted documents into a FieldIndex, + * updating the underlying posting lists in that index. + * + * This is done by doing a single pass scan of the dictionary of the FieldIndex, + * and for each word updating the posting list with docId adds / removes. * - * Insert order must be properly sorted, by (word, docId) + * Insert order must be properly sorted, first by word, then by docId. */ -class OrderedDocumentInserter : public IOrderedDocumentInserter -{ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { +private: vespalib::stringref _word; uint32_t _prevDocId; bool _prevAdd; @@ -29,7 +31,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter using PostingListKeyDataType = FieldIndex::PostingListKeyDataType; FieldIndex &_fieldIndex; DictionaryTree::Iterator _dItr; - IDocumentInsertListener &_listener; + IFieldIndexInsertListener &_listener; // Pending changes to posting list for (_word) std::vector<uint32_t> _removes; @@ -39,7 +41,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter static constexpr uint32_t noFieldId = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t noDocId = std::numeric_limits<uint32_t>::max(); - /* + /** * Flush pending changes to postinglist for (_word). * * _dItr is located at correct position. @@ -47,13 +49,13 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter void flushWord(); public: - OrderedDocumentInserter(FieldIndex &fieldIndex); - ~OrderedDocumentInserter() override; + OrderedFieldIndexInserter(FieldIndex &fieldIndex); + ~OrderedFieldIndexInserter() override; void setNextWord(const vespalib::stringref word) override; void add(uint32_t docId, const index::DocIdAndFeatures &features) override; void remove(uint32_t docId) override; - /* + /** * Flush pending changes to postinglist for (_word). Also flush * insert listener. * @@ -61,7 +63,7 @@ public: */ void flush() override; - /* + /** * Rewind iterator, to start new pass. */ void rewind() override; diff --git a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp index ca56299f906..4c29ec321e3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "postingiterator.h" +#include "posting_iterator.h" +#include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> -#include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.postingiterator"); +LOG_SETUP(".searchlib.memoryindex.posting_iterator"); namespace search::memoryindex { diff --git a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.h b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h index 2838c65c5eb..de337ef49f3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.h +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h @@ -10,8 +10,7 @@ namespace search::memoryindex { /** * Search iterator for memory field index posting list. */ -class PostingIterator : public queryeval::RankedSearchIteratorBase -{ +class PostingIterator : public queryeval::RankedSearchIteratorBase { private: FieldIndex::PostingList::ConstIterator _itr; const FeatureStore &_featureStore; diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp index 2c290f17782..c185ec93c9d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp @@ -1,20 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "urlfieldinverter.h" -#include "fieldinverter.h" +#include "field_inverter.h" +#include "url_field_inverter.h" #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sort.h> +#include <stdexcept> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.urlfieldinverter"); +LOG_SETUP(".memoryindex.url_field_inverter"); namespace search::memoryindex { @@ -46,7 +46,6 @@ lowercaseToken(vespalib::string &dest, const char *src, size_t srcSize) } - using document::ArrayFieldValue; using document::DataType; using document::FieldValue; @@ -61,7 +60,6 @@ using search::index::schema::CollectionType; using search::util::URL; using vespalib::make_string; - void UrlFieldInverter::startDoc(uint32_t docId) { @@ -75,7 +73,6 @@ UrlFieldInverter::startDoc(uint32_t docId) _hostname->startDoc(docId); } - void UrlFieldInverter::endDoc() { @@ -89,7 +86,6 @@ UrlFieldInverter::endDoc() _hostname->endDoc(); } - void UrlFieldInverter::startElement(int32_t weight) { @@ -103,7 +99,6 @@ UrlFieldInverter::startElement(int32_t weight) _hostname->startElement(weight); } - void UrlFieldInverter::endElement() { @@ -117,7 +112,6 @@ UrlFieldInverter::endElement() _hostname->endElement(); } - void UrlFieldInverter::processUrlSubField(FieldInverter *inverter, const StructFieldValue &field, @@ -145,7 +139,6 @@ UrlFieldInverter::processUrlSubField(FieldInverter *inverter, } } - void UrlFieldInverter::processAnnotatedUrlField(const StructFieldValue & field) { @@ -159,7 +152,6 @@ UrlFieldInverter::processAnnotatedUrlField(const StructFieldValue & field) processUrlSubField(_hostname, field, UrlDataType::FIELD_HOST, true); } - void UrlFieldInverter::processUrlField(const FieldValue &url_field) { @@ -207,7 +199,9 @@ UrlFieldInverter::processUrlField(const FieldValue &url_field) processUrlOldStyle(s); } -void UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) { +void +UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) +{ URL url(reinterpret_cast<const unsigned char *>(s.data()), s.size()); _hostname->addWord(HOSTNAME_BEGIN); @@ -264,7 +258,6 @@ void UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) { _hostname->addWord(HOSTNAME_END); } - void UrlFieldInverter::processArrayUrlField(const ArrayFieldValue &field) { @@ -276,7 +269,6 @@ UrlFieldInverter::processArrayUrlField(const ArrayFieldValue &field) } } - void UrlFieldInverter::processWeightedSetUrlField(const WeightedSetFieldValue &field) { @@ -292,13 +284,16 @@ UrlFieldInverter::processWeightedSetUrlField(const WeightedSetFieldValue &field) } namespace { -bool isUriType(const DataType &type) { + +bool +isUriType(const DataType &type) +{ return type == UrlDataType::getInstance() - || type == *DataType::STRING - || type == *DataType::URI; + || type == *DataType::STRING + || type == *DataType::URI; } -} // namespace +} void UrlFieldInverter::invertUrlField(const FieldValue &val) @@ -366,7 +361,6 @@ UrlFieldInverter::removeDocument(uint32_t docId) _hostname->removeDocument(docId); } - UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, FieldInverter *all, FieldInverter *scheme, @@ -389,6 +383,5 @@ UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, { } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.h index c902feaf5a6..1659e460af3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.h @@ -9,8 +9,7 @@ namespace search::memoryindex { class FieldInverter; -class UrlFieldInverter -{ +class UrlFieldInverter { FieldInverter *_all; FieldInverter *_scheme; FieldInverter *_host; @@ -31,11 +30,10 @@ class UrlFieldInverter void endElement(); - void - processUrlSubField(FieldInverter *inverter, - const document::StructFieldValue &field, - vespalib::stringref subField, - bool addAnchors); + void processUrlSubField(FieldInverter *inverter, + const document::StructFieldValue &field, + vespalib::stringref subField, + bool addAnchors); void processAnnotatedUrlField(const document::StructFieldValue &field); diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp index b65fe192e58..ffdc26f5eb0 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "wordstore.h" +#include "word_store.h" #include <vespa/searchlib/datastore/datastore.hpp> namespace search::memoryindex { diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.h b/searchlib/src/vespa/searchlib/memoryindex/word_store.h index b909f26157f..4c1526df527 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.h @@ -7,8 +7,7 @@ namespace search::memoryindex { -class WordStore -{ +class WordStore { public: using DataStoreType = datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2>>; using RefType = DataStoreType::RefType; @@ -23,8 +22,7 @@ public: WordStore(); ~WordStore(); datastore::EntryRef addWord(const vespalib::stringref word); - const char * getWord(datastore::EntryRef ref) const - { + const char *getWord(datastore::EntryRef ref) const { RefType internalRef(ref); return _store.getEntry<char>(internalRef); } diff --git a/searchlib/src/vespa/searchlib/predicate/document_features_store.h b/searchlib/src/vespa/searchlib/predicate/document_features_store.h index 2cf3e15337a..4c55b67cb11 100644 --- a/searchlib/src/vespa/searchlib/predicate/document_features_store.h +++ b/searchlib/src/vespa/searchlib/predicate/document_features_store.h @@ -4,7 +4,7 @@ #include "predicate_tree_annotator.h" #include <vespa/searchlib/btree/btree.h> -#include <vespa/searchlib/memoryindex/wordstore.h> +#include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/hash_map.h> #include <unordered_set> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 2d7a9abbf79..9cbbd136148 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -2,13 +2,13 @@ #include "fakememtreeocc.h" #include "fpfactory.h" -#include <vespa/searchlib/queryeval/iterators.h> -#include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> -#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> +#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/btree/btreeroot.hpp> +#include <vespa/searchlib/memoryindex/posting_iterator.h> +#include <vespa/searchlib/queryeval/iterators.h> #include <vespa/searchlib/util/postingpriorityqueue.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h index 7fa46fc7531..f0363500559 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h @@ -4,7 +4,7 @@ #include "fakeword.h" #include "fakeposting.h" #include "fpfactory.h" -#include <vespa/searchlib/memoryindex/featurestore.h> +#include <vespa/searchlib/memoryindex/feature_store.h> #include <vespa/searchlib/memoryindex/field_index.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h index 4802a7571c2..08473f9fc6c 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h @@ -2,20 +2,12 @@ #pragma once -#include <vespa/searchlib/memoryindex/iordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h> #include <sstream> -namespace search -{ +namespace search::memoryindex::test { -namespace memoryindex -{ - -namespace test -{ - -class OrderedDocumentInserter : public IOrderedDocumentInserter -{ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { std::stringstream _ss; bool _first; bool _verbose; @@ -31,7 +23,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter } } public: - OrderedDocumentInserter() + OrderedFieldIndexInserter() : _ss(), _first(true), _verbose(false), @@ -115,6 +107,4 @@ public: void setVerbose() { _verbose = true; } }; -} // namespace test -} // namespace memoryindex -} // namespace search +} |