diff options
25 files changed, 133 insertions, 105 deletions
diff --git a/http-utils/README.md b/http-utils/README.md index 3a4f3b68482..4e1fcd75d07 100644 --- a/http-utils/README.md +++ b/http-utils/README.md @@ -1,2 +1,4 @@ <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Http utilities for Java + +NOTE: This must be built with JDK 8, because it's used by vespa-http-client.
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 725e3fccec5..874e9cbe2c5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -369,6 +369,8 @@ public class MetricsReporter extends NodeRepositoryMaintainer { static Map<String, String> dimensions(ApplicationId application, ClusterSpec.Id cluster) { Map<String, String> dimensions = new HashMap<>(dimensions(application)); + //TODO: Remove "clusterId" once internal aggregation uses "clusterid" + dimensions.put("clusterid", cluster.value()); dimensions.put("clusterId", cluster.value()); return dimensions; } diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 14a74e3be8e..1b4848f9c8b 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -23,7 +23,6 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/document/update/documentupdate.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/vespalib/util/time.h> #include <vespa/log/log.h> LOG_SETUP("feedview_test"); @@ -149,10 +148,12 @@ struct MyIndexWriter : public test::MockIndexWriter (void) doc; _tracer.tracePut(indexAdapterTypeName, serialNum, lid); } - void remove(SerialNum serialNum, const search::DocumentIdT lid) override { - LOG(info, "MyIndexAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)", serialNum, lid); - _removes.push_back(lid); - _tracer.traceRemove(indexAdapterTypeName, serialNum, lid); + void removeDocs(SerialNum serialNum, LidVector lids) override { + for (search::DocumentIdT lid : lids) { + LOG(info, "MyIndexAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)", serialNum, lid); + _removes.push_back(lid); + _tracer.traceRemove(indexAdapterTypeName, serialNum, lid); + } } void commit(SerialNum serialNum, OnWriteDoneType) override { ++_commitCount; diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp index c1cdbdf9889..ac17c17892c 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp @@ -49,22 +49,21 @@ struct MyIndexManager : public test::MockIndexManager return toString(removes[lid]); } // Implements IIndexManager - virtual void putDocument(uint32_t lid, const Document &, - SerialNum serialNum) override { + void putDocument(uint32_t lid, const Document &, SerialNum serialNum) override { puts[lid].push_back(serialNum); } - virtual void removeDocument(uint32_t lid, - SerialNum serialNum) override { - removes[lid].push_back(serialNum); + void removeDocuments(LidVector lids, SerialNum serialNum) override { + for (uint32_t lid : lids) { + removes[lid].push_back(serialNum); + } } - virtual void commit(SerialNum serialNum, - OnWriteDoneType) override { + void commit(SerialNum serialNum, OnWriteDoneType) override { commitSerial = serialNum; } - virtual SerialNum getCurrentSerialNum() const override { + SerialNum getCurrentSerialNum() const override { return current; } - virtual SerialNum getFlushedSerialNum() const override { + SerialNum getFlushedSerialNum() const override { return flushed; } void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override { diff --git a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h index b68af5227e9..8a920d3d580 100644 --- a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h @@ -17,14 +17,20 @@ public: typedef std::shared_ptr<IIndexWriter> SP; using IIndexManager = searchcorespi::IIndexManager; using OnWriteDoneType = IIndexManager::OnWriteDoneType; + using LidVector = std::vector<search::DocumentIdT>; - virtual ~IIndexWriter() {} + virtual ~IIndexWriter() = default; virtual const std::shared_ptr<IIndexManager> &getIndexManager() const = 0; // feed interface virtual void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid) = 0; - virtual void remove(search::SerialNum serialNum, const search::DocumentIdT lid) = 0; + void remove(search::SerialNum serialNum, search::DocumentIdT lid) { + LidVector lids; + lids.push_back(lid); + removeDocs(serialNum, std::move(lids)); + } + virtual void removeDocs(search::SerialNum serialNum, LidVector lids) = 0; virtual void commit(search::SerialNum serialNum, OnWriteDoneType onWriteDone) = 0; virtual void heartBeat(search::SerialNum serialNum) = 0; virtual void compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp index 0d2f3914400..f86ada8c45c 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp @@ -43,13 +43,15 @@ IndexWriter::put(search::SerialNum serialNum, const document::Document &doc, con } void -IndexWriter::remove(search::SerialNum serialNum, const search::DocumentIdT lid) +IndexWriter::removeDocs(search::SerialNum serialNum, LidVector lids) { if (serialNum <= _mgr->getFlushedSerialNum()) { return; } - VLOG(getDebugLevel(lid, NULL), "Handle remove: serial(%" PRIu64 "), lid(%u)", serialNum, lid); - _mgr->removeDocument(lid, serialNum); + for (search::DocumentIdT lid : lids) { + VLOG(getDebugLevel(lid, NULL), "Handle remove: serial(%" PRIu64 "), num_lids(%lu)", serialNum, lids.size()); + } + _mgr->removeDocuments(std::move(lids), serialNum); } void diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.h b/searchcore/src/vespa/searchcore/proton/index/index_writer.h index 48781fe23cf..28357d6fd55 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.h @@ -15,22 +15,14 @@ private: public: IndexWriter(const IIndexManager::SP &mgr); - ~IndexWriter(); - /** - * Implements IIndexWriter. - */ - virtual const IIndexManager::SP & getIndexManager() const override { return _mgr; } - - virtual void put(search::SerialNum serialNum, - const document::Document &doc, - const search::DocumentIdT lid) override; - virtual void remove(search::SerialNum serialNum, - const search::DocumentIdT lid) override; - virtual void commit(search::SerialNum serialNum, - OnWriteDoneType onWriteDone) override; - - virtual void - heartBeat(search::SerialNum serialNum) override; + ~IndexWriter() override; + const IIndexManager::SP & getIndexManager() const override { return _mgr; } + + void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid) override; + void removeDocs(search::SerialNum serialNum, LidVector lids) override; + void commit(search::SerialNum serialNum, OnWriteDoneType onWriteDone) override; + + void heartBeat(search::SerialNum serialNum) override; void compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index af36e960202..8212978527a 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -90,8 +90,8 @@ public: _maintainer.putDocument(lid, doc, serialNum); } - void removeDocument(uint32_t lid, SerialNum serialNum) override { - _maintainer.removeDocument(lid, serialNum); + void removeDocuments(LidVector lids, SerialNum serialNum) override { + _maintainer.removeDocuments(std::move(lids), serialNum); } void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) override { diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index 11898d1a630..e4eaad538cb 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -79,8 +79,8 @@ public: void insertDocument(uint32_t lid, const document::Document &doc) override { _index.insertDocument(lid, doc); } - void removeDocument(uint32_t lid) override { - _index.removeDocument(lid); + void removeDocuments(LidVector lids) override { + _index.removeDocuments(std::move(lids)); } uint64_t getStaticMemoryFootprint() const override { return _index.getStaticMemoryFootprint(); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 5ee3f17e127..6101e8f0918 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -153,9 +153,9 @@ SearchableFeedView::performIndexRemove(SerialNum serialNum, const LidVector &lid VLOG(getDebugLevel(lid, nullptr), "database(%s): performIndexRemove: serialNum(%" PRIu64 "), lid(%d)", _params._docTypeName.toString().c_str(), serialNum, lid); - - _indexWriter->remove(serialNum, lid); } + + _indexWriter->removeDocs(serialNum, lidsToRemove); } void diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h index 9dc1fbdd9f6..7c4de5be2db 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h @@ -3,35 +3,31 @@ #include <vespa/searchcorespi/index/iindexmanager.h> -namespace proton { - -namespace test { +namespace proton::test { /** * Mock of the IIndexManager interface used for unit testing. */ struct MockIndexManager : public searchcorespi::IIndexManager { - virtual void putDocument(uint32_t, const Document &, SerialNum) override {} - virtual void removeDocument(uint32_t, SerialNum) override {} - virtual void commit(SerialNum, OnWriteDoneType) override {} - virtual SerialNum getCurrentSerialNum() const override { return 0; } - virtual SerialNum getFlushedSerialNum() const override { return 0; } - virtual searchcorespi::IndexSearchable::SP getSearchable() const override { + void putDocument(uint32_t, const Document &, SerialNum) override {} + void removeDocuments(LidVector, SerialNum) override {} + void commit(SerialNum, OnWriteDoneType) override {} + SerialNum getCurrentSerialNum() const override { return 0; } + SerialNum getFlushedSerialNum() const override { return 0; } + searchcorespi::IndexSearchable::SP getSearchable() const override { return searchcorespi::IndexSearchable::SP(); } - virtual search::SearchableStats getSearchableStats() const override { + search::SearchableStats getSearchableStats() const override { return search::SearchableStats(); } - virtual searchcorespi::IFlushTarget::List getFlushTargets() override { + searchcorespi::IFlushTarget::List getFlushTargets() override { return searchcorespi::IFlushTarget::List(); } - virtual void setSchema(const Schema &, SerialNum) override {} - virtual void heartBeat(SerialNum) override {} + void setSchema(const Schema &, SerialNum) override {} + void heartBeat(SerialNum) override {} void compactLidSpace(uint32_t, SerialNum) override {} - virtual void setMaxFlushed(uint32_t) override { } + void setMaxFlushed(uint32_t) override { } }; -} // namespace test - -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h index 30a2f638fc5..96af605b27c 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h @@ -3,9 +3,7 @@ #include <vespa/searchcore/proton/index/i_index_writer.h> -namespace proton { - -namespace test { +namespace proton::test { /** * Mock of the IIndexWriter interface used for unit testing. @@ -15,14 +13,12 @@ struct MockIndexWriter : public IIndexWriter IIndexManager::SP _idxMgr; MockIndexWriter() : _idxMgr() {} MockIndexWriter(const IIndexManager::SP &idxMgr) : _idxMgr(idxMgr) {} - virtual const IIndexManager::SP &getIndexManager() const override { return _idxMgr; } - virtual void put(search::SerialNum, const document::Document &, const search::DocumentIdT) override {} - virtual void remove(search::SerialNum, const search::DocumentIdT) override {} - virtual void commit(search::SerialNum, OnWriteDoneType) override {} - virtual void heartBeat(search::SerialNum) override {} + const IIndexManager::SP &getIndexManager() const override { return _idxMgr; } + void put(search::SerialNum, const document::Document &, const search::DocumentIdT) override {} + void removeDocs(search::SerialNum, LidVector) override {} + void commit(search::SerialNum, OnWriteDoneType) override {} + void heartBeat(search::SerialNum) override {} void compactLidSpace(search::SerialNum, const search::DocumentIdT) override {} }; -} // namespace test - -} // namespace proton +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h index afc7a327723..02adcbc11ce 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h +++ b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h @@ -30,10 +30,10 @@ namespace searchcorespi { */ class IIndexManager { protected: - typedef document::Document Document; - typedef search::SerialNum SerialNum; - typedef search::index::Schema Schema; - + using Document = document::Document; + using SerialNum = search::SerialNum; + using Schema = search::index::Schema; + using LidVector = std::vector<uint32_t>; public: using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; @@ -107,7 +107,12 @@ public: * @param serialNum The unique monotoninc increasing serial number * for this operation. **/ - virtual void removeDocument(uint32_t lid, SerialNum serialNum) = 0; + void removeDocument(uint32_t lid, SerialNum serialNum) { + LidVector lids; + lids.push_back(lid); + removeDocuments(std::move(lids), serialNum); + } + virtual void removeDocuments(LidVector lids, SerialNum serialNum) = 0; /** * Commits the document puts and removes since the last commit, diff --git a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h index 735beb620f6..bff929206a0 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h +++ b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h @@ -16,6 +16,7 @@ namespace searchcorespi::index { * Interface for a memory index as seen from an index maintainer. */ struct IMemoryIndex : public searchcorespi::IndexSearchable { + using LidVector = std::vector<uint32_t>; using SP = std::shared_ptr<IMemoryIndex>; using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; virtual ~IMemoryIndex() {} @@ -49,7 +50,12 @@ struct IMemoryIndex : public searchcorespi::IndexSearchable { * * @param lid the local document id. */ - virtual void removeDocument(uint32_t lid) = 0; + void removeDocument(uint32_t lid) { + LidVector lids; + lids.push_back(lid); + removeDocuments(std::move(lids)); + } + virtual void removeDocuments(LidVector lids) = 0; /** * Commits the inserts and removes since the last commit, making them searchable. diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index d58d0ab4ad6..74848e93411 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -1183,15 +1183,17 @@ IndexMaintainer::putDocument(uint32_t lid, const Document &doc, SerialNum serial } void -IndexMaintainer::removeDocument(uint32_t lid, SerialNum serialNum) +IndexMaintainer::removeDocuments(LidVector lids, SerialNum serialNum) { assert(_ctx.getThreadingService().index().isCurrentThread()); LockGuard lock(_index_update_lock); - _current_index->removeDocument(lid); - _selector->setSource(lid, _current_index_id); - _source_list->setSource(lid); - ++_source_selector_changes; + for (uint32_t lid : lids) { + _selector->setSource(lid, _current_index_id); + _source_list->setSource(lid); + } + _source_selector_changes += lids.size(); _current_serial_num = serialNum; + _current_index->removeDocuments(std::move(lids)); } void diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index be9d14ee1ad..816ec76cd1f 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -325,7 +325,7 @@ public: // Implements IIndexManager void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) override; - void removeDocument(uint32_t lid, SerialNum serialNum) override; + void removeDocuments(LidVector lids, SerialNum serialNum) override; void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; void heartBeat(search::SerialNum serialNum) override; void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override; diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 9ee47cc6849..dd445745f18 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -227,7 +227,9 @@ struct Fixture { index.insertDocument(id, *doc); } void remove(uint32_t id) { - index.removeDocument(id); + std::vector<uint32_t> lids; + lids.push_back(id); + index.removeDocuments(std::move(lids)); } void readWork(uint32_t cnt); diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 59ade118990..20cfb045081 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -112,7 +112,9 @@ struct Index { return d; } Index &remove(uint32_t id) { - index.removeDocument(id); + std::vector<uint32_t> lids; + lids.push_back(id); + index.removeDocuments(std::move(lids)); internalSyncCommit(); return *this; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index 4f03a5cb95f..127ff1d52c3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -116,6 +116,7 @@ DocumentInverter::buildFieldPath(const document::DocumentType &docType, void DocumentInverter::invertDocument(uint32_t docId, const Document &doc) { + // Might want to batch inverters as we do for attributes const document::DataType *dataType(doc.getDataType()); if (_indexedFieldPaths.empty() || _dataType != dataType) { buildFieldPath(doc.getType(), dataType); @@ -152,20 +153,31 @@ DocumentInverter::invertDocument(uint32_t docId, const Document &doc) } void -DocumentInverter::removeDocument(uint32_t docId) +DocumentInverter::removeDocument(uint32_t docId) { + LidVector lids; + lids.push_back(docId); + removeDocuments(std::move(lids)); +} +void +DocumentInverter::removeDocuments(LidVector lids) { + // Might want to batch inverters as we do for attributes for (uint32_t fieldId : _schemaIndexFields._textFields) { FieldInverter *inverter = _inverters[fieldId].get(); - _invertThreads.execute(fieldId, [inverter, docId]() { - inverter->removeDocument(docId); + _invertThreads.execute(fieldId, [inverter, lids]() { + for (uint32_t lid : lids) { + inverter->removeDocument(lid); + } }); } uint32_t urlId = 0; for (const auto & fi : _schemaIndexFields._uriFields) { uint32_t fieldId = fi._all; UrlFieldInverter *inverter = _urlInverters[urlId].get(); - _invertThreads.execute(fieldId,[inverter, docId]() { - inverter->removeDocument(docId); + _invertThreads.execute(fieldId, [inverter, lids]() { + for (uint32_t lid : lids) { + inverter->removeDocument(lid); + } }); ++urlId; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index 7ab80e1a2e5..ed06a0b39cc 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -40,6 +40,7 @@ private: void addFieldPath(const document::DocumentType &docType, uint32_t fieldId); void buildFieldPath(const document::DocumentType & docType, const document::DataType *dataType); + using LidVector = std::vector<uint32_t>; using FieldPath = document::Field; using IndexedFieldPaths = std::vector<std::unique_ptr<FieldPath>>; IndexedFieldPaths _indexedFieldPaths; @@ -100,6 +101,7 @@ public: * (using a field inverter) is added to the 'invert threads' executor', then this function returns. */ void removeDocument(uint32_t docId); + void removeDocuments(LidVector lids); FieldInverter *getInverter(uint32_t fieldId) const { return _inverters[fieldId].get(); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index cae4190b774..d96b8491027 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -8,7 +8,6 @@ #include <vespa/document/annotation/spanlist.h> #include <vespa/document/annotation/spantree.h> #include <vespa/document/annotation/spantreevisitor.h> -#include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> @@ -16,9 +15,9 @@ #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <vespa/vespalib/text/lowercase.h> #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <stdexcept> #include <vespa/log/log.h> @@ -362,7 +361,6 @@ FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId, _elems(), _positions(), _features(), - _elementWordRefs(), _wordRefs(1), _terms(), _abortedDocs(), diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 8dd308f1a4d..56cb1677f67 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -11,7 +11,6 @@ #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/vespalib/stllike/allocator.h> #include <limits> -#include <map> namespace search::index { class FieldLengthCalculator; } @@ -36,8 +35,7 @@ public: uint32_t _wordPos; uint32_t _elemRef; // Offset in _elems - static constexpr uint32_t _elemRemoved = - std::numeric_limits<uint32_t>::max(); + static constexpr uint32_t _elemRemoved = std::numeric_limits<uint32_t>::max(); PosInfo() noexcept : _wordNum(0), @@ -174,7 +172,6 @@ private: ElemInfoVec _elems; PosInfoVec _positions; index::DocIdAndPosOccFeatures _features; - UInt32Vector _elementWordRefs; UInt32Vector _wordRefs; using SpanTerm = std::pair<document::Span, const document::FieldValue *>; @@ -182,9 +179,9 @@ private: SpanTermVector _terms; // Info about aborted and pending documents. - std::vector<PositionRange> _abortedDocs; - std::map<uint32_t, PositionRange> _pendingDocs; - UInt32Vector _removeDocs; + std::vector<PositionRange> _abortedDocs; + vespalib::hash_map<uint32_t, PositionRange> _pendingDocs; + UInt32Vector _removeDocs; FieldIndexRemover &_remover; IOrderedFieldIndexInserter &_inserter; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 177d8e612bd..cb198f4d33a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -93,17 +93,20 @@ MemoryIndex::insertDocument(uint32_t docId, const document::Document &doc) } void -MemoryIndex::removeDocument(uint32_t docId) +MemoryIndex::removeDocuments(LidVector lids) { if (_frozen) { - LOG(warning, "Memory index frozen: ignoring remove of document (%u)", docId); + LOG(warning, "Memory index frozen: ignoring remove of %lu documents", lids.size()); return; } - _inverter->removeDocument(docId); - if (_indexedDocs.find(docId) != _indexedDocs.end()) { - _indexedDocs.erase(docId); - decNumDocs(); + for (uint32_t lid : lids) { + + if (_indexedDocs.find(lid) != _indexedDocs.end()) { + _indexedDocs.erase(lid); + decNumDocs(); + } } + _inverter->removeDocuments(std::move(lids)); } void diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index 83270aaf2ce..c02e66f790a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -41,6 +41,7 @@ class FieldIndexCollection; class MemoryIndex : public queryeval::Searchable { private: using ISequencedTaskExecutor = vespalib::ISequencedTaskExecutor; + using LidVector = std::vector<uint32_t>; index::Schema _schema; ISequencedTaskExecutor &_invertThreads; ISequencedTaskExecutor &_pushThreads; @@ -115,7 +116,7 @@ public: * * This function is async. commit() must be called for changes to take effect. */ - void removeDocument(uint32_t docId); + void removeDocuments(LidVector lids); /** * Commits the inserts and removes since the last commit, making them searchable. diff --git a/vespa-http-client/README b/vespa-http-client/README index 96083f71cc5..49d375316f6 100644 --- a/vespa-http-client/README +++ b/vespa-http-client/README @@ -1,3 +1,5 @@ Java API for feeding to Vespa from external sources Cross-colo etc. + +NOTE: This must be built with JDK 8, because it's used on the grid. |