diff options
Diffstat (limited to 'searchlib')
13 files changed, 44 insertions, 68 deletions
diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index dec7b911f65..1a6b0a5a1c6 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -25,6 +25,7 @@ struct NullDataStore : IDataStore { size_t memoryMeta() const override { return 0; } size_t getDiskFootprint() const override { return 0; } size_t getDiskBloat() const override { return 0; } + size_t getMaxCompactGain() const override { return 0; } uint64_t lastSyncToken() const override { return 0; } uint64_t tentativeLastSyncToken() const override { return 0; } vespalib::system_time getLastFlushTime() const override { return vespalib::system_time(); } diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp index b114a355bb4..8790bdd9885 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp @@ -30,13 +30,17 @@ remap_enum_store_refs(const EnumIndexRemapper& remapper, AttributeVector& v, att v.logEnumStoreEvent("compactfixup", "drain"); { AttributeVector::EnumModifier enum_guard(v.getEnumModifier()); + auto& filter = remapper.get_entry_ref_filter(); v.logEnumStoreEvent("compactfixup", "start"); for (uint32_t doc = 0; doc < v.getNumDocs(); ++doc) { vespalib::ConstArrayRef<WeightedIndex> indicesRef(multi_value_mapping.get(doc)); WeightedIndexVector indices(indicesRef.cbegin(), indicesRef.cend()); for (uint32_t i = 0; i < indices.size(); ++i) { - EnumIndex oldIndex = indices[i].value(); - indices[i] = WeightedIndex(remapper.remap(oldIndex), indices[i].weight()); + EnumIndex ref = indices[i].value(); + if (ref.valid() && filter.has(ref)) { + ref = remapper.remap(ref); + } + indices[i] = WeightedIndex(ref, indices[i].weight()); } std::atomic_thread_fence(std::memory_order_release); multi_value_mapping.replace(doc, indices); diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp index 4323e57f6b1..18805a7b20f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp @@ -49,13 +49,16 @@ SingleValueEnumAttributeBase::remap_enum_store_refs(const EnumIndexRemapper& rem { // update _enumIndices with new EnumIndex values after enum store has been compacted. v.logEnumStoreEvent("reenumerate", "reserved"); - auto new_indexes = std::make_unique<vespalib::Array<EnumIndex>>(); - new_indexes->reserve(_enumIndices.capacity()); + vespalib::Array<EnumIndex> new_indexes; + new_indexes.reserve(_enumIndices.capacity()); v.logEnumStoreEvent("reenumerate", "start"); + auto& filter = remapper.get_entry_ref_filter(); for (uint32_t i = 0; i < _enumIndices.size(); ++i) { - EnumIndex old_index = _enumIndices[i]; - EnumIndex new_index = remapper.remap(old_index); - new_indexes->push_back_fast(new_index); + EnumIndex ref = _enumIndices[i]; + if (ref.valid() && filter.has(ref)) { + ref = remapper.remap(ref); + } + new_indexes.push_back_fast(ref); } v.logEnumStoreEvent("compactfixup", "drain"); { diff --git a/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h b/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h index bb404f27709..cea251272dc 100644 --- a/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h +++ b/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h @@ -11,7 +11,7 @@ namespace search::common { * Interface for a component that has a lid space that can be compacted and shrunk. */ struct ICompactableLidSpace { - virtual ~ICompactableLidSpace() {} + virtual ~ICompactableLidSpace() = default; /** * Compacts the lid space down to the wanted given doc id limit. diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp index 38f3fbef0b0..26fb79f8a4e 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp +++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp @@ -26,7 +26,7 @@ BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const Compress _bucketizer(bucketizer), _writeCount(0), _maxBucketGuardDuration(vespalib::duration::zero()), - _lastSample(), + _lastSample(vespalib::steady_clock::now()), _lock(), _backingMemory(Alloc::alloc(0x40000000), &_lock), _tmpStore(), diff --git a/searchlib/src/vespa/searchlib/docstore/idatastore.h b/searchlib/src/vespa/searchlib/docstore/idatastore.h index b18bb0a3827..82656ad7e69 100644 --- a/searchlib/src/vespa/searchlib/docstore/idatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/idatastore.h @@ -17,14 +17,14 @@ class IBufferVisitor; class IDataStoreVisitor { public: - virtual ~IDataStoreVisitor() { } + virtual ~IDataStoreVisitor() = default; virtual void visit(uint32_t lid, const void *buffer, size_t sz) = 0; }; class IDataStoreVisitorProgress { public: - virtual ~IDataStoreVisitorProgress() { } + virtual ~IDataStoreVisitorProgress() = default; virtual void updateProgress(double progress) = 0; }; @@ -46,11 +46,7 @@ public: * @param dirName The directory that will contain the data file. **/ IDataStore(const vespalib::string & dirName); - - /** - * Allow inhertitance. - **/ - virtual ~IDataStore(); + ~IDataStore() override; /** * Read data from the data store into a buffer. @@ -125,7 +121,7 @@ public: * to avoid misuse we let the report a more conservative number here if necessary. * @return diskspace to be gained. */ - virtual size_t getMaxCompactGain() const { return getDiskBloat(); } + virtual size_t getMaxCompactGain() const = 0; /** diff --git a/searchlib/src/vespa/searchlib/docstore/idocumentstore.cpp b/searchlib/src/vespa/searchlib/docstore/idocumentstore.cpp index e1558f2238b..4f9b91f3e15 100644 --- a/searchlib/src/vespa/searchlib/docstore/idocumentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/idocumentstore.cpp @@ -5,10 +5,6 @@ namespace search { -IDocumentStore::IDocumentStore() = default; - -IDocumentStore::~IDocumentStore() = default; - void IDocumentStore::visit(const LidVector & lids, const document::DocumentTypeRepo &repo, IDocumentVisitor & visitor) const { for (uint32_t lid : lids) { visitor.visit(lid, read(lid, repo)); diff --git a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h index 2a7864a6f47..0e73e4d7993 100644 --- a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h @@ -22,7 +22,7 @@ class IDocumentStoreReadVisitor { public: using DocumentSP = std::shared_ptr<document::Document>; - virtual ~IDocumentStoreReadVisitor() { } + virtual ~IDocumentStoreReadVisitor() = default; virtual void visit(uint32_t lid, const DocumentSP &doc) = 0; virtual void visit(uint32_t lid) = 0; }; @@ -31,14 +31,14 @@ class IDocumentStoreRewriteVisitor { public: using DocumentSP = std::shared_ptr<document::Document>; - virtual ~IDocumentStoreRewriteVisitor() { } + virtual ~IDocumentStoreRewriteVisitor() = default; virtual void visit(uint32_t lid, const DocumentSP &doc) = 0; }; class IDocumentStoreVisitorProgress { public: - virtual ~IDocumentStoreVisitorProgress() { } + virtual ~IDocumentStoreVisitorProgress() = default; virtual void updateProgress(double progress) = 0; }; @@ -47,7 +47,7 @@ class IDocumentVisitor { public: using DocumentUP = std::unique_ptr<document::Document>; - virtual ~IDocumentVisitor() { } + virtual ~IDocumentVisitor() = default; virtual void visit(uint32_t lid, DocumentUP doc) = 0; virtual bool allowVisitCaching() const = 0; private: @@ -68,17 +68,6 @@ public: using LidVector = std::vector<uint32_t>; using DocumentUP = std::unique_ptr<document::Document>; - - /** - * Construct a document store. - * - * @throws vespalib::IoException if the file is corrupt or other IO problems occur. - * @param docMan The document type manager to use when deserializing. - * @param baseDir The path to a directory where the implementaion specific files will reside. - **/ - IDocumentStore(); - virtual ~IDocumentStore(); - /** * Make a Document from a stored serialized data blob. * @param lid The local ID associated with the document. @@ -169,7 +158,7 @@ public: * to avoid misuse we let the report a more conservative number here if necessary. * @return diskspace to be gained. */ - virtual size_t getMaxCompactGain() const { return getDiskBloat(); } + virtual size_t getMaxCompactGain() const = 0; /** * Returns statistics about the cache. diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 0e11b88a178..f43dc96fac9 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -111,9 +111,6 @@ public: size_t getDiskBloat() const override; size_t getMaxCompactGain() const override; - /** - * Will compact the docsummary up to a lower limit of 5% bloat. - */ void compact(uint64_t syncToken); const Config & getConfig() const { return _config; } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index d3c2998333a..86090f2ac92 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -79,12 +79,6 @@ DenseTensorStore::~DenseTensorStore() _store.dropBuffers(); } -const void * -DenseTensorStore::getRawBuffer(RefType ref) const -{ - return _store.getEntryArray<char>(ref, _bufferType.getArraySize()); -} - namespace { void clearPadAreaAfterBuffer(char *buffer, size_t bufSize, size_t alignedBufSize) { @@ -136,15 +130,6 @@ DenseTensorStore::getTensor(EntryRef ref) const return std::make_unique<vespalib::eval::DenseValueView>(_type, cells_ref); } -vespalib::eval::TypedCells -DenseTensorStore::get_typed_cells(EntryRef ref) const -{ - if (!ref.valid()) { - return vespalib::eval::TypedCells(&_emptySpace[0], _type.cell_type(), getNumCells()); - } - return vespalib::eval::TypedCells(getRawBuffer(ref), _type.cell_type(), getNumCells()); -} - template <class TensorType> TensorStore::EntryRef DenseTensorStore::setDenseTensor(const TensorType &tensor) diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 3b7cb71863e..06492596f70 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -50,12 +50,9 @@ private: ValueType _type; // type of dense tensor std::vector<char> _emptySpace; - size_t unboundCells(const void *buffer) const; - template <class TensorType> TensorStore::EntryRef setDenseTensor(const TensorType &tensor); - public: DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); ~DenseTensorStore() override; @@ -63,12 +60,17 @@ public: const ValueType &type() const { return _type; } size_t getNumCells() const { return _tensorSizeCalc._numCells; } size_t getBufSize() const { return _tensorSizeCalc.bufSize(); } - const void *getRawBuffer(RefType ref) const; + const void *getRawBuffer(RefType ref) const { + return _store.getEntryArray<char>(ref, _bufferType.getArraySize()); + } vespalib::datastore::Handle<char> allocRawBuffer(); void holdTensor(EntryRef ref) override; EntryRef move(EntryRef ref) override; std::unique_ptr<vespalib::eval::Value> getTensor(EntryRef ref) const; - vespalib::eval::TypedCells get_typed_cells(EntryRef ref) const; + vespalib::eval::TypedCells get_typed_cells(EntryRef ref) const { + return vespalib::eval::TypedCells(ref.valid() ? getRawBuffer(ref) : &_emptySpace[0], + _type.cell_type(), getNumCells()); + } EntryRef setTensor(const vespalib::eval::Value &tensor); // The following method is meant to be used only for unit tests. uint32_t getArraySize() const { return _bufferType.getArraySize(); } diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp index 7f9f20e07c4..43596478a6f 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp @@ -43,4 +43,13 @@ HammingDistance::calc(const vespalib::eval::TypedCells& lhs, } } +double +HammingDistance::calc_with_limit(const vespalib::eval::TypedCells& lhs, + const vespalib::eval::TypedCells& rhs, + double) const +{ + // consider optimizing: + return calc(lhs, rhs); +} + } diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.h b/searchlib/src/vespa/searchlib/tensor/hamming_distance.h index f0b7b159b90..c64fc5b532d 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.h @@ -15,7 +15,7 @@ namespace search::tensor { * or (for int8 cells, aka binary data only) * "number of bits that are different" */ -class HammingDistance : public DistanceFunction { +class HammingDistance final : public DistanceFunction { public: HammingDistance(vespalib::eval::CellType expected) : DistanceFunction(expected) {} double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override; @@ -26,13 +26,7 @@ public: double score = 1.0 / (1.0 + distance); return score; } - double calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double) const override - { - // consider optimizing: - return calc(lhs, rhs); - } + double calc_with_limit(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs, double) const override; }; } |