diff options
author | Geir Storli <geirst@yahooinc.com> | 2024-02-16 16:46:59 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-02-16 16:46:59 +0100 |
commit | 7a013aa963a8daca5e49172c2ddcb5229c0d67ed (patch) | |
tree | 257ab1dccc329054ad800292b9f7860ce5bf0f11 /searchlib | |
parent | 04ca60ee4d89e0a9a4227d70c19c09ba1f6a8913 (diff) | |
parent | 3b49a8d386624fb5dfb3bf30486dda530eb5e816 (diff) |
Merge pull request #30296 from vespa-engine/toregge/add-search-predicate-attribute-saver
Add search::PredicateAttributeSaver.
Diffstat (limited to 'searchlib')
5 files changed, 110 insertions, 22 deletions
diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index 96bd07bc5a3..4f886cf2dd4 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -111,6 +111,7 @@ vespa_add_library(searchlib_attribute OBJECT postinglisttraits.cpp postingstore.cpp predicate_attribute.cpp + predicate_attribute_saver.cpp raw_attribute.cpp raw_buffer_store.cpp raw_buffer_store_reader.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp index 60898450e50..075616a9fe9 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp @@ -4,6 +4,7 @@ #include "attribute_header.h" #include "iattributesavetarget.h" #include "load_utils.h" +#include "predicate_attribute_saver.h" #include <vespa/document/fieldvalue/predicatefieldvalue.h> #include <vespa/document/predicate/predicate.h> #include <vespa/searchlib/predicate/i_saver.h> @@ -143,29 +144,23 @@ PredicateAttribute::before_inc_generation(generation_t current_gen) _index->assign_generation(current_gen); } -void -PredicateAttribute::onSave(IAttributeSaveTarget &saveTarget) { - LOG(info, "Saving predicate attribute version %d name '%s'", getVersion(), getName().c_str()); - IAttributeSaveTarget::Buffer buffer(saveTarget.datWriter().allocBuf(4_Ki)); - { - DataBufferWriter writer(*buffer); - auto saver = _index->make_saver(); - saver->save(writer); - writer.flush(); - } - uint32_t highest_doc_id = static_cast<uint32_t>(_min_feature.size() - 1); - buffer->writeInt32(highest_doc_id); - for (size_t i = 1; i <= highest_doc_id; ++i) { - buffer->writeInt8(_min_feature[i]); - } - for (size_t i = 1; i <= highest_doc_id; ++i) { - buffer->writeInt16(_interval_range_vector[i]); - } - buffer->writeInt16(_max_interval_range); - saveTarget.datWriter().writeBuf(std::move(buffer)); +std::unique_ptr<AttributeSaver> +PredicateAttribute::onInitSave(vespalib::stringref fileName) +{ + auto guard(getGenerationHandler().takeGuard()); + auto header = this->createAttributeHeader(fileName); + auto min_feature_view = _min_feature.make_read_view(_min_feature.size()); + auto interval_range_vector_view = _interval_range_vector.make_read_view(_interval_range_vector.size()); + return std::make_unique<PredicateAttributeSaver> + (std::move(guard), + std::move(header), + getVersion(), + _index->make_saver(), + PredicateAttributeSaver::MinFeatureVector{ min_feature_view.begin(), min_feature_view.end() }, + PredicateAttributeSaver::IntervalRangeVector{ interval_range_vector_view.begin(), interval_range_vector_view.end() }, + _max_interval_range); } - uint32_t PredicateAttribute::getVersion() const { return PREDICATE_ATTRIBUTE_VERSION; diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h index acb4cbc5833..f188ac29941 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h @@ -45,7 +45,7 @@ public: predicate::PredicateIndex &getIndex() { return *_index; } - void onSave(IAttributeSaveTarget & saveTarget) override; + std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; bool onLoad(vespalib::Executor *executor) override; void onCommit() override; void reclaim_memory(generation_t oldest_used_gen) override; diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.cpp new file mode 100644 index 00000000000..aa1c3ab45a6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.cpp @@ -0,0 +1,52 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "predicate_attribute_saver.h" +#include "iattributesavetarget.h" +#include <vespa/searchlib/predicate/i_saver.h> +#include <vespa/searchlib/predicate/nbo_write.h> +#include <filesystem> + +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.attribute.predicate_attribute_saver"); + +using search::predicate::nbo_write; + +namespace search { + +PredicateAttributeSaver::PredicateAttributeSaver(GenerationHandler::Guard &&guard, + const attribute::AttributeHeader &header, + uint32_t version, + std::unique_ptr<predicate::ISaver> index_saver, + MinFeatureVector min_feature, + IntervalRangeVector interval_range_vector, + uint16_t max_interval_range) + : AttributeSaver(std::move(guard), header), + _version(version), + _index_saver(std::move(index_saver)), + _min_feature(std::move(min_feature)), + _interval_range_vector(std::move(interval_range_vector)), + _max_interval_range(max_interval_range) +{ +} + +PredicateAttributeSaver::~PredicateAttributeSaver() = default; + +bool +PredicateAttributeSaver::onSave(IAttributeSaveTarget& save_target) +{ + auto name = std::filesystem::path(get_file_name()).filename().native(); + LOG(info, "Saving predicate attribute version %u name '%s'", _version, name.c_str()); + auto writer = save_target.datWriter().allocBufferWriter(); + _index_saver->save(*writer); + uint32_t highest_doc_id = static_cast<uint32_t>(_min_feature.size() - 1); + nbo_write<uint32_t>(*writer, highest_doc_id); + writer->write(&_min_feature[1], highest_doc_id); + for (size_t i = 1; i <= highest_doc_id; ++i) { + nbo_write<uint16_t>(*writer, _interval_range_vector[i]); + } + nbo_write<uint16_t>(*writer, _max_interval_range); + writer->flush(); + return true; +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.h b/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.h new file mode 100644 index 00000000000..28e7094091f --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.h @@ -0,0 +1,40 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "attributesaver.h" +#include <vespa/vespalib/stllike/allocator.h> + +namespace search::predicate { class ISaver; } + +namespace search { + +/** + * Class for saving a predicate attribute. + */ +class PredicateAttributeSaver : public AttributeSaver +{ +public: + using GenerationHandler = vespalib::GenerationHandler; + using MinFeatureVector = std::vector<uint8_t, vespalib::allocator_large<uint8_t>>; + using IntervalRangeVector = std::vector<uint16_t, vespalib::allocator_large<uint16_t>>; + +private: + uint32_t _version; + std::unique_ptr<predicate::ISaver> _index_saver; + MinFeatureVector _min_feature; + IntervalRangeVector _interval_range_vector; + uint16_t _max_interval_range; +public: + PredicateAttributeSaver(GenerationHandler::Guard&& guard, + const attribute::AttributeHeader& header, + uint32_t version, + std::unique_ptr<predicate::ISaver> index_saver, + MinFeatureVector min_feature, + IntervalRangeVector interval_range_vector, + uint16_t max_interval_range); + ~PredicateAttributeSaver() override; + bool onSave(IAttributeSaveTarget& save_target) override; +}; + +} |