diff options
11 files changed, 70 insertions, 80 deletions
diff --git a/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp b/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp index bc97e57d0b3..1a868bcb57a 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp @@ -8,7 +8,6 @@ #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/vespalib/util/compressor.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/searchlib/common/packets.h> diff --git a/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h b/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h index 5cdfec78ec1..d00a0714045 100644 --- a/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h +++ b/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h @@ -2,12 +2,12 @@ #pragma once -#include <vespa/vespalib/data/output.h> #include "rawbuf.h" +#include <vespa/vespalib/data/output.h> namespace search { -class SlimeOutputRawBufAdapter : public ::vespalib::Output +class SlimeOutputRawBufAdapter : public vespalib::Output { private: RawBuf &_buf; diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 7265dd89be4..a00592400b5 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -2,9 +2,7 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsum_field_writer.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> @@ -40,7 +38,7 @@ struct AttributeCombinerTest : public ::testing::Test std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; AttributeCombinerTest(); - ~AttributeCombinerTest(); + ~AttributeCombinerTest() override; void set_field(const vespalib::string &field_name, bool filter_elements); void assertWritten(const vespalib::string &exp, uint32_t docId); }; diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 160e4cec973..28c0aad6c40 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -14,7 +14,6 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index cab6090da43..971f6101203 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -17,11 +17,12 @@ #include <vespa/searchsummary/docsummary/keywordextractor.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/searchlib/util/rawbuf.h> +#include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/util/size_literals.h> using namespace vespalib::slime::convenience; using namespace search::docsummary; +using vespalib::slime::BinaryFormat; using search::MatchingElements; using document::ByteFieldValue; using document::DataType; @@ -49,15 +50,12 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { DocsumFixture(); ~DocsumFixture() override; void getDocsum(Slime &slime) { - uint32_t classId; - search::RawBuf buf(4_Ki); - writer->WriteDocsum(1u, &state, this, &buf); - ASSERT_GREATER(buf.GetUsedLen(), sizeof(classId)); - memcpy(&classId, buf.GetDrainPos(), sizeof(classId)); - buf.Drain(sizeof(classId)); - EXPECT_EQUAL(classId, SLIME_MAGIC_ID); - EXPECT_GREATER(vespalib::slime::BinaryFormat - ::decode(Memory(buf.GetDrainPos(), buf.GetUsedLen()), slime), 0u); + Slime slimeOut; + SlimeInserter inserter(slimeOut); + writer->WriteDocsum(1u, &state, this, inserter); + vespalib::SmartBuffer buf(4_Ki); + BinaryFormat::encode(slimeOut, buf); + EXPECT_GREATER(BinaryFormat::decode(buf.obtain(), slime), 0u); } uint32_t getNumDocs() const override { return 2; } std::unique_ptr<const IDocsumStoreDocument> getMappedDocsum(uint32_t docid) override { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 507734ac2e4..28c55f1c2a6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -6,30 +6,19 @@ #include "i_docsum_store_document.h" #include "keywordextractor.h" #include <vespa/document/fieldvalue/fieldvalue.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/attribute/iattributemanager.h> -#include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.docsumwriter"); -using namespace vespalib::slime::convenience; using vespalib::Issue; +using vespalib::slime::ObjectInserter; +using vespalib::Memory; namespace search::docsummary { -uint32_t -IDocsumWriter::slime2RawBuf(const Slime & slime, RawBuf & buf) -{ - const uint32_t preUsed = buf.GetUsedLen(); - const uint32_t magic = SLIME_MAGIC_ID; - buf.append(&magic, sizeof(magic)); - SlimeOutputRawBufAdapter adapter(buf); - vespalib::slime::BinaryFormat::encode(slime, adapter); - return (buf.GetUsedLen() - preUsed); -} - DynamicDocsumWriter::ResolveClassInfo DynamicDocsumWriter::resolveClassInfo(vespalib::stringref outputClassName) const { @@ -61,7 +50,7 @@ DynamicDocsumWriter::resolveOutputClass(vespalib::stringref summaryClass) const void DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::slime::Inserter& topInserter) + IDocsumStore *docinfos, Inserter& topInserter) { if (rci.mustSkip || rci.outputClass == nullptr) { // Use empty docsum when illegal docsum class has been requested @@ -172,14 +161,11 @@ DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *sta } -uint32_t -DynamicDocsumWriter::WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, search::RawBuf *target) +void +DynamicDocsumWriter::WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, Inserter& inserter) { - vespalib::Slime slime; - vespalib::slime::SlimeInserter inserter(slime); ResolveClassInfo rci = resolveClassInfo(state->_args.getResultClassName()); insertDocsum(rci, docid, state, docinfos, inserter); - return slime2RawBuf(slime, *target); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index e54463ea202..e8ef80663f9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -12,7 +12,6 @@ namespace search { class IAttributeManager; - class RawBuf; } namespace vespalib { class Slime; } @@ -26,6 +25,7 @@ static constexpr uint32_t SLIME_MAGIC_ID = 0x55555555; class IDocsumWriter { public: + using Inserter = vespalib::slime::Inserter; struct ResolveClassInfo { bool mustSkip; bool allGenerated; @@ -38,13 +38,11 @@ public: virtual ~IDocsumWriter() = default; virtual void InitState(search::IAttributeManager & attrMan, GetDocsumsState *state) = 0; - virtual uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, search::RawBuf *target) = 0; + virtual void WriteDocsum(uint32_t docid, GetDocsumsState *state, + IDocsumStore *docinfos, Inserter & target) = 0; virtual void insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::slime::Inserter & target) = 0; + IDocsumStore *docinfos, Inserter & target) = 0; virtual ResolveClassInfo resolveClassInfo(vespalib::stringref outputClassName) const = 0; - - static uint32_t slime2RawBuf(const vespalib::Slime & slime, RawBuf & buf); }; //-------------------------------------------------------------------------- @@ -70,11 +68,11 @@ public: bool Override(const char *fieldName, std::unique_ptr<DocsumFieldWriter> writer); void InitState(search::IAttributeManager & attrMan, GetDocsumsState *state) override; - uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, search::RawBuf *target) override; + void WriteDocsum(uint32_t docid, GetDocsumsState *state, + IDocsumStore *docinfos, Inserter & inserter) override; void insertDocsum(const ResolveClassInfo & outputClassInfo, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::slime::Inserter & target) override; + IDocsumStore *docinfos, Inserter & inserter) override; ResolveClassInfo resolveClassInfo(vespalib::stringref outputClassName) const override; }; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index c2bbf4d09da..898860bdf73 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -3,6 +3,7 @@ #include "querytermdata.h" #include "searchenvironment.h" #include "searchvisitor.h" +#include "matching_elements_filler.h" #include <vespa/persistence/spi/docentry.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/datatype/documenttype.h> @@ -17,8 +18,8 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/data/slime/slime.h> #include <vespa/fnet/databuffer.h> -#include "matching_elements_filler.h" #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.searchvisitor"); @@ -92,7 +93,7 @@ createMultiValueAttribute(const vespalib::string & name, const document::FieldVa LOG(debug, "Can not make an multivalue attribute out of %s with data type '%s' (%s)", name.c_str(), ndt->getName().c_str(), fv.className()); } - return AttributeVector::SP(); + return {}; } AttributeVector::SP @@ -108,7 +109,7 @@ createAttribute(const vespalib::string & name, const document::FieldValue & fv) } else { LOG(debug, "Can not make an attribute out of %s of type '%s'.", name.c_str(), fv.className()); } - return AttributeVector::SP(); + return {}; } SearchVisitor::SummaryGenerator::SummaryGenerator() : @@ -117,7 +118,7 @@ SearchVisitor::SummaryGenerator::SummaryGenerator() : _docsumState(_callback), _docsumFilter(), _docsumWriter(nullptr), - _rawBuf(4_Ki) + _buf(4_Ki) { } @@ -128,12 +129,20 @@ vespalib::ConstBufferRef SearchVisitor::SummaryGenerator::fillSummary(AttributeVector::DocId lid, const HitsAggregationResult::SummaryClassType & summaryClass) { if (_docsumWriter != nullptr) { - _rawBuf.reset(); _docsumState._args.setResultClassName(summaryClass); - uint32_t docsumLen = _docsumWriter->WriteDocsum(lid, &_docsumState, _docsumFilter.get(), &_rawBuf); - return vespalib::ConstBufferRef(_rawBuf.GetDrainPos(), docsumLen); + vespalib::Slime slime; + vespalib::slime::SlimeInserter inserter(slime); + _docsumWriter->WriteDocsum(lid, &_docsumState, _docsumFilter.get(), inserter); + + _buf.reset(); + vespalib::WritableMemory magicId = _buf.reserve(4); + memcpy(magicId.data, &search::docsummary::SLIME_MAGIC_ID, 4); + _buf.commit(4); + vespalib::slime::BinaryFormat::encode(slime, _buf); + vespalib::Memory mem = _buf.obtain(); + return {mem.data, mem.size}; } - return vespalib::ConstBufferRef(); + return {}; } void SearchVisitor::HitsResultPreparator::execute(vespalib::Identifiable & obj) @@ -612,10 +621,10 @@ SearchVisitor::registerAdditionalFields(const std::vector<vsm::DocsumTools::Fiel for (const vsm::DocsumTools::FieldSpec & spec : docsumSpec) { fieldList.push_back(spec.getOutputName()); const std::vector<vespalib::string> & inputNames = spec.getInputNames(); - for (size_t j = 0; j < inputNames.size(); ++j) { - fieldList.push_back(inputNames[j]); - if (PositionDataType::isZCurveFieldName(inputNames[j])) { - fieldList.emplace_back(PositionDataType::cutZCurveFieldName(inputNames[j])); + for (const auto & name : inputNames) { + fieldList.push_back(name); + if (PositionDataType::isZCurveFieldName(name)) { + fieldList.emplace_back(PositionDataType::cutZCurveFieldName(name)); } } } @@ -732,7 +741,7 @@ SearchVisitor::setupAttributeVectors() void SearchVisitor::setupAttributeVector(const FieldPath &fieldPath) { vespalib::string attrName(fieldPath.front().getName()); - for (FieldPath::const_iterator ft(fieldPath.begin() + 1), fmt(fieldPath.end()); ft != fmt; ft++) { + for (auto ft(fieldPath.begin() + 1), fmt(fieldPath.end()); ft != fmt; ft++) { attrName.append("."); attrName.append((*ft)->getName()); } @@ -855,7 +864,7 @@ private: bool SearchVisitor::compatibleDocumentTypes(const document::DocumentType& typeA, - const document::DocumentType& typeB) const + const document::DocumentType& typeB) { if (&typeA == &typeB) { return true; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 2b88286d381..98aa6b89c9c 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -18,13 +18,13 @@ #include <vespa/vsm/vsm/vsm-adapter.h> #include <vespa/vespalib/objects/objectoperation.h> #include <vespa/vespalib/objects/objectpredicate.h> +#include <vespa/vespalib/data/smart_buffer.h> #include <vespa/searchlib/query/streaming/query.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/attributemanager.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/common/sortspec.h> -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/storage/visiting/visitor.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/documentapi/messagebus/messages/queryresultmessage.h> @@ -45,7 +45,7 @@ public: SearchVisitor(storage::StorageComponent&, storage::VisitorEnvironment& vEnv, const vdslib::Parameters & params); - ~SearchVisitor(); + ~SearchVisitor() override; private: /** * This struct wraps an attribute vector. @@ -102,7 +102,7 @@ private: class PositionInserter : public AttributeInserter { public: PositionInserter(search::AttributeVector & attribute, search::AttributeVector::DocId docId); - ~PositionInserter(); + ~PositionInserter() override; private: void onPrimitive(uint32_t fid, const Content & c) override; void onStructStart(const Content & fv) override; @@ -129,9 +129,9 @@ private: /** * Process attribute hints and add needed attributes to the given list. **/ - void processHintedAttributes(const IndexEnvironment & indexEnv, bool rank, - const search::IAttributeManager & attrMan, - std::vector<AttrInfo> & attributeFields); + static void processHintedAttributes(const IndexEnvironment & indexEnv, bool rank, + const search::IAttributeManager & attrMan, + std::vector<AttrInfo> & attributeFields); public: RankController(); @@ -245,8 +245,8 @@ private: * @param docsumSpec config with the field names used by the docsum setup. * @param fieldList list of field names that are built. **/ - void registerAdditionalFields(const std::vector<vsm::DocsumTools::FieldSpec> & docsumSpec, - std::vector<vespalib::string> & fieldList); + static void registerAdditionalFields(const std::vector<vsm::DocsumTools::FieldSpec> & docsumSpec, + std::vector<vespalib::string> & fieldList); /** * Setup the field searchers used when matching the query with the stream of documents. @@ -302,8 +302,8 @@ private: DocEntryList& entries, HitCounter& hitCounter) override; - bool compatibleDocumentTypes(const document::DocumentType& typeA, - const document::DocumentType& typeB) const; + static bool compatibleDocumentTypes(const document::DocumentType& typeA, + const document::DocumentType& typeB); /** * Process one document @@ -370,7 +370,7 @@ private: class GroupingEntry : std::shared_ptr<Grouping> { public: - GroupingEntry(Grouping * grouping); + explicit GroupingEntry(Grouping * grouping); ~GroupingEntry(); void aggregate(const document::Document & doc, search::HitRank rank); const Grouping & operator * () const { return *_grouping; } @@ -388,32 +388,32 @@ private: { public: SummaryGenerator(); - ~SummaryGenerator(); + ~SummaryGenerator() override; GetDocsumsState & getDocsumState() { return _docsumState; } vsm::GetDocsumsStateCallback & getDocsumCallback() { return _callback; } void setFilter(std::unique_ptr<vsm::DocsumFilter> filter) { _docsumFilter = std::move(filter); } void setDocsumCache(const vsm::IDocSumCache & cache) { _docsumFilter->setDocSumStore(cache); } void setDocsumWriter(IDocsumWriter & docsumWriter) { _docsumWriter = & docsumWriter; } - virtual vespalib::ConstBufferRef fillSummary(search::AttributeVector::DocId lid, const HitsAggregationResult::SummaryClassType & summaryClass) override; + vespalib::ConstBufferRef fillSummary(search::AttributeVector::DocId lid, const HitsAggregationResult::SummaryClassType & summaryClass) override; private: vsm::GetDocsumsStateCallback _callback; GetDocsumsState _docsumState; std::unique_ptr<vsm::DocsumFilter> _docsumFilter; search::docsummary::IDocsumWriter * _docsumWriter; - search::RawBuf _rawBuf; + vespalib::SmartBuffer _buf; }; class HitsResultPreparator : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { public: - HitsResultPreparator(SummaryGenerator & summaryGenerator) : + explicit HitsResultPreparator(SummaryGenerator & summaryGenerator) : _summaryGenerator(summaryGenerator), _numHitsAggregators(0) { } size_t getNumHitsAggregators() const { return _numHitsAggregators; } private: - virtual void execute(vespalib::Identifiable &obj) override; - virtual bool check(const vespalib::Identifiable &obj) const override; + void execute(vespalib::Identifiable &obj) override; + bool check(const vespalib::Identifiable &obj) const override; SummaryGenerator & _summaryGenerator; size_t _numHitsAggregators; }; @@ -462,7 +462,7 @@ class SearchVisitorFactory : public storage::VisitorFactory { storage::Visitor* makeVisitor(storage::StorageComponent&, storage::VisitorEnvironment&env, const vdslib::Parameters& params) override; public: - SearchVisitorFactory(const config::ConfigUri & configUri); + explicit SearchVisitorFactory(const config::ConfigUri & configUri); }; } diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp index de079261d6d..8b7bed62ac5 100644 --- a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp @@ -32,8 +32,7 @@ SmartBuffer::drop() { alloc::Alloc empty_buf; _data.swap(empty_buf); - _read_pos = 0; - _write_pos = 0; + reset(); } SmartBuffer::SmartBuffer(size_t initial_size) diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.h b/vespalib/src/vespa/vespalib/data/smart_buffer.h index eb817e71bbf..17fb7614f0e 100644 --- a/vespalib/src/vespa/vespalib/data/smart_buffer.h +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.h @@ -39,6 +39,10 @@ public: drop(); } } + void reset() { + _read_pos = 0; + _write_pos = 0; + } Memory obtain() override; Input &evict(size_t bytes) override; WritableMemory reserve(size_t bytes) override; |