diff options
42 files changed, 102 insertions, 2477 deletions
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 1098de4438f..f4ee877da0d 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -75,8 +75,6 @@ using vespalib::GateCallback; using vespalib::Slime; using namespace vespalib::slime; -typedef std::unique_ptr<GeneralResult> GeneralResultPtr; - namespace proton { class DirMaker @@ -95,26 +93,6 @@ private: vespalib::string _dir; }; -DocsumBlobEntryFilter make_blob_entry_filter() -{ - return DocsumBlobEntryFilter(). - add_skip(RES_INT). - add_skip(RES_SHORT). - add_skip(RES_BOOL). - add_skip(RES_BYTE). - add_skip(RES_FLOAT). - add_skip(RES_DOUBLE). - add_skip(RES_INT64). - add_skip(RES_STRING). - add_skip(RES_DATA). - add_skip(RES_LONG_STRING). - add_skip(RES_LONG_DATA). - add_skip(RES_JSONSTRING). - add_skip(RES_TENSOR). - add_skip(RES_FEATUREDATA); -} - - class BuildContext { public: @@ -330,21 +308,11 @@ public: } }; -GeneralResultPtr -getResult(DocumentStoreAdapter & dsa, const DocsumStoreValue& docsum) -{ - ASSERT_TRUE(docsum.pt() != nullptr); - auto retval = std::make_unique<GeneralResult>(dsa.getResultClass()); - ASSERT_TRUE(retval->inplaceUnpack(docsum)); - return retval; -} - bool assertString(const std::string & exp, const std::string & fieldName, DocumentStoreAdapter &dsa, uint32_t id) { - auto docsum = dsa.getMappedDocsum(id); - GeneralResultPtr res = getResult(dsa, docsum); + auto res = dsa.getMappedDocsum(id); return EXPECT_EQUAL(exp, res->get_field_value(fieldName)->getAsString()); } @@ -352,10 +320,8 @@ bool assertAnnotatedString(const std::string & exp, const std::string & fieldName, DocumentStoreAdapter &dsa, uint32_t id) { - auto docsum = dsa.getMappedDocsum(id); - GeneralResultPtr res = getResult(dsa, docsum); - auto document = res->get_document(); - return EXPECT_EQUAL(exp, document->get_juniper_input(fieldName).get_value()); + auto res = dsa.getMappedDocsum(id); + return EXPECT_EQUAL(exp, res->get_juniper_input(fieldName).get_value()); } void @@ -423,8 +389,7 @@ TEST_F("requireThatAdapterHandlesAllFieldTypes", Fixture) f.getResultConfig(), "class0", bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class0"), f.getMarkupFields()); - auto docsum = dsa.getMappedDocsum(0); - GeneralResultPtr res = getResult(dsa, docsum); + auto res = dsa.getMappedDocsum(0); EXPECT_EQUAL(-1, res->get_field_value("a")->getAsInt()); EXPECT_EQUAL(32767, res->get_field_value("b")->getAsInt()); EXPECT_EQUAL(2147483647, res->get_field_value("c")->getAsInt()); @@ -459,22 +424,19 @@ TEST_F("requireThatAdapterHandlesMultipleDocuments", Fixture) bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class1"), f.getMarkupFields()); { // doc 0 - auto docsum = dsa.getMappedDocsum(0); - GeneralResultPtr res = getResult(dsa, docsum); + auto res = dsa.getMappedDocsum(0); EXPECT_EQUAL(1000, res->get_field_value("a")->getAsInt()); } { // doc 1 - auto docsum = dsa.getMappedDocsum(1); - GeneralResultPtr res = getResult(dsa, docsum); + auto res = dsa.getMappedDocsum(1); EXPECT_EQUAL(2000, res->get_field_value("a")->getAsInt()); } { // doc 2 - auto docsum = dsa.getMappedDocsum(2); - EXPECT_TRUE(docsum.pt() == nullptr); + auto res = dsa.getMappedDocsum(2); + EXPECT_TRUE(!res); } { // doc 0 (again) - auto docsum = dsa.getMappedDocsum(0); - GeneralResultPtr res = getResult(dsa, docsum); + auto res = dsa.getMappedDocsum(0); EXPECT_EQUAL(1000, res->get_field_value("a")->getAsInt()); } EXPECT_EQUAL(0u, bc._str.lastSyncToken()); @@ -495,11 +457,10 @@ TEST_F("requireThatAdapterHandlesDocumentIdField", Fixture) DocumentStoreAdapter dsa(bc._str, *bc._repo, f.getResultConfig(), "class4", bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class4"), f.getMarkupFields()); - auto docsum = dsa.getMappedDocsum(0); - GeneralResultPtr res = getResult(dsa, docsum); + auto res = dsa.getMappedDocsum(0); vespalib::Slime slime; vespalib::slime::SlimeInserter inserter(slime); - res->get_document()->insert_document_id(inserter); + res->insert_document_id(inserter); EXPECT_EQUAL("id:ns:searchdocument::0", slime.get().asString().make_string()); } @@ -988,8 +949,7 @@ TEST_F("requireThatUrisAreUsed", Fixture) DocumentStoreAdapter dsa(store, *bc._repo, f.getResultConfig(), "class0", bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class0"), f.getMarkupFields()); - auto docsum = dsa.getMappedDocsum(1); - GeneralResultPtr res = getResult(dsa, docsum); + auto res = dsa.getMappedDocsum(1); EXPECT_EQUAL("http://www.example.com:81/fluke?ab=2#4", SummaryFieldConverter::convertSummaryField(false, *res->get_field_value("urisingle"))->getAsString()); { vespalib::Slime slime; @@ -1119,8 +1079,7 @@ TEST_F("requireThatRawFieldsWorks", Fixture) ASSERT_TRUE(assertString(raw1s, "i", dsa, 1)); - auto docsum = dsa.getMappedDocsum(1); - GeneralResultPtr res = getResult(dsa, docsum); + auto res = dsa.getMappedDocsum(1); { vespalib::Slime slime; decode(*SummaryFieldConverter::convertSummaryField(false, *res->get_field_value("araw")), slime); @@ -1155,7 +1114,7 @@ TEST_F("requireThatFieldCacheRepoCanReturnDefaultFieldCache", Fixture) Fixture::Fixture() : _summaryCfg(), - _resultCfg(make_blob_entry_filter()), + _resultCfg(), _markupFields() { std::string cfgId("summary"); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index 2b7f5509acb..353ee8d73fb 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -86,7 +86,7 @@ DocsumContext::createSlimeReply() Cursor &docSumC = array.addObject(); ObjectSymbolInserter inserter(docSumC, docsumSym); if ((docId != search::endDocId) && !rci.mustSkip) { - _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, *response, inserter); + _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, inserter); } num_ok++; } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp index ee5be6abdcf..90850007467 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp @@ -22,128 +22,6 @@ const vespalib::string DOCUMENT_ID_FIELD("documentid"); } -bool -DocumentStoreAdapter::writeStringField(const char * buf, uint32_t buflen, ResType type) -{ - switch (type) { - case RES_STRING: - return _resultPacker.AddString(buf, buflen); - case RES_LONG_STRING: - case RES_JSONSTRING: - return _resultPacker.AddLongString(buf, buflen); - default: - break; - } - return false; -} - -bool -DocumentStoreAdapter::writeField(const FieldValue &value, ResType type) -{ - switch (type) { - case RES_BYTE: - return _resultPacker.AddByte(value.getAsInt()); - case RES_BOOL: - return _resultPacker.AddByte(value.getAsInt()); - case RES_SHORT: - return _resultPacker.AddShort(value.getAsInt()); - case RES_INT: - return _resultPacker.AddInteger(value.getAsInt()); - case RES_INT64: - return _resultPacker.AddInt64(value.getAsLong()); - case RES_FLOAT: - return _resultPacker.AddFloat(value.getAsFloat()); - case RES_DOUBLE: - return _resultPacker.AddDouble(value.getAsDouble()); - case RES_STRING: - case RES_LONG_STRING: - case RES_JSONSTRING: - { - if (value.isLiteral()) { - auto & lfv = static_cast<const LiteralFieldValueB &>(value); - vespalib::stringref s = lfv.getValueRef(); - return writeStringField(s.data(), s.size(), type); - } else { - vespalib::string s = value.getAsString(); - return writeStringField(s.data(), s.size(), type); - } - } - case RES_DATA: - { - std::pair<const char *, size_t> buf = value.getAsRaw(); - return _resultPacker.AddData(buf.first, buf.second); - } - case RES_LONG_DATA: - { - std::pair<const char *, size_t> buf = value.getAsRaw(); - return _resultPacker.AddLongData(buf.first, buf.second); - } - case RES_TENSOR: - { - vespalib::nbostream serialized; - if (value.isA(FieldValue::Type::TENSOR)) { - const auto &tvalue = static_cast<const TensorFieldValue &>(value); - auto tensor = tvalue.getAsTensorPtr(); - if (tensor) { - encode_value(*tensor, serialized); - } - } - return _resultPacker.AddSerializedTensor(serialized.peek(), serialized.size()); - } - default: - LOG(warning, "Unknown docsum field type: %s. Add empty field",ResultConfig::GetResTypeName(type)); - return _resultPacker.AddEmpty(); - } - return false; -} - - -void -DocumentStoreAdapter::convertFromSearchDoc(Document &doc, uint32_t docId) -{ - for (size_t i = 0; i < _resultClass->GetNumEntries(); ++i) { - if (i != _resultPacker.get_entry_idx()) { - // Entry is not present in docsum blob - continue; - } - const ResConfigEntry * entry = _resultClass->GetEntry(i); - const vespalib::string fieldName(entry->_bindname); - bool markup = _markupFields.find(fieldName) != _markupFields.end(); - if (fieldName == DOCUMENT_ID_FIELD) { - StringFieldValue value(doc.getId().toString()); - if (!writeField(value, entry->_type)) { - LOG(warning, "Error while writing field '%s' for docId %u", fieldName.c_str(), docId); - } - continue; - } - const Field *field = _fieldCache->getField(i); - if (!field) { - LOG(debug, "Did not find field '%s' in the document for docId %u. Adding empty field", - fieldName.c_str(), docId); - _resultPacker.AddEmpty(); - continue; - } - FieldValue::UP fieldValue = doc.getValue(*field); - if ( ! fieldValue) { - LOG(spam, "No field value for field '%s' in the document for docId %u. Adding empty field", - fieldName.c_str(), docId); - _resultPacker.AddEmpty(); - continue; - } - LOG(spam, "writeField(%s): value(%s), type(%d)", fieldName.c_str(), fieldValue->toString().c_str(), entry->_type); - FieldValue::UP convertedFieldValue = SummaryFieldConverter::convertSummaryField(markup, *fieldValue); - if (convertedFieldValue) { - if (!writeField(*convertedFieldValue, entry->_type)) { - LOG(warning, "Error while writing field '%s' for docId %u", fieldName.c_str(), docId); - } - } else { - LOG(spam, "No converted field value for field '%s' in the document for docId %u. Adding empty field", - fieldName.c_str(), docId); - _resultPacker.AddEmpty(); - } - } -} - DocumentStoreAdapter:: DocumentStoreAdapter(const search::IDocumentStore & docStore, const DocumentTypeRepo &repo, @@ -156,7 +34,6 @@ DocumentStoreAdapter(const search::IDocumentStore & docStore, _resultConfig(resultConfig), _resultClass(resultConfig. LookupResultClass(resultConfig.LookupResultClassId(resultClassName.c_str()))), - _resultPacker(&_resultConfig), _fieldCache(fieldCache), _markupFields(markupFields) { @@ -164,27 +41,16 @@ DocumentStoreAdapter(const search::IDocumentStore & docStore, DocumentStoreAdapter::~DocumentStoreAdapter() = default; -DocsumStoreValue +std::unique_ptr<const IDocsumStoreDocument> DocumentStoreAdapter::getMappedDocsum(uint32_t docId) { - if (!_resultPacker.Init(getSummaryClassId())) { - LOG(warning, "Error during init of result class '%s' with class id %u", _resultClass->GetClassName(), getSummaryClassId()); - return DocsumStoreValue(); - } Document::UP document = _docStore.read(docId, _repo); if ( ! document) { LOG(debug, "Did not find summary document for docId %u. Returning empty docsum", docId); - return DocsumStoreValue(); + return {}; } LOG(spam, "getMappedDocSum(%u): document={\n%s\n}", docId, document->toString(true).c_str()); - convertFromSearchDoc(*document, docId); - const char * buf; - uint32_t buflen; - if (!_resultPacker.GetDocsumBlob(&buf, &buflen)) { - LOG(warning, "Error while getting the docsum blob for docId %u. Returning empty docsum", docId); - return DocsumStoreValue(); - } - return DocsumStoreValue(buf, buflen, std::make_unique<DocsumStoreDocument>(std::move(document))); + return std::make_unique<DocsumStoreDocument>(std::move(document)); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h index cef62751339..7a3a810bb55 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h @@ -5,7 +5,6 @@ #include "fieldcache.h" #include <vespa/searchsummary/docsummary/docsumstore.h> #include <vespa/searchsummary/docsummary/resultconfig.h> -#include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/docstore/idocumentstore.h> @@ -18,22 +17,9 @@ private: const document::DocumentTypeRepo & _repo; const search::docsummary::ResultConfig & _resultConfig; const search::docsummary::ResultClass * _resultClass; - search::docsummary::ResultPacker _resultPacker; FieldCache::CSP _fieldCache; const std::set<vespalib::string> & _markupFields; - bool - writeStringField(const char * buf, - uint32_t buflen, - search::docsummary::ResType type); - - bool - writeField(const document::FieldValue &value, - search::docsummary::ResType type); - - void - convertFromSearchDoc(document::Document &doc, uint32_t docId); - public: DocumentStoreAdapter(const search::IDocumentStore &docStore, const document::DocumentTypeRepo &repo, @@ -48,7 +34,7 @@ public: } uint32_t getNumDocs() const override { return _docStore.getDocIdLimit(); } - search::docsummary::DocsumStoreValue getMappedDocsum(uint32_t docId) override; + std::unique_ptr<const search::docsummary::IDocsumStoreDocument> getMappedDocsum(uint32_t docId) override; uint32_t getSummaryClassId() const override { return _resultClass->GetClassID(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 4e1fdff6d13..28a43325e50 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -93,22 +93,7 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, _repo(repo), _markupFields() { - DocsumBlobEntryFilter docsum_blob_entry_filter; - docsum_blob_entry_filter.add_skip(RES_INT); - docsum_blob_entry_filter.add_skip(RES_SHORT); - docsum_blob_entry_filter.add_skip(RES_BOOL); - docsum_blob_entry_filter.add_skip(RES_BYTE); - docsum_blob_entry_filter.add_skip(RES_FLOAT); - docsum_blob_entry_filter.add_skip(RES_DOUBLE); - docsum_blob_entry_filter.add_skip(RES_INT64); - docsum_blob_entry_filter.add_skip(RES_STRING); - docsum_blob_entry_filter.add_skip(RES_DATA); - docsum_blob_entry_filter.add_skip(RES_LONG_STRING); - docsum_blob_entry_filter.add_skip(RES_LONG_DATA); - docsum_blob_entry_filter.add_skip(RES_JSONSTRING); - docsum_blob_entry_filter.add_skip(RES_TENSOR); - docsum_blob_entry_filter.add_skip(RES_FEATUREDATA); - auto resultConfig = std::make_unique<ResultConfig>(docsum_blob_entry_filter); + auto resultConfig = std::make_unique<ResultConfig>(); if (!resultConfig->ReadConfig(summaryCfg, make_string("SummaryManager(%s)", baseDir.c_str()).c_str())) { std::ostringstream oss; ::config::OstreamConfigWriter writer(oss); diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 8c34f1b2016..60f85c07b86 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -15,7 +15,6 @@ vespa_define_module( src/vespa/searchsummary/test TESTS - src/tests/docsumformat src/tests/docsummary src/tests/docsummary/attribute_combiner src/tests/docsummary/attributedfw diff --git a/searchsummary/src/tests/docsumformat/.gitignore b/searchsummary/src/tests/docsumformat/.gitignore deleted file mode 100644 index 6612a2f38bb..00000000000 --- a/searchsummary/src/tests/docsumformat/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -*.cfg -*.core -*.ilk -*.out -*.pdb -*.pid -.depend -Makefile -core -core.* -searchsummary_docsum-pack_app diff --git a/searchsummary/src/tests/docsumformat/CMakeLists.txt b/searchsummary/src/tests/docsumformat/CMakeLists.txt deleted file mode 100644 index 147b90dffbe..00000000000 --- a/searchsummary/src/tests/docsumformat/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchsummary_docsum-pack_app TEST - SOURCES - docsum-pack.cpp - DEPENDS - searchsummary -) -vespa_add_test(NAME searchsummary_docsum-pack_app COMMAND searchsummary_docsum-pack_app) diff --git a/searchsummary/src/tests/docsumformat/docsum-pack.cpp b/searchsummary/src/tests/docsumformat/docsum-pack.cpp deleted file mode 100644 index 07aceea83e0..00000000000 --- a/searchsummary/src/tests/docsumformat/docsum-pack.cpp +++ /dev/null @@ -1,380 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/searchsummary/docsummary/general_result.h> -#include <vespa/searchsummary/docsummary/resultconfig.h> -#include <vespa/searchsummary/docsummary/resultpacker.h> -#include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/signalhandler.h> -#include <vespa/log/log.h> -LOG_SETUP("docsum-pack"); - -using namespace search::docsummary; - -// needed to resolve external symbol from httpd.h on AIX -void FastS_block_usr2() {} - - -class MyApp -{ -private: - bool _rc; - uint32_t _cnt; - ResultConfig _config; - ResultPacker _packer; - -public: - MyApp(); - ~MyApp(); - - // log test results - void ReportTestResult(uint32_t line, bool rc); - bool RTR(uint32_t line, bool rc) - { ReportTestResult(line, rc); return rc; } - - // compare runtime info (,but ignore result class) - bool Equal(ResEntry *a, ResEntry *b); - bool Equal(GeneralResult *a, GeneralResult *b); - - void TestIntValue(uint32_t line, GeneralResult *gres, const char *field, uint32_t value); - void TestDoubleValue(uint32_t line, GeneralResult *gres, const char *field, double value); - void TestInt64Value(uint32_t line, GeneralResult *gres, const char *field, uint64_t value); - void TestStringValue(uint32_t line, GeneralResult *gres, const char *field, const char *value); - void TestDataValue(uint32_t line, GeneralResult *gres, const char *field, const char *value); - - void TestFailLong(); - void TestFailShort(); - void TestFailOrder(); - void TestBasicInplace(); - - int main(int argc, char **argv); -}; - -MyApp::MyApp() - : _rc(false), - _cnt(0u), - _config(), - _packer(&_config) -{} - -MyApp::~MyApp() = default; - -void -MyApp::ReportTestResult(uint32_t line, bool rc) -{ - _cnt++; - - if (rc) { - LOG(info, "Test case %d: SUCCESS", _cnt); - } else { - LOG(error, "Test case %d: FAIL (see %s:%d)", _cnt, __FILE__, line); - _rc = false; - } -} - - -bool -MyApp::Equal(ResEntry *a, ResEntry *b) -{ - if (a->_type != b->_type) - return false; - - if (a->_intval != b->_intval) - return false; - - if (a->_type != RES_INT && - memcmp(a->_pt, b->_pt, a->_intval) != 0) - return false; - - return true; -} - - -bool -MyApp::Equal(GeneralResult *a, GeneralResult *b) -{ - uint32_t numEntries = a->GetClass()->GetNumEntries(); - - if (b->GetClass()->GetNumEntries() != numEntries) - return false; - - for (uint32_t i = 0; i < numEntries; i++) { - - if (!Equal(a->GetEntry(i), b->GetEntry(i))) - return false; - - if (a->GetClass()->GetEntry(i)->_bindname != b->GetClass()->GetEntry(i)->_bindname) - return false; - } - - return true; -} - -void -MyApp::TestIntValue(uint32_t line, GeneralResult *gres, const char *field, uint32_t value) -{ - ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; - - bool rc = (entry != nullptr && - entry->_type == RES_INT && - entry->_intval == value); - - RTR(line, rc); -} - -void -MyApp::TestDoubleValue(uint32_t line, GeneralResult *gres, const char *field, double value) -{ - ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; - - bool rc = (entry != nullptr && - entry->_type == RES_DOUBLE && - entry->_doubleval == value); - - RTR(line, rc); -} - -void -MyApp::TestInt64Value(uint32_t line, GeneralResult *gres, const char *field, uint64_t value) -{ - ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; - - bool rc = (entry != nullptr && - entry->_type == RES_INT64 && - entry->_int64val == value); - - RTR(line, rc); -} - - -void -MyApp::TestStringValue(uint32_t line, GeneralResult *gres, const char *field, const char *value) -{ - ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; - - bool rc = (entry != nullptr && - entry->_type == RES_STRING && - entry->_stringlen == strlen(value) && - strncmp(entry->_stringval, value, entry->_stringlen) == 0); - - if (!rc && entry != nullptr) { - LOG(warning,"string value '%.*s' != '%s'", - (int) entry->_stringlen, entry->_stringval, value); - } - - RTR(line, rc); -} - -void -MyApp::TestDataValue(uint32_t line, GeneralResult *gres, const char *field, const char *value) -{ - ResEntry *entry = (gres != nullptr) ? gres->GetPresentEntry(field) : nullptr; - - bool rc = (entry != nullptr && - entry->_type == RES_DATA && - entry->_datalen == strlen(value) && - strncmp(entry->_dataval, value, entry->_datalen) == 0); - - RTR(line, rc); -} - -void -MyApp::TestFailLong() -{ - const char *buf; - uint32_t buflen; - - uint32_t intval = 4; - uint16_t shortval = 2; - uint8_t byteval = 1; - float floatval = 4.5; - double doubleval = 8.75; - uint64_t int64val = 8; - const char *strval = "This is a string"; - const char *datval = "This is data"; - const char *lstrval = "This is a long string"; - const char *ldatval = "This is long data"; - - RTR(__LINE__, _packer.Init(0)); - RTR(__LINE__, _packer.AddInteger(intval)); - RTR(__LINE__, _packer.AddShort(shortval)); - RTR(__LINE__, _packer.AddByte(byteval)); - RTR(__LINE__, _packer.AddFloat(floatval)); - RTR(__LINE__, _packer.AddDouble(doubleval)); - RTR(__LINE__, _packer.AddInt64(int64val)); - RTR(__LINE__, _packer.AddString(strval, strlen(strval))); - RTR(__LINE__, _packer.AddData(datval, strlen(datval))); - RTR(__LINE__, _packer.AddLongString(lstrval, strlen(lstrval))); - RTR(__LINE__, _packer.AddLongData(ldatval, strlen(ldatval))); - RTR(__LINE__, !_packer.AddByte(byteval)); - RTR(__LINE__, !_packer.GetDocsumBlob(&buf, &buflen)); -} - -void -MyApp::TestFailShort() -{ - const char *buf; - uint32_t buflen; - - uint32_t intval = 4; - uint16_t shortval = 2; - uint8_t byteval = 1; - float floatval = 4.5; - double doubleval = 8.75; - uint64_t int64val = 8; - const char *strval = "This is a string"; - const char *datval = "This is data"; - const char *lstrval = "This is a long string"; - - RTR(__LINE__, _packer.Init(0)); - RTR(__LINE__, _packer.AddInteger(intval)); - RTR(__LINE__, _packer.AddShort(shortval)); - RTR(__LINE__, _packer.AddByte(byteval)); - RTR(__LINE__, _packer.AddFloat(floatval)); - RTR(__LINE__, _packer.AddDouble(doubleval)); - RTR(__LINE__, _packer.AddInt64(int64val)); - RTR(__LINE__, _packer.AddString(strval, strlen(strval))); - RTR(__LINE__, _packer.AddData(datval, strlen(datval))); - RTR(__LINE__, _packer.AddLongString(lstrval, strlen(lstrval))); - RTR(__LINE__, !_packer.GetDocsumBlob(&buf, &buflen)); -} - - -void -MyApp::TestFailOrder() -{ - const char *buf; - uint32_t buflen; - - uint32_t intval = 4; - uint16_t shortval = 2; - uint8_t byteval = 1; - float floatval = 4.5; - double doubleval = 8.75; - uint64_t int64val = 8; - const char *strval = "This is a string"; - const char *datval = "This is data"; - const char *lstrval = "This is a long string"; - const char *ldatval = "This is long data"; - - RTR(__LINE__, _packer.Init(0)); - RTR(__LINE__, _packer.AddInteger(intval)); - RTR(__LINE__, _packer.AddShort(shortval)); - RTR(__LINE__, !_packer.AddString(strval, strlen(strval))); - RTR(__LINE__, !_packer.AddByte(byteval)); - RTR(__LINE__, !_packer.AddFloat(floatval)); - RTR(__LINE__, !_packer.AddDouble(doubleval)); - RTR(__LINE__, !_packer.AddInt64(int64val)); - RTR(__LINE__, !_packer.AddData(datval, strlen(datval))); - RTR(__LINE__, !_packer.AddLongString(lstrval, strlen(lstrval))); - RTR(__LINE__, !_packer.AddLongData(ldatval, strlen(ldatval))); - RTR(__LINE__, !_packer.GetDocsumBlob(&buf, &buflen)); -} - - - -void -MyApp::TestBasicInplace() -{ - const char *buf; - uint32_t buflen; - - const ResultClass *resClass; - GeneralResult *gres; - - uint32_t intval = 4; - uint16_t shortval = 2; - uint8_t byteval = 1; - float floatval = 4.5; - double doubleval = 8.75; - uint64_t int64val = 8; - const char *strval = "This is a string"; - const char *datval = "This is data"; - const char *lstrval = "This is a long string"; - const char *ldatval = "This is long data"; - - RTR(__LINE__, _packer.Init(0)); - RTR(__LINE__, _packer.AddInteger(intval)); - RTR(__LINE__, _packer.AddShort(shortval)); - RTR(__LINE__, _packer.AddByte(byteval)); - RTR(__LINE__, _packer.AddFloat(floatval)); - RTR(__LINE__, _packer.AddDouble(doubleval)); - RTR(__LINE__, _packer.AddInt64(int64val)); - RTR(__LINE__, _packer.AddString(strval, strlen(strval))); - RTR(__LINE__, _packer.AddData(datval, strlen(datval))); - RTR(__LINE__, _packer.AddLongString(lstrval, strlen(lstrval))); - RTR(__LINE__, _packer.AddLongData(ldatval, strlen(ldatval))); - RTR(__LINE__, _packer.GetDocsumBlob(&buf, &buflen)); - - resClass = _config.LookupResultClass(_config.GetClassID(buf, buflen)); - if (resClass == nullptr) { - gres = nullptr; - } else { - DocsumStoreValue value(buf, buflen); - gres = new GeneralResult(resClass); - if (!gres->inplaceUnpack(value)) { - delete gres; - gres = nullptr; - } - } - - RTR(__LINE__, gres != nullptr); - TestIntValue (__LINE__, gres, "integer", 4); - TestIntValue (__LINE__, gres, "short", 2); - TestIntValue (__LINE__, gres, "byte", 1); - TestDoubleValue(__LINE__, gres, "float", floatval); - TestDoubleValue(__LINE__, gres, "double", doubleval); - TestInt64Value (__LINE__, gres, "int64", int64val); - TestStringValue(__LINE__, gres, "string", strval); - TestDataValue (__LINE__, gres, "data", datval); - TestStringValue(__LINE__, gres, "longstring", lstrval); - TestDataValue (__LINE__, gres, "longdata", ldatval); - RTR(__LINE__, (gres != nullptr && - gres->GetClass()->GetNumEntries() == 10)); - RTR(__LINE__, (gres != nullptr && - gres->GetClass()->GetClassID() == 0)); - delete gres; -} - -int -MyApp::main(int, char **) -{ - _rc = true; - _cnt = 0; - - ResultClass *resClass; - - resClass = _config.AddResultClass("c0", 0); - resClass->AddConfigEntry("integer", RES_INT); - resClass->AddConfigEntry("short", RES_SHORT); - resClass->AddConfigEntry("byte", RES_BYTE); - resClass->AddConfigEntry("float", RES_FLOAT); - resClass->AddConfigEntry("double", RES_DOUBLE); - resClass->AddConfigEntry("int64", RES_INT64); - resClass->AddConfigEntry("string", RES_STRING); - resClass->AddConfigEntry("data", RES_DATA); - resClass->AddConfigEntry("longstring", RES_LONG_STRING); - resClass->AddConfigEntry("longdata", RES_LONG_DATA); - - resClass = _config.AddResultClass("c1", 1); - resClass->AddConfigEntry("text", RES_STRING); - resClass->AddConfigEntry("data", RES_DATA); - - resClass = _config.AddResultClass("c2", 2); - resClass->AddConfigEntry("text", RES_LONG_STRING); - resClass->AddConfigEntry("data", RES_LONG_DATA); - - TestFailLong(); - TestFailShort(); - TestFailOrder(); - TestBasicInplace(); - - LOG(info, "CONCLUSION: %s", (_rc) ? "SUCCESS" : "FAIL"); - return (_rc ? 0 : 1); -} - -int main(int argc, char **argv) { - vespalib::SignalHandler::PIPE.ignore(); - MyApp myapp; - return myapp.main(argc, argv); -} diff --git a/searchsummary/src/tests/docsumformat/dotest.sh b/searchsummary/src/tests/docsumformat/dotest.sh deleted file mode 100755 index f619a392bc9..00000000000 --- a/searchsummary/src/tests/docsumformat/dotest.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e - -echo "running pack test..." -./docsum-pack > packtest.out 2>&1 -res=$? -if [ $res -eq 0 ]; then - echo "pack test PASSED" -else - echo "pack test FAILED!" - echo "please check packtest.out" - exit 1 -fi diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp index 0ea78b722c0..9143f17cb67 100644 --- a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp +++ b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp @@ -5,12 +5,10 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/searchsummary/docsummary/docsum_blob_entry_filter.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/searchsummary/docsummary/document_id_dfw.h> -#include <vespa/searchsummary/docsummary/general_result.h> +#include <vespa/searchsummary/docsummary/resultclass.h> #include <vespa/searchsummary/docsummary/resultconfig.h> -#include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <iostream> @@ -22,15 +20,11 @@ using document::DocumentType; using document::DocumentTypeRepo; using document::config_builder::DocumenttypesConfigBuilderHelper; using document::config_builder::Struct; -using search::docsummary::DocsumBlobEntryFilter; using search::docsummary::DocsumStoreDocument; -using search::docsummary::DocsumStoreValue; using search::docsummary::DocumentIdDFW; using search::docsummary::IDocsumStoreDocument; using search::docsummary::ResultClass; using search::docsummary::ResultConfig; -using search::docsummary::ResultPacker; -using search::docsummary::GeneralResult; using vespalib::Slime; using vespalib::slime::Cursor; using vespalib::slime::ObjectInserter; @@ -57,9 +51,7 @@ class DocumentIdDFWTest : public ::testing::Test { vespalib::string _field_name; vespalib::Memory _field_name_view; - DocsumBlobEntryFilter _docsum_blob_entry_filter; std::unique_ptr<ResultConfig> _result_config; - std::unique_ptr<ResultPacker> _packer; // owns docsum blob std::unique_ptr<const DocumentTypeRepo> _repo; const DocumentType* _document_type; @@ -67,9 +59,8 @@ protected: DocumentIdDFWTest(); ~DocumentIdDFWTest() override; - std::unique_ptr<DocsumStoreDocument> make_docsum_store_document(const vespalib::string &id); - std::unique_ptr<DocsumStoreValue> make_docsum_store_value(std::unique_ptr<DocsumStoreDocument> doc); - vespalib::Slime write(const DocsumStoreValue& value); + std::unique_ptr<IDocsumStoreDocument> make_docsum_store_document(const vespalib::string &id); + vespalib::Slime write(const IDocsumStoreDocument* doc); vespalib::Memory get_field_name_view() const noexcept { return _field_name_view; } }; @@ -77,9 +68,7 @@ DocumentIdDFWTest::DocumentIdDFWTest() : testing::Test(), _field_name("documentid"), _field_name_view(_field_name.data(), _field_name.size()), - _docsum_blob_entry_filter(DocsumBlobEntryFilter().add_skip(search::docsummary::RES_LONG_STRING)), - _result_config(std::make_unique<ResultConfig>(_docsum_blob_entry_filter)), - _packer(std::make_unique<ResultPacker>(_result_config.get())), + _result_config(std::make_unique<ResultConfig>()), _repo(make_doc_type_repo()), _document_type(_repo->getDocumentType(doc_type_name)) { @@ -92,7 +81,7 @@ DocumentIdDFWTest::DocumentIdDFWTest() DocumentIdDFWTest::~DocumentIdDFWTest() = default; -std::unique_ptr<DocsumStoreDocument> +std::unique_ptr<IDocsumStoreDocument> DocumentIdDFWTest::make_docsum_store_document(const vespalib::string& id) { auto doc = std::make_unique<Document>(*_document_type, DocumentId(id)); @@ -100,27 +89,15 @@ DocumentIdDFWTest::make_docsum_store_document(const vespalib::string& id) return std::make_unique<DocsumStoreDocument>(std::move(doc)); } -std::unique_ptr<DocsumStoreValue> -DocumentIdDFWTest::make_docsum_store_value(std::unique_ptr<DocsumStoreDocument> doc) -{ - EXPECT_TRUE(_packer->Init(0)); - const char *ptr = nullptr; - uint32_t len = 0; - EXPECT_TRUE(_packer->GetDocsumBlob(&ptr, &len)); - return std::make_unique<DocsumStoreValue>(ptr, len, std::move(doc)); -} - vespalib::Slime -DocumentIdDFWTest::write(const DocsumStoreValue& value) +DocumentIdDFWTest::write(const IDocsumStoreDocument* doc) { - auto result = std::make_unique<GeneralResult>(_result_config->LookupResultClass(0)); - EXPECT_TRUE(result->inplaceUnpack(value)); Slime slime; SlimeInserter top_inserter(slime); Cursor & docsum = top_inserter.insertObject(); ObjectInserter field_inserter(docsum, _field_name_view); DocumentIdDFW writer; - writer.insertField(0, result.get(), nullptr, search::docsummary::RES_LONG_STRING, field_inserter); + writer.insertField(0, doc, nullptr, search::docsummary::RES_LONG_STRING, field_inserter); return slime; } @@ -128,23 +105,21 @@ TEST_F(DocumentIdDFWTest, insert_document_id) { vespalib::string id("id::test::0"); auto doc = make_docsum_store_document(id); - auto dsvalue = make_docsum_store_value(std::move(doc)); - auto slime = write(*dsvalue); + auto slime = write(doc.get()); EXPECT_TRUE(slime.get()[get_field_name_view()].valid()); EXPECT_EQ(id, slime.get()[get_field_name_view()].asString().make_string()); } TEST_F(DocumentIdDFWTest, insert_document_id_no_document_doc) { - auto dsvalue = make_docsum_store_value(std::make_unique<DocsumStoreDocument>(std::unique_ptr<Document>())); - auto slime = write(*dsvalue); + auto doc = std::make_unique<DocsumStoreDocument>(std::unique_ptr<Document>()); + auto slime = write(doc.get()); EXPECT_FALSE(slime.get()[get_field_name_view()].valid()); } TEST_F(DocumentIdDFWTest, insert_document_id_no_docsum_store_doc) { - auto dsvalue = make_docsum_store_value({}); - auto slime = write(*dsvalue); + auto slime = write(nullptr); EXPECT_FALSE(slime.get()[get_field_name_view()].valid()); } diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 06ff6b1fbe7..1ecc4853ad4 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -16,13 +16,11 @@ #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> -#include <vespa/searchsummary/docsummary/docsumstorevalue.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> -#include <vespa/searchsummary/docsummary/general_result.h> #include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> +#include <vespa/searchsummary/docsummary/resultclass.h> #include <vespa/searchsummary/docsummary/resultconfig.h> -#include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> #include <vespa/searchsummary/test/slime_value.h> #include <vespa/vespalib/data/slime/slime.h> @@ -41,6 +39,8 @@ using search::attribute::CollectionType; using search::attribute::Config; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; +using search::docsummary::IDocsumStoreDocument; +using search::docsummary::DocsumStoreDocument; using search::docsummary::test::SlimeValue; using vespalib::Slime; @@ -64,9 +64,7 @@ constexpr uint32_t doc_id = 2; class DocsumStore { private: - DocsumBlobEntryFilter _docsum_blob_entry_filter; ResultConfig _config; - ResultPacker _packer; DocumentType _doc_type; StructDataType::UP _elem_type; ArrayDataType _array_type; @@ -81,9 +79,7 @@ private: public: DocsumStore() - : _docsum_blob_entry_filter(DocsumBlobEntryFilter().add_skip(ResType::RES_JSONSTRING)), - _config(_docsum_blob_entry_filter), - _packer(&_config), + : _config(), _doc_type("test"), _elem_type(make_struct_elem_type()), _array_type(*_elem_type), @@ -102,8 +98,7 @@ public: ~DocsumStore(); const ResultConfig& get_config() const { return _config; } const ResultClass* get_class() const { return _config.LookupResultClass(class_id); } - search::docsummary::DocsumStoreValue getMappedDocsum() { - assert(_packer.Init(class_id)); + std::unique_ptr<IDocsumStoreDocument> getMappedDocsum() { auto doc = std::make_unique<Document>(_doc_type, DocumentId("id:test:test::0")); { ArrayFieldValue array_value(_array_type); @@ -124,10 +119,7 @@ public: map2_value.put(StringFieldValue("dummy"), *make_elem_value("dummy", 2)); doc->setValue("map2", map2_value); } - const char* buf; - uint32_t buf_len; - assert(_packer.GetDocsumBlob(&buf, &buf_len)); - return DocsumStoreValue(buf, buf_len, std::make_unique<DocsumStoreDocument>(std::move(doc))); + return std::make_unique<DocsumStoreDocument>(std::move(doc)); } }; @@ -189,15 +181,13 @@ private: Slime run_filter_field_writer(const std::string& input_field_name, const ElementVector& matching_elements) { auto writer = make_field_writer(input_field_name); - GeneralResult result(_doc_store.get_class()); - auto docsum = _doc_store.getMappedDocsum(); - result.inplaceUnpack(docsum); + auto doc = _doc_store.getMappedDocsum(); StateCallback callback(input_field_name, matching_elements); GetDocsumsState state(callback); Slime slime; SlimeInserter inserter(slime); - writer->insertField(doc_id, &result, &state, ResType::RES_JSONSTRING, inserter); + writer->insertField(doc_id, doc.get(), &state, ResType::RES_JSONSTRING, inserter); return slime; } diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 13ae741c8a0..2565509f50f 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -13,7 +13,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchsummary/docsummary/docsumwriter.h> -#include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/vespalib/data/slime/slime.h> @@ -58,9 +57,7 @@ struct FieldBlock { }; struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { - DocsumBlobEntryFilter docsum_blob_entry_filter; std::unique_ptr<DynamicDocsumWriter> writer; - std::unique_ptr<ResultPacker> packer; StructDataType int_pair_type; DocumentType doc_type; GetDocsumsState state; @@ -78,9 +75,8 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { ::decode(Memory(buf.GetDrainPos(), buf.GetUsedLen()), slime), 0u); } uint32_t getNumDocs() const override { return 2; } - DocsumStoreValue getMappedDocsum(uint32_t docid) override { + std::unique_ptr<const IDocsumStoreDocument> getMappedDocsum(uint32_t docid) override { EXPECT_EQUAL(1u, docid); - EXPECT_TRUE(packer->Init(0)); auto doc = std::make_unique<Document>(doc_type, DocumentId("id:test:test::0")); doc->setValue("int_field", IntFieldValue(4)); doc->setValue("short_field", ShortFieldValue(2)); @@ -98,10 +94,7 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { int_pair.setValue("bar", IntFieldValue(2)); doc->setValue("int_pair_field", int_pair); } - const char *buf; - uint32_t len; - EXPECT_TRUE(packer->GetDocsumBlob(&buf, &len)); - return DocsumStoreValue(buf, len, std::make_unique<DocsumStoreDocument>(std::move(doc))); + return std::make_unique<DocsumStoreDocument>(std::move(doc)); } uint32_t getSummaryClassId() const override { return 0; } void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } @@ -111,13 +104,12 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { DocsumFixture::DocsumFixture() - : docsum_blob_entry_filter(DocsumBlobEntryFilter().add_skip(RES_INT).add_skip(RES_SHORT).add_skip(RES_BYTE).add_skip(RES_FLOAT).add_skip(RES_DOUBLE).add_skip(RES_INT64).add_skip(RES_STRING).add_skip(RES_DATA).add_skip(RES_LONG_STRING).add_skip(RES_LONG_DATA).add_skip(RES_JSONSTRING)), - writer(), packer(), + : writer(), int_pair_type("int_pair"), doc_type("test"), state(*this) { - ResultConfig *config = new ResultConfig(docsum_blob_entry_filter); + ResultConfig *config = new ResultConfig; ResultClass *cfg = config->AddResultClass("default", 0); EXPECT_TRUE(cfg != 0); EXPECT_TRUE(cfg->AddConfigEntry("int_field", RES_INT)); @@ -133,7 +125,6 @@ DocsumFixture::DocsumFixture() EXPECT_TRUE(cfg->AddConfigEntry("int_pair_field", RES_JSONSTRING)); config->CreateEnumMaps(); writer.reset(new DynamicDocsumWriter(config, 0)); - packer.reset(new ResultPacker(writer->GetResultConfig())); int_pair_type.addField(Field("foo", *DataType::INT)); int_pair_type.addField(Field("bar", *DataType::INT)); doc_type.addField(Field("int_field", *DataType::INT)); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index f27bac80d5b..aec86d49d7d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -11,12 +11,10 @@ vespa_add_library(searchsummary_docsummary OBJECT docsum_field_writer.cpp docsum_store_document.cpp docsumstate.cpp - docsumstorevalue.cpp docsumwriter.cpp document_id_dfw.cpp dynamicteaserdfw.cpp empty_dfw.cpp - general_result.cpp geoposdfw.cpp getdocsumargs.cpp juniper_input.cpp @@ -29,7 +27,6 @@ vespa_add_library(searchsummary_docsummary OBJECT res_type_utils.cpp resultclass.cpp resultconfig.cpp - resultpacker.cpp searchdatatype.cpp simple_dfw.cpp struct_fields_resolver.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp index 836273ce3d8..af039820c4e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "copy_dfw.h" -#include "general_result.h" #include "i_docsum_store_document.h" +#include "resultclass.h" #include "resultconfig.h" #include <vespa/vespalib/data/slime/slime.h> @@ -29,110 +29,15 @@ CopyDFW::Init(const ResultConfig & config, const char *inputField) LOG(warning, "no docsum format contains field '%s'; copied fields will be empty", inputField); } - for (const auto & result_class : config) { - const ResConfigEntry *entry = result_class.GetEntry(result_class.GetIndexFromEnumValue(_inputFieldEnumValue)); - - if (entry != nullptr && !entry->_not_present && - !IsRuntimeCompatible(entry->_type, RES_INT) && - !IsRuntimeCompatible(entry->_type, RES_DOUBLE) && - !IsRuntimeCompatible(entry->_type, RES_INT64) && - !IsRuntimeCompatible(entry->_type, RES_STRING) && - !IsRuntimeCompatible(entry->_type, RES_DATA)) { - - LOG(warning, "cannot use docsum field '%s' as input to copy; type conflict with result class %d (%s)", - inputField, result_class.GetClassID(), result_class.GetClassName()); - return false; - } - } return true; } void -CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *, ResType type, +CopyDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) { - int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); - ResEntry *entry = gres->GetPresentEntry(idx); - - if (entry == nullptr) { - const auto* document = gres->get_document(); - if (document != nullptr) { - document->insert_summary_field(_input_field_name, target); - } - } else if (IsRuntimeCompatible(entry->_type, type)) { - switch (type) { - case RES_INT: { - uint32_t val32 = entry->_intval; - target.insertLong(val32); - break; } - - case RES_SHORT: { - uint16_t val16 = entry->_intval; - target.insertLong(val16); - break; } - - case RES_BYTE: { - uint8_t val8 = entry->_intval; - target.insertLong(val8); - break; } - case RES_BOOL: { - target.insertBool(entry->_intval != 0); - break; } - - case RES_FLOAT: { - float valfloat = entry->_doubleval; - target.insertDouble(valfloat); - break; } - - case RES_DOUBLE: { - double valdouble = entry->_doubleval; - target.insertDouble(valdouble); - break; } - - case RES_INT64: { - uint64_t valint64 = entry->_int64val; - target.insertLong(valint64); - break; } - - case RES_JSONSTRING: { - uint32_t len; - const char *spt; - // resolve field - entry->_resolve_field(&spt, &len); - if (len != 0) { - // note: 'JSONSTRING' really means 'structured data' - vespalib::Slime input_field_as_slime; - size_t d = vespalib::slime::BinaryFormat::decode(vespalib::Memory(spt, len), input_field_as_slime); - if (d != len) { - LOG(warning, "could not decode %u bytes: %zu bytes decoded", len, d); - } - if (d != 0) { - inject(input_field_as_slime.get(), target); - } - } - break; } - case RES_FEATUREDATA: - case RES_LONG_STRING: - case RES_STRING: { - uint32_t len; - const char *spt; - // resolve field - entry->_resolve_field(&spt, &len); - vespalib::Memory value(spt, len); - target.insertString(value); - break; } - - case RES_TENSOR: - case RES_LONG_DATA: - case RES_DATA: { - uint32_t len; - const char *dpt; - // resolve field - entry->_resolve_field(&dpt, &len); - vespalib::Memory value(dpt, len); - target.insertData(value); - break; } - } + if (doc != nullptr) { + doc->insert_summary_field(_input_field_name, target); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h index dab7417f60a..f679863ccc4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h @@ -24,7 +24,7 @@ public: bool Init(const ResultConfig & config, const char *inputField); bool IsGenerated() const override { return false; } - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h deleted file mode 100644 index 1d006386d35..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_blob_entry_filter.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "res_type.h" -#include <bitset> - -namespace search::docsummary { - -/* - * Class containing the set of result types not stored in docsum blobs. - * This is used for gradual migration towards elimination of docsum blobs. - */ -class DocsumBlobEntryFilter { - std::bitset<14> _skip_types; - -public: - DocsumBlobEntryFilter() - : _skip_types() - { - } - bool skip(ResType type) const noexcept { return _skip_types.test(type); } - DocsumBlobEntryFilter &add_skip(ResType type) { - _skip_types.set(type); - return *this; - } -}; - -} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h index 764f3507380..455b38f87fd 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h @@ -9,7 +9,7 @@ namespace vespalib::slime { struct Inserter; } namespace search::docsummary { -class GeneralResult; +class IDocsumStoreDocument; class GetDocsumsState; /* @@ -27,7 +27,7 @@ public: return ResTypeUtils::IsRuntimeCompatible(a, b); } virtual bool IsGenerated() const = 0; - virtual void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; + virtual void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; virtual const vespalib::string & getAttributeName() const; virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const; void setIndex(size_t v) { _index = v; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h index d6a7456cd95..2c2908ff95d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h @@ -2,10 +2,12 @@ #pragma once -#include "docsumstorevalue.h" +#include <memory> namespace search::docsummary { +class IDocsumStoreDocument; + /** * Interface for object able to fetch docsum blobs based on local * document id. @@ -33,10 +35,10 @@ public: * owns the memory (which is either mmap()ed or from a memory-based * index of some kind). * - * @return docsum blob location and size + * @return unique pointer to interface class providing access to document * @param docid local document id **/ - virtual DocsumStoreValue getMappedDocsum(uint32_t docid) = 0; + virtual std::unique_ptr<const IDocsumStoreDocument> getMappedDocsum(uint32_t docid) = 0; /** * Will return default input class used. diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.cpp deleted file mode 100644 index a60ae93b58f..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.cpp +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "docsumstorevalue.h" -#include "i_docsum_store_document.h" - -namespace search::docsummary { - -DocsumStoreValue::DocsumStoreValue() - : _value(static_cast<const char*>(0), 0), - _document() -{ -} - -DocsumStoreValue::DocsumStoreValue(const char *pt_, uint32_t len_) - : _value(pt_, len_), - _document() -{ -} - -DocsumStoreValue::DocsumStoreValue(const char *pt_, uint32_t len_, std::unique_ptr<IDocsumStoreDocument> document) - : _value(pt_, len_), - _document(std::move(document)) -{ -} - -DocsumStoreValue::~DocsumStoreValue() = default; - -} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.h deleted file mode 100644 index bdc4e7b9e19..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <cstdint> -#include <memory> -#include <utility> - -namespace search::docsummary { - -class IDocsumStoreDocument; - -/** - * Simple wrapper class containing the location and size of a docsum - * blob located in memory. The memory containing the docsum blob is - * owned by the object that emitted the docsum store value object. - * Always start with an uint32_t representing the result class ID. - **/ -class DocsumStoreValue -{ -private: - std::pair<const char *, uint32_t> _value; - // An interface for accessing the document instance that was used - // to generate the docsum blob. - // Note: This is temporary until the docsummary framework is simplified, - // and the docsum blob concept is removed. - std::unique_ptr<IDocsumStoreDocument> _document; - -public: - DocsumStoreValue(const DocsumStoreValue&) = delete; - DocsumStoreValue& operator=(const DocsumStoreValue&) = delete; - - /** - * Construct object representing an empty docsum blob. - **/ - DocsumStoreValue(); - - /** - * Construct object encapsulating the given location and size. - * - * @param pt_ docsum location - * @param len_ docsum size - **/ - DocsumStoreValue(const char *pt_, uint32_t len_); - - /** - * Construct object encapsulating the given location and size. - * - * @param pt_ docsum location - * @param len_ docsum size - * @param document docsum store document instance - **/ - DocsumStoreValue(const char *pt_, uint32_t len_, std::unique_ptr<IDocsumStoreDocument> document); - - ~DocsumStoreValue(); - - /** - * @return docsum blob location - **/ - const char *pt() const { return _value.first; } - - /** - * @return docsum blob size - **/ - uint32_t len() const { return _value.second; } - - /** - * @return pointer to start of serialized docsum fields - **/ - const char *fieldsPt() const { return _value.first + sizeof(uint32_t); } - - /** - * @return size of serialized docsum fields - **/ - uint32_t fieldsSz() const { return _value.second - sizeof(uint32_t); } - - /** - * @return true if this has a valid blob - **/ - bool valid() const { return (_value.first != 0) && (_value.second >= sizeof(uint32_t)); } - - const IDocsumStoreDocument* get_document() const { return _document.get(); } -}; - -} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 1492ce2b435..29af67a6fe0 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -84,84 +84,9 @@ DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, uint32_t id) const } } -constexpr uint32_t default_32bits_int = search::attribute::getUndefined<int32_t>(); -constexpr uint64_t default_64bits_int = search::attribute::getUndefined<int64_t>(); - -static void convertEntry(const ResConfigEntry *resCfg, - const ResEntry *entry, - const GeneralResult& gres, - Inserter &inserter, - Slime &slime) -{ - using vespalib::slime::BinaryFormat; - const char *ptr; - uint32_t len; - - LOG_ASSERT(resCfg != nullptr); - if (entry == nullptr || entry->_not_present) { - // Entry is not present in docsum blob - const auto* document = gres.get_document(); - if (document != nullptr) { - document->insert_summary_field(resCfg->_bindname, inserter); - } - return; - } - - switch (resCfg->_type) { - case RES_INT: - case RES_SHORT: - case RES_BYTE: - if (entry->_intval != default_32bits_int) { - inserter.insertLong(entry->_intval); - } - break; - case RES_BOOL: - inserter.insertBool(entry->_intval != 0); - break; - case RES_FLOAT: - case RES_DOUBLE: - if (! std::isnan(entry->_doubleval)) { - inserter.insertDouble(entry->_doubleval); - } - break; - case RES_INT64: - if (entry->_int64val != default_64bits_int) { - inserter.insertLong(entry->_int64val); - } - break; - case RES_STRING: - case RES_LONG_STRING: - case RES_FEATUREDATA: - entry->_resolve_field(&ptr, &len); - if (len != 0) { - inserter.insertString(Memory(ptr, len)); - } - break; - case RES_DATA: - case RES_TENSOR: - case RES_LONG_DATA: - entry->_resolve_field(&ptr, &len); - if (len != 0) { - inserter.insertData(Memory(ptr, len)); - } - break; - case RES_JSONSTRING: - entry->_resolve_field(&ptr, &len); - if (len != 0) { - // note: 'JSONSTRING' really means 'structured data' - size_t d = BinaryFormat::decode_into(Memory(ptr, len), slime, inserter); - if (d != len) { - LOG(warning, "could not decode %u bytes: %zu bytes decoded", len, d); - } - } - break; - } -} - - void DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::Slime & slime, vespalib::slime::Inserter & topInserter) + IDocsumStore *docinfos, vespalib::slime::Inserter& topInserter) { if (rci.allGenerated) { // generate docsum entry on-the-fly @@ -177,14 +102,8 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, } } else { // look up docsum entry - DocsumStoreValue value = docinfos->getMappedDocsum(docid); - // re-pack docsum blob - GeneralResult gres(rci.inputClass); - if (! gres.inplaceUnpack(value)) { - LOG(debug, "Unpack failed: illegal docsum entry for document %d. This is expected during lidspace compaction.", docid); - topInserter.insertNix(); - return; - } + auto doc = docinfos->getMappedDocsum(docid); + // insert docsum blob vespalib::slime::Cursor & docsum = topInserter.insertObject(); for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *outCfg = rci.outputClass->GetEntry(i); @@ -193,25 +112,11 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, ObjectInserter inserter(docsum, field_name); if (writer != nullptr) { if (! writer->isDefaultValue(docid, state)) { - writer->insertField(docid, &gres, state, outCfg->_type, inserter); + writer->insertField(docid, doc.get(), state, outCfg->_type, inserter); } } else { - if (rci.inputClass == rci.outputClass) { - convertEntry(outCfg, gres.GetEntry(i), gres, inserter, slime); - } else { - int inIdx = rci.inputClass->GetIndexFromEnumValue(outCfg->_enumValue); - const ResConfigEntry *inCfg = rci.inputClass->GetEntry(inIdx); - if (inCfg != nullptr && inCfg->_type == outCfg->_type && !inCfg->_not_present) { - // copy field - const ResEntry *entry = gres.GetEntry(inIdx); - LOG_ASSERT(entry != nullptr); - convertEntry(outCfg, entry, gres, inserter, slime); - } else { - const auto* document = gres.get_document(); - if (document != nullptr) { - document->insert_summary_field(outCfg->_bindname, inserter); - } - } + if (doc) { + doc->insert_summary_field(outCfg->_bindname, inserter); } } } @@ -341,7 +246,7 @@ DynamicDocsumWriter::WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsum vespalib::Slime slime; vespalib::slime::SlimeInserter inserter(slime); ResolveClassInfo rci = resolveClassInfo(state->_args.getResultClassName(), docinfos->getSummaryClassId()); - insertDocsum(rci, docid, state, docinfos, slime, inserter); + insertDocsum(rci, docid, state, docinfos, inserter); return slime2RawBuf(slime, *target); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index b3182221b68..ba1ffb25d22 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -3,7 +3,7 @@ #pragma once #include "juniperproperties.h" -#include "general_result.h" +#include "resultclass.h" #include "resultconfig.h" #include "docsumstore.h" #include "keywordextractor.h" @@ -42,7 +42,7 @@ public: virtual uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, search::RawBuf *target) = 0; virtual void insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::Slime & slime, vespalib::slime::Inserter & target) = 0; + IDocsumStore *docinfos, vespalib::slime::Inserter & target) = 0; virtual ResolveClassInfo resolveClassInfo(vespalib::stringref outputClassName, uint32_t inputClassId) const = 0; static uint32_t slime2RawBuf(const vespalib::Slime & slime, RawBuf & buf); @@ -81,7 +81,7 @@ public: IDocsumStore *docinfos, search::RawBuf *target) override; void insertDocsum(const ResolveClassInfo & outputClassInfo, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::Slime & slime, vespalib::slime::Inserter & target) override; + IDocsumStore *docinfos, vespalib::slime::Inserter & target) override; ResolveClassInfo resolveClassInfo(vespalib::stringref outputClassName, uint32_t inputClassId) const override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp index 8fcf7c95cdc..725e156798b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_id_dfw.h" -#include "general_result.h" #include "i_docsum_store_document.h" namespace search::docsummary { @@ -19,12 +18,11 @@ DocumentIdDFW::IsGenerated() const } void -DocumentIdDFW::insertField(uint32_t, GeneralResult *gres, GetDocsumsState *, ResType, +DocumentIdDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) { - const auto* document = gres->get_document(); - if (document != nullptr) { - document->insert_document_id(target); + if (doc != nullptr) { + doc->insert_document_id(target); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h index f6353912384..b1323489026 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h @@ -16,7 +16,7 @@ public: DocumentIdDFW(); ~DocumentIdDFW() override; bool IsGenerated() const override; - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 667cd68e59e..f1a768775cb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -331,40 +331,7 @@ JuniperTeaserDFW::Init( const ResultConfig & config, const char *inputField) { - bool rc = JuniperDFW::Init(fieldName, langFieldName, config, inputField); - - for (ResultConfig::const_iterator it(config.begin()), mt(config.end()); rc && it != mt; it++) { - - const ResConfigEntry *entry = - it->GetEntry(it->GetIndexFromEnumValue(_inputFieldEnumValue)); - - if (entry != nullptr && !entry->_not_present && - !IsRuntimeCompatible(entry->_type, RES_STRING) && - !IsRuntimeCompatible(entry->_type, RES_DATA)) - { - LOG(warning, "cannot use docsum field '%s' as input to dynamicteaser; bad type in result class %d (%s)", - inputField, it->GetClassID(), it->GetClassName()); - rc = false; - } - } - return rc; -} - -JuniperInput -DynamicTeaserDFW::getJuniperInput(GeneralResult *gres) { - int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); - ResEntry *entry = gres->GetPresentEntry(idx); - if (entry != nullptr) { - const char *buf; - uint32_t buflen; - entry->_resolve_field(&buf, &buflen); - return JuniperInput(vespalib::stringref(buf, buflen)); - } - const auto* document = gres->get_document(); - if (document != nullptr) { - return document->get_juniper_input(_input_field_name); - } - return {}; + return JuniperDFW::Init(fieldName, langFieldName, config, inputField); } vespalib::string @@ -419,14 +386,16 @@ DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, vespalib::stringref input, G } void -DynamicTeaserDFW::insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType, +DynamicTeaserDFW::insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) { - auto input = getJuniperInput(gres); - if (!input.empty()) { - vespalib::string teaser = makeDynamicTeaser(docid, input.get_value(), state); - vespalib::Memory value(teaser.c_str(), teaser.size()); - target.insertString(value); + if (doc != nullptr) { + auto input = doc->get_juniper_input(_input_field_name); + if (!input.empty()) { + vespalib::string teaser = makeDynamicTeaser(docid, input.get_value(), state); + vespalib::Memory value(teaser.c_str(), teaser.size()); + target.insertString(value); + } } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp deleted file mode 100644 index 359959391fd..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "general_result.h" -#include "i_docsum_store_document.h" -#include "resultconfig.h" -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/datatype/datatype.h> -#include <cassert> - -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.docsummary.urlresult"); - -namespace search::docsummary { - -void -GeneralResult::AllocEntries() -{ - uint32_t cnt = _resClass->GetNumEntries(); - uint32_t needMem = cnt * sizeof(ResEntry); - - if (cnt > 0) { - _entrycnt = cnt; - _entries = (ResEntry *) malloc(needMem); - assert(_entries != nullptr); - memset(_entries, 0, cnt * sizeof(ResEntry)); - } else { - _entrycnt = 0; - _entries = nullptr; - } -} - -void -GeneralResult::FreeEntries() -{ - free(_entries); // free '_entries' chunk -} - -GeneralResult::GeneralResult(const ResultClass *resClass) - : _resClass(resClass), - _entrycnt(0), - _entries(nullptr), - _document() -{ -} - -GeneralResult::~GeneralResult() -{ - FreeEntries(); -} - -ResEntry * -GeneralResult::GetPresentEntry(const char *name) -{ - int idx = _resClass->GetIndexFromName(name); - return GetPresentEntry(idx); -} - -ResEntry * -GeneralResult::GetPresentEntryFromEnumValue(uint32_t value) -{ - int idx = _resClass->GetIndexFromEnumValue(value); - return GetPresentEntry(idx); -} - -DocsumStoreFieldValue -GeneralResult::get_field_value(const vespalib::string& field_name) const -{ - if (_document != nullptr) { - return _document->get_field_value(field_name); - } - return DocsumStoreFieldValue(); -} - -bool -GeneralResult::unpack(const char *buf, const size_t buflen) -{ - bool rc = true; - const char *ebuf = buf + buflen; // Ref to first after buffer - const char *p = buf; // current position in buffer - - if (_entries != nullptr) - FreeEntries(); - - AllocEntries(); - - for (uint32_t i = 0; rc && i < _entrycnt; i++) { - const ResConfigEntry *entry = _resClass->GetEntry(i); - - _entries[i]._not_present = entry->_not_present; - if (entry->_not_present) { - // Entry is not present in docsum blob - _entries[i]._type = entry->_type; - continue; - } - switch (entry->_type) { - - case RES_INT: { - if (p + sizeof(_entries[i]._intval) <= ebuf) { - memcpy(&_entries[i]._intval, p, sizeof(_entries[i]._intval)); - _entries[i]._type = RES_INT; - p += sizeof(_entries[i]._intval); - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(..._intval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_SHORT: { - uint16_t shortval; - if (p + sizeof(shortval) <= ebuf) { - memcpy(&shortval, p, sizeof(shortval)); - _entries[i]._intval = (uint32_t)shortval; - _entries[i]._type = RES_INT; // type promotion - p += sizeof(shortval); - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(shortval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - case RES_BOOL: - case RES_BYTE: { - uint8_t byteval; - if (p + sizeof(byteval) <= ebuf) { - memcpy(&byteval, p, sizeof(byteval)); - _entries[i]._intval = (uint32_t)byteval; - _entries[i]._type = RES_INT; // type promotion - p += sizeof(byteval); - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(byteval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_FLOAT: { - float floatval; - if (p + sizeof(floatval) <= ebuf) { - memcpy(&floatval, p, sizeof(floatval)); - _entries[i]._doubleval = (double)floatval; - _entries[i]._type = RES_DOUBLE; // type promotion - p += sizeof(floatval); - } else { - LOG(debug, "GeneralResult::unpack: p + sizeof(floatval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_DOUBLE: { - if (p + sizeof(_entries[i]._doubleval) <= ebuf) { - memcpy(&_entries[i]._doubleval, p, sizeof(_entries[i]._doubleval)); - _entries[i]._type = RES_DOUBLE; - p += sizeof(_entries[i]._doubleval); - } else { - LOG(debug, "GeneralResult::unpack: p + sizeof(..._doubleval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_INT64: { - if (p + sizeof(_entries[i]._int64val) <= ebuf) { - memcpy(&_entries[i]._int64val, p, sizeof(_entries[i]._int64val)); - _entries[i]._type = RES_INT64; - p += sizeof(_entries[i]._int64val); - } else { - LOG(debug, "GeneralResult::unpack: p + sizeof(..._int64val) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_STRING: { - uint16_t slen; - if (p + sizeof(slen) <= ebuf) { - memcpy(&slen, p, sizeof(slen)); - p += sizeof(slen); - if (p + slen <= ebuf) { - _entries[i]._stringval = const_cast<char *>(p); - _entries[i]._stringlen = slen; - _entries[i]._type = RES_STRING; - p += slen; - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + slen > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(slen) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_DATA: { - uint16_t dlen; - if (p + sizeof(dlen) <= ebuf) { - memcpy(&dlen, p, sizeof(dlen)); - p += sizeof(dlen); - if (p + dlen <= ebuf) { - _entries[i]._dataval = const_cast<char *>(p); - _entries[i]._datalen = dlen; - _entries[i]._type = RES_DATA; - p += dlen; - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + dlen > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(dlen) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_JSONSTRING: - case RES_FEATUREDATA: - case RES_LONG_STRING: { - uint32_t flen; - uint32_t lslen; - if (p + sizeof(flen) <= ebuf) { - memcpy(&flen, p, sizeof(flen)); - p += sizeof(flen); - lslen = flen & 0x7fffffff; - if (lslen != flen) { - LOG(error, "GeneralResult::_inplace_unpack: compressed data"); - rc = false; - } - if (p + lslen <= ebuf) { - _entries[i]._stringval = const_cast<char *>(p); - _entries[i]._stringlen = lslen; - _entries[i]._type = RES_STRING; // type normalization - p += lslen; - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + lslen > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(lslen) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - case RES_TENSOR : - case RES_LONG_DATA: { - uint32_t flen; - uint32_t ldlen; - if (p + sizeof(flen) <= ebuf) { - memcpy(&flen, p, sizeof(flen)); - p += sizeof(flen); - ldlen = flen & 0x7fffffff; - if (p + ldlen <= ebuf) { - _entries[i]._dataval = const_cast<char *>(p); - _entries[i]._datalen = flen; // with compression flag - _entries[i]._type = RES_DATA; // type normalization - p += ldlen; - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + ldlen > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(ldlen) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - default: - LOG(warning, "GeneralResult::_inplace_unpack: no such type:%d", entry->_type); - LOG(error, "Incorrect type in document summary, couldn't unpack"); - rc = false; - break; - } // END -- switch (entry->_type) { - } // END -- for (uint32_t i = 0; rc && i < _entrycnt; i++) { - - if (rc && p != ebuf) { - LOG(debug, "GeneralResult::_inplace_unpack: p:%p != ebuf:%p", p, ebuf); - LOG(error, "Document summary too long, couldn't unpack."); - rc = false; - } - - if (rc) - return true; // SUCCESS - - // clean up on failure - FreeEntries(); - _entrycnt = 0; - _entries = nullptr; - - return false; // FAIL -} - -} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.h b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h deleted file mode 100644 index 8f7a1377502..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "resultclass.h" -#include "docsumstorevalue.h" -#include "docsum_store_field_value.h" - -namespace search::docsummary { - -class DocsumStoreDocument; - -class GeneralResult -{ -private: - GeneralResult(const GeneralResult &); - GeneralResult& operator=(const GeneralResult &); - - const ResultClass *_resClass; - uint32_t _entrycnt; - ResEntry *_entries; - const IDocsumStoreDocument* _document; - - void AllocEntries(); - void FreeEntries(); - -public: - GeneralResult(const ResultClass *resClass); - ~GeneralResult(); - - const ResultClass *GetClass() const { return _resClass; } - ResEntry *GetEntry(uint32_t idx) { return (idx < _entrycnt) ? &_entries[idx] : nullptr; } - ResEntry *GetPresentEntry(uint32_t idx) { - if (idx >= _entrycnt) { - return nullptr; - } - ResEntry* entry = &_entries[idx]; - return entry->_not_present ? nullptr : entry; - } - ResEntry *GetPresentEntry(const char *name); - ResEntry *GetPresentEntryFromEnumValue(uint32_t val); - DocsumStoreFieldValue get_field_value(const vespalib::string& field_name) const; - bool unpack(const char *buf, const size_t buflen); - - bool inplaceUnpack(const DocsumStoreValue &value) { - if (value.valid()) { - _document = value.get_document(); - return unpack(value.fieldsPt(), value.fieldsSz()); - } else { - return false; - } - } - - const IDocsumStoreDocument *get_document() const noexcept { return _document; } -}; - -} - diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h index 382cc999956..b4cc79ef205 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h @@ -2,7 +2,6 @@ #pragma once -#include "general_result.h" #include "resultconfig.h" #include "docsum_field_writer.h" #include "juniper_input.h" @@ -48,14 +47,13 @@ protected: class DynamicTeaserDFW : public JuniperTeaserDFW { - JuniperInput getJuniperInput(GeneralResult *gres) __attribute__((noinline)); vespalib::string makeDynamicTeaser(uint32_t docid, vespalib::stringref input, GetDocsumsState *state); public: DynamicTeaserDFW(juniper::Juniper * juniper) : JuniperTeaserDFW(juniper) { } - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index c05fec7a0ce..56bbfe2d040 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -2,7 +2,7 @@ #include "matched_elements_filter_dfw.h" #include "docsumstate.h" -#include "general_result.h" +#include "i_docsum_store_document.h" #include "struct_fields_resolver.h" #include "summaryfieldconverter.h" #include <vespa/document/fieldvalue/document.h> @@ -66,15 +66,6 @@ MatchedElementsFilterDFW::~MatchedElementsFilterDFW() = default; namespace { void -decode_input_field_to_slime(const ResEntry& entry, Slime& input_field_as_slime) -{ - const char* buf; - uint32_t buf_len; - entry._resolve_field(&buf, &buf_len); - BinaryFormat::decode(vespalib::Memory(buf, buf_len), input_field_as_slime); -} - -void filter_matching_elements_in_input_field_while_converting_to_slime(const FieldValue& input_field_value, const std::vector<uint32_t>& matching_elems, vespalib::slime::Inserter& target) @@ -91,52 +82,18 @@ filter_matching_elements_in_input_field_while_converting_to_slime(const FieldVal inject(input_field_as_slime.get(), target); } -bool -resolve_input_field_as_slime(GeneralResult& result, int entry_idx, Slime& input_field_as_slime) -{ - ResEntry* entry = result.GetPresentEntry(entry_idx); - if (entry != nullptr) { - decode_input_field_to_slime(*entry, input_field_as_slime); - return true; - } - return false; -} - -void -filter_matching_elements_in_input_field(const Slime& input_field, const std::vector<uint32_t>& matching_elems, Slime& output_field) -{ - SlimeInserter output_inserter(output_field); - Inspector& input_inspector = input_field.get(); - ArrayInserter array_inserter(output_inserter.insertArray(matching_elems.size())); - auto elems_itr = matching_elems.begin(); - for (size_t i = 0; (i < input_inspector.entries()) && (elems_itr != matching_elems.end()); ++i) { - assert(*elems_itr >= i); - if (*elems_itr == i) { - inject(input_inspector[i], array_inserter); - ++elems_itr; - } - } -} - } void -MatchedElementsFilterDFW::insertField(uint32_t docid, GeneralResult* result, GetDocsumsState *state, +MatchedElementsFilterDFW::insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter& target) { assert(type == ResType::RES_JSONSTRING); - int entry_idx = result->GetClass()->GetIndexFromEnumValue(_input_field_enum); Slime input_field; - if (resolve_input_field_as_slime(*result, entry_idx, input_field)) { - Slime output_field; - filter_matching_elements_in_input_field(input_field, get_matching_elements(docid, *state), output_field); - inject(output_field.get(), target); - } else { - // Use the document instance if the input field is not in the docsum blob. - auto field_value = result->get_field_value(_input_field_name); - if (field_value) { - filter_matching_elements_in_input_field_while_converting_to_slime(*field_value, get_matching_elements(docid, *state), target); - } + // Use the document instance if the input field is not in the docsum blob. + auto field_value = doc->get_field_value(_input_field_name); + if (field_value) { + filter_matching_elements_in_input_field_while_converting_to_slime(*field_value, get_matching_elements(docid, *state), target); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h index b117da541d6..c47ca0b21b1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h @@ -35,7 +35,7 @@ public: std::shared_ptr<MatchingElementsFields> matching_elems_fields); ~MatchedElementsFilterDFW(); bool IsGenerated() const override { return false; } - void insertField(uint32_t docid, GeneralResult* result, GetDocsumsState *state, + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter& target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index 65db72efc40..37be8e0a1b2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -7,7 +7,7 @@ namespace search::docsummary { -ResultClass::ResultClass(const char *name, uint32_t id, util::StringEnum & fieldEnum, const DocsumBlobEntryFilter& docsum_blob_entry_filter) +ResultClass::ResultClass(const char *name, uint32_t id, util::StringEnum & fieldEnum) : _name(name), _classID(id), _entries(), @@ -15,8 +15,7 @@ ResultClass::ResultClass(const char *name, uint32_t id, util::StringEnum & field _fieldEnum(fieldEnum), _enumMap(), _dynInfo(NULL), - _omit_summary_features(false), - _docsum_blob_entry_filter(docsum_blob_entry_filter) + _omit_summary_features(false) { } @@ -38,7 +37,6 @@ ResultClass::AddConfigEntry(const char *name, ResType type) _nameMap[name] = _entries.size(); ResConfigEntry e; e._type = type; - e._not_present = _docsum_blob_entry_filter.skip(type); e._bindname = name; e._enumValue = _fieldEnum.Add(name); assert(e._enumValue >= 0); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index 47feed70e97..555d518b799 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -2,7 +2,7 @@ #pragma once -#include "docsum_blob_entry_filter.h" +#include "res_type.h" #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> @@ -11,61 +11,20 @@ namespace search::docsummary { /** - * This struct describes a single docsum field (name and type). A - * docsum blob is unpacked into an array of ResEntry instances - * by interpreting it as described by an array of ResConfigEntry - * instances. + * This struct describes a single docsum field (name and type). **/ struct ResConfigEntry { ResType _type; - bool _not_present; // Entry not present in docsum blob when _not_present is set vespalib::string _bindname; int _enumValue; }; /** - * This struct holds the actual value of a single docsum field. A - * docsum blob is unpacked into an array of ResEntry instances - * by interpreting it as described by an array of ResConfigEntry - * instances. Note that type normalization is performed when unpacking - * docsum fields. Fields of type RES_BYTE and RES_SHORT are promoted - * to RES_INT. Fields of type RES_FLOAT are promoted to RES_DOUBLE. - **/ -struct ResEntry -{ - ResType _type; - bool _not_present; // Entry not present in docsum blob when _not_present is set - union { - uint32_t _intval; - uint32_t _stringlen; - uint32_t _datalen; - uint32_t _len; - uint64_t _int64val; - double _doubleval; - }; - union { - char *_stringval; - char *_dataval; - void *_pt; - }; - - void _resolve_field(const char **buf, uint32_t *buflen) const - { - // precond: IsVariableSize(_type) - *buf = (char *) _pt; - *buflen = _len; - } -}; - -/** * This class represents a specific docsum format (docsum class). It * contains an array of ResConfigEntry instances (config - * entries) that may be used to unpack docsum blobs into - * ResEntry arrays. It also contains methods for mapping both - * field name and field name enum value into field index. The field - * index may then be used to access the actual field in the - * GeneralResult object representing the unpacked docsum blob. + * entries). It also contains methods for mapping both + * field name and field name enum value into field index. **/ class ResultClass { @@ -92,7 +51,6 @@ private: // Whether or not summary features should be omitted when filling this summary class. // As default, summary features are always included. bool _omit_summary_features; - DocsumBlobEntryFilter _docsum_blob_entry_filter; public: typedef std::unique_ptr<ResultClass> UP; @@ -105,7 +63,7 @@ public: * @param id the numeric id of this result class. * @param fieldEnum shared object used to enumerate field names. **/ - ResultClass(const char *name, uint32_t id, util::StringEnum & fieldEnum, const DocsumBlobEntryFilter& docsum_blob_entry_filter); + ResultClass(const char *name, uint32_t id, util::StringEnum & fieldEnum); /** * Destructor. Delete internal structures. diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index 168b4c81374..3679bddc57e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -26,15 +26,9 @@ ResultConfig::Init() ResultConfig::ResultConfig() - : ResultConfig(DocsumBlobEntryFilter()) -{ -} - -ResultConfig::ResultConfig(const DocsumBlobEntryFilter& docsum_blob_entry_filter) : _defaultSummaryId(-1), _classLookup(), - _nameLookup(), - _docsum_blob_entry_filter(docsum_blob_entry_filter) + _nameLookup() { Init(); } @@ -62,7 +56,7 @@ ResultConfig::AddResultClass(const char *name, uint32_t id) ResultClass *ret = nullptr; if (id != NoClassID() && (_classLookup.find(id) == _classLookup.end())) { - ResultClass::UP rc(new ResultClass(name, id, _fieldEnum, _docsum_blob_entry_filter)); + ResultClass::UP rc(new ResultClass(name, id, _fieldEnum)); ret = rc.get(); _classLookup[id] = std::move(rc); if (_nameLookup.find(name) != _nameLookup.end()) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h index 945eef8514f..b37b652935f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h @@ -2,7 +2,6 @@ #pragma once -#include "docsum_blob_entry_filter.h" #include "res_type_utils.h" #include <vespa/config-summary.h> #include <vespa/searchlib/util/stringenum.h> @@ -38,7 +37,6 @@ private: search::util::StringEnum _fieldEnum; IdMap _classLookup; NameMap _nameLookup; // name -> class id - DocsumBlobEntryFilter _docsum_blob_entry_filter; void Clean(); void Init(); @@ -81,7 +79,6 @@ public: * NOTE: This method simply calls the Init method. **/ ResultConfig(); - ResultConfig(const DocsumBlobEntryFilter& docsum_blob_entry_filter); /** * Destructor. Delete all internal structures. NOTE: This method diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp deleted file mode 100644 index 66b15c1c8a9..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "resultpacker.h" -#include "resultconfig.h" -#include <vespa/searchcommon/common/undefinedvalues.h> -#include <vespa/vespalib/util/size_literals.h> - -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.docsummary.resultpacker"); - -namespace search::docsummary { - -void -ResultPacker::WarnType(ResType type) const -{ - LOG(debug, - "ResultPacker: got '%s', expected '%s' " - "(fields are binary compatible)", - GetResTypeName(type), - GetResTypeName(_cfgEntry->_type)); -} - -void -ResultPacker::skip_entries_not_present() -{ - while (_cfgEntry != nullptr && _cfgEntry->_not_present) { - _cfgEntry = _resClass->GetEntry(++_entryIdx); - } -} - -bool ResultPacker::CheckEntry(ResType type) -{ - if (_error) - return false; - - bool rc = (_cfgEntry != nullptr && - IsBinaryCompatible(_cfgEntry->_type, type)); - - if (rc) { - if (_cfgEntry->_type != type) { - WarnType(type); - } - _cfgEntry = _resClass->GetEntry(++_entryIdx); - skip_entries_not_present(); - } else { - SetFormatError(type); - } - - return rc; -} - -void -ResultPacker::SetFormatError(ResType type) -{ - _error = true; - - if (_cfgEntry != nullptr) { - LOG(error, - "ResultPacker: format error: got '%s', expected '%s'", - GetResTypeName(type), - GetResTypeName(_cfgEntry->_type)); - } else { - LOG(error, - "ResultPacker: format error: " - "got '%s', no more fields expected", GetResTypeName(type)); - } -} - - -ResultPacker::ResultPacker(const ResultConfig *resConfig) - : _buf(32_Ki), - _cbuf(32_Ki), - _resConfig(resConfig), - _resClass(nullptr), - _entryIdx(0), - _cfgEntry(nullptr), - _error(true) -{ -} - - -ResultPacker::~ResultPacker() = default; - -void -ResultPacker::InitPlain() -{ - _buf.reset(); -} - -bool -ResultPacker::Init(uint32_t classID) -{ - _buf.reset(); - _resClass = (_resConfig != nullptr) ? - _resConfig->LookupResultClass(classID) : nullptr; - _entryIdx = 0; - if (_resClass != nullptr) { - uint32_t id = _resClass->GetClassID(); - _buf.append(&id, sizeof(id)); - _cfgEntry = _resClass->GetEntry(_entryIdx); - skip_entries_not_present(); - _error = false; - } else { - _cfgEntry = nullptr; - _error = true; - - LOG(error, "ResultPacker: resultclass %d does not exist", classID); - } - - return !_error; -} - - -bool -ResultPacker::AddEmpty() -{ - if (!_error && _cfgEntry != nullptr) { - switch (_cfgEntry->_type) { - case RES_INT: return AddInteger(search::attribute::getUndefined<int32_t>()); - case RES_SHORT: return AddShort(search::attribute::getUndefined<int16_t>()); - case RES_BOOL: return AddByte(0); - case RES_BYTE: return AddByte(search::attribute::getUndefined<int8_t>()); - case RES_FLOAT: return AddFloat(search::attribute::getUndefined<float>()); - case RES_DOUBLE: return AddDouble(search::attribute::getUndefined<double>()); - case RES_INT64: return AddInt64(search::attribute::getUndefined<int64_t>()); - case RES_STRING: return AddString(nullptr, 0); - case RES_DATA: return AddData(nullptr, 0); - case RES_JSONSTRING: - case RES_FEATUREDATA: - case RES_LONG_STRING: return AddLongString(nullptr, 0); - case RES_TENSOR: return AddSerializedTensor(nullptr, 0); - case RES_LONG_DATA: return AddLongData(nullptr, 0); - } - } - return AddInteger(0); // to provoke error condition -} - - -bool -ResultPacker::AddByte(uint8_t value) -{ - if (CheckEntry(RES_BYTE)) - AddByteForce(value); - return !_error; -} - -void -ResultPacker::AddByteForce(uint8_t value) -{ - _buf.append(&value, sizeof(value)); -} - -bool -ResultPacker::AddShort(uint16_t value) -{ - if (CheckEntry(RES_SHORT)) - AddShortForce(value); - return !_error; -} - -void -ResultPacker::AddShortForce(uint16_t value) -{ - _buf.append(&value, sizeof(value)); -} - - -bool -ResultPacker::AddInteger(uint32_t value) -{ - if (CheckEntry(RES_INT)) - AddIntegerForce(value); - return !_error; -} - -void -ResultPacker::AddIntegerForce(uint32_t value) -{ - _buf.append(&value, sizeof(value)); -} - - -bool -ResultPacker::AddFloat(float value) -{ - if (CheckEntry(RES_FLOAT)) - _buf.append(&value, sizeof(value)); - return !_error; -} - - -bool -ResultPacker::AddDouble(double value) -{ - if (CheckEntry(RES_DOUBLE)) - _buf.append(&value, sizeof(value)); - return !_error; -} - - -bool -ResultPacker::AddInt64(uint64_t value) -{ - if (CheckEntry(RES_INT64)) - _buf.append(&value, sizeof(value)); - return !_error; -} - - -bool -ResultPacker::AddString(const char *str, uint32_t slen) -{ - if (CheckEntry(RES_STRING)) - AddStringForce(str, slen); - return !_error; -} - -void -ResultPacker::AddStringForce(const char *str, uint32_t slen) -{ - uint16_t len = slen; - _buf.append(&len, sizeof(len)); - _buf.append(str, len); -} - - -bool -ResultPacker::AddData(const char *buf, uint32_t buflen) -{ - if (CheckEntry(RES_DATA)) { - uint16_t len = buflen; - _buf.append(&len, sizeof(len)); - _buf.append(buf, len); - } - return !_error; -} - - -bool -ResultPacker::AddLongString(const char *str, uint32_t slen) -{ - if (CheckEntry(RES_LONG_STRING)) { - _buf.append(&slen, sizeof(slen)); - _buf.append(str, slen); - } - return !_error; -} - - -bool -ResultPacker::AddLongData(const char *buf, uint32_t buflen) -{ - if (CheckEntry(RES_LONG_DATA)) { - _buf.append(&buflen, sizeof(buflen)); - _buf.append(buf, buflen); - } - return !_error; -} - - -bool -ResultPacker::AddSerializedTensor(const char *buf, uint32_t buflen) -{ - if (CheckEntry(RES_TENSOR)) { - _buf.append(&buflen, sizeof(buflen)); - _buf.append(buf, buflen); - } - return !_error; -} - - -bool -ResultPacker::GetDocsumBlob(const char **buf, uint32_t *buflen) -{ - if (!_error && - _entryIdx != _resClass->GetNumEntries()) - { - _error = true; - LOG(error, - "ResultPacker: format error: %d fields are missing", - _resClass->GetNumEntries() - _entryIdx); - } - if (_error) { - *buf = nullptr; - *buflen = 0; - return false; - } else { - *buf = _buf.GetDrainPos(); - *buflen = _buf.GetUsedLen(); - return true; - } -} - -} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h deleted file mode 100644 index 816433652b8..00000000000 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "res_type_utils.h" -#include "resultclass.h" -#include <vespa/searchlib/util/rawbuf.h> - -namespace search::docsummary { - -class ResultConfig; - -/** - * An Object of this class may be used to create docsum blobs. A - * single blob is created by first indicating what result class the - * blob should conform to. After that, each docsum field is added with - * an individual method call. The blob may then be extracted by a - * final method call. Note that objects of this class may be re-used - * to create multiple blobs each. - **/ -class ResultPacker -{ -private: - ResultPacker(const ResultPacker &); - ResultPacker& operator=(const ResultPacker &); - - search::RawBuf _buf; // packing buffer - search::RawBuf _cbuf; // compression buffer - const ResultConfig *_resConfig; // result config - const ResultClass *_resClass; // result class of current blob - uint32_t _entryIdx; // current field index of current blob - const ResConfigEntry *_cfgEntry; // current field of current blob - bool _error; // error flag for current blob - - static const char *GetResTypeName(ResType type) { return ResTypeUtils::GetResTypeName(type); } - static bool IsBinaryCompatible(ResType a, ResType b) { return ResTypeUtils::IsBinaryCompatible(a, b); } - - void WarnType(ResType type) const; - void SetFormatError(ResType type); - - void skip_entries_not_present(); - bool CheckEntry(ResType type); - -public: - /** - * Create a result packer based on the given result config. Note - * that the result config object is NOT handed over; it is the - * responsibility of the application to ensure that the lifetime of - * the result config object is longer than the lifetime of the - * created result packer object. - * - * @param resConfig result configuration. - **/ - explicit ResultPacker(const ResultConfig *resConfig); - ~ResultPacker(); - - - /** - * Start creating new docsum blob without result class. - * (Bypassing type-checks.) - **/ - void InitPlain(); - - /** - * Start creating a new docsum blob of the given result class. - * - * @return true(ok)/false(error). - * @param classID the id of the result class we want to create a - * docsum blob conforming to. - **/ - bool Init(uint32_t classID); - - /** - * Add empty field of appropriate type. - * - * @return true(ok)/false(error). - **/ - bool AddEmpty(); - - /** - * Add a 'byte' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. - * - * @return true(ok)/false(error). - * @param value byte value of field to add. - **/ - bool AddByte(uint8_t value); - - void AddByteForce(uint8_t value); - - /** - * Add a 'short' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. - * - * @return true(ok)/false(error). - * @param value short value of field to add. - **/ - bool AddShort(uint16_t value); - - void AddShortForce(uint16_t value); - - - /** - * Add a 'integer' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. - * - * @return true(ok)/false(error). - * @param value integer value of field to add. - **/ - bool AddInteger(uint32_t value); - - void AddIntegerForce(uint32_t value); - - - /** - * Add a 'float' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. - * - * @return true(ok)/false(error). - * @param value float value of field to add. - **/ - bool AddFloat(float value); - - - /** - * Add a 'double' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. - * - * @return true(ok)/false(error). - * @param value double value of field to add. - **/ - bool AddDouble(double value); - - - /** - * Add a 'int64' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. - * - * @return true(ok)/false(error). - * @param value int64 value of field to add. - **/ - bool AddInt64(uint64_t value); - - - /** - * Add a 'string' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. The maximum length - * of this field is 64kB. - * - * @return true(ok)/false(error). - * @param str pointer to string to add. - * @param slen length of string to add. - **/ - bool AddString(const char *str, uint32_t slen); - - void AddStringForce(const char *str, uint32_t slen); - - /** - * Add a 'data' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. The maximum length - * of this field is 64kB. - * - * @return true(ok)/false(error). - * @param buf pointer to data to add. - * @param buflen length of data to add. - **/ - bool AddData(const char *buf, uint32_t buflen); - - - /** - * Add a 'longstring' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. The maximum length - * of this field is 2GB. - * - * @return true(ok)/false(error). - * @param str pointer to string to add. - * @param slen length of string to add. - **/ - bool AddLongString(const char *str, uint32_t slen); - - - /** - * Add a 'longdata' field to the docsum blob we are currently - * creating. Note that this method will fail if the type of the - * added field is not compatible with the field type sequence - * defined in the result class config. This method will also fail if - * an error condition is already detected. The only way to clear the - * error state is with another call to @ref Init. The maximum length - * of this field is 2GB. - * - * @return true(ok)/false(error). - * @param buf pointer to data to add. - * @param buflen length of data to add. - **/ - bool AddLongData(const char *buf, uint32_t buflen); - - /* - * Add a 'tensor' field to the docsum blob we are currently creating. - * - * @return true(ok)/false(error). - * @param buf pointer to serialized tensor to add. - * @param buflen length of serialized tensor to add. - **/ - bool AddSerializedTensor(const char *buf, uint32_t buflen); - - /** - * Obtain a pointer to, and the length of, the created docsum - * blob. This method will fail if an error was previously detected, - * or if any docsum fields were missing (too few fields were - * added). Note that calling the @ref Init method invalidates the - * obtained docsum blob. - * - * @return true(ok)/false(error). - * @param buf where to store the pointer to the docsum blob. - * @param buflen where to store the length of the docsum blob. - **/ - bool GetDocsumBlob(const char **buf, uint32_t *buflen); - - /* - * Get index of next entry to add. Used by proton::DocumentStoreAdapter - * and vsm::DocsumFilter to track entries to skip. - */ - uint32_t get_entry_idx() const noexcept { return _entryIdx; } -}; - -} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp index 8cc872378bd..238bb512852 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp @@ -5,7 +5,7 @@ namespace search::docsummary { void -SimpleDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) +SimpleDFW::insertField(uint32_t docid, const IDocsumStoreDocument *, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) { insertField(docid, state, type, target); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h index abebb7de2b8..20ec4ef2e82 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h @@ -14,7 +14,7 @@ class SimpleDFW : public DocsumFieldWriter { public: virtual void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; - void insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; + void insertField(uint32_t docid, const IDocsumStoreDocument*, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index bc05ec72638..f6090232b02 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -14,110 +14,6 @@ LOG_SETUP(".vsm.docsumfilter"); using namespace search::docsummary; -namespace { - -class Handler : public document::fieldvalue::IteratorHandler { -public: -}; - -struct IntResultHandler : public Handler { - int32_t value; - IntResultHandler() : value(0) {} - void onPrimitive(uint32_t, const Content & c) override { - value = c.getValue().getAsInt(); - } -}; - -struct LongResultHandler : public Handler { - int64_t value; - LongResultHandler() : value(0) {} - void onPrimitive(uint32_t, const Content & c) override { - value = c.getValue().getAsLong(); - } -}; - -struct FloatResultHandler : public Handler { - float value; - FloatResultHandler() : value(0) {} - void onPrimitive(uint32_t, const Content & c) override { - value = c.getValue().getAsFloat(); - } -}; - -struct DoubleResultHandler : public Handler { - double value; - DoubleResultHandler() : value(0) {} - void onPrimitive(uint32_t, const Content & c) override { - value = c.getValue().getAsDouble(); - } -}; - -class StringResultHandler : public Handler { -private: - ResType _type; - ResultPacker & _packer; - void addToPacker(const char * buf, size_t len) { - switch (_type) { - case RES_STRING: - _packer.AddString(buf, len); - break; - case RES_LONG_STRING: - _packer.AddLongString(buf, len); - break; - default: - break; - } - } - -public: - StringResultHandler(ResType t, ResultPacker & p) : _type(t), _packer(p) {} - void onPrimitive(uint32_t, const Content & c) override { - const document::FieldValue & fv = c.getValue(); - if (fv.isLiteral()) { - const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(fv); - vespalib::stringref s = lfv.getValueRef(); - addToPacker(s.data(), s.size()); - } else { - vespalib::string s = fv.toString(); - addToPacker(s.c_str(), s.size()); - } - } -}; - -class RawResultHandler : public Handler { -private: - ResType _type; - ResultPacker & _packer; - -public: - RawResultHandler(ResType t, ResultPacker & p) : _type(t), _packer(p) {} - void onPrimitive(uint32_t, const Content & c) override { - const document::FieldValue & fv = c.getValue(); - try { - std::pair<const char *, size_t> buf = fv.getAsRaw(); - if (buf.first != nullptr) { - switch (_type) { - case RES_DATA: - _packer.AddData(buf.first, buf.second); - break; - case RES_LONG_DATA: - _packer.AddLongData(buf.first, buf.second); - break; - default: - break; - } - } - } catch (document::InvalidDataTypeConversionException & e) { - LOG(warning, "RawResultHandler: Could not get field value '%s' as raw. Skipping writing this field", fv.toString().c_str()); - _packer.AddEmpty(); - } - } -}; - - -} - - namespace vsm { namespace { @@ -306,7 +202,6 @@ DocsumFilter::DocsumFilter(const DocsumToolsPtr &tools, const IDocSumCache & doc _tools(tools), _fields(), _highestFieldNo(0), - _packer(tools ? tools->getResultConfig() : nullptr), _flattenWriter(), _snippetModifiers(nullptr), _cachedValue(), @@ -341,110 +236,6 @@ DocsumFilter::getNumDocs() const return std::numeric_limits<uint32_t>::max(); } -void -DocsumFilter::writeField(const document::FieldValue & fv, const FieldPath & path, ResType type, ResultPacker & packer) -{ - switch (type) { - case RES_INT: { - IntResultHandler rh; - fv.iterateNested(path, rh); - uint32_t val = rh.value; - packer.AddInteger(val); - break; } - case RES_SHORT: { - IntResultHandler rh; - fv.iterateNested(path, rh); - uint16_t val = rh.value; - packer.AddShort(val); - break; } - case RES_BYTE: { - IntResultHandler rh; - fv.iterateNested(path, rh); - uint8_t val = rh.value; - packer.AddByte(val); - break; } - case RES_BOOL: { - IntResultHandler rh; - fv.iterateNested(path, rh); - uint8_t val = rh.value; - packer.AddByte(val); - break; } - case RES_FLOAT: { - FloatResultHandler rh; - fv.iterateNested(path, rh); - float val = rh.value; - packer.AddFloat(val); - break; } - case RES_DOUBLE: { - DoubleResultHandler rh; - fv.iterateNested(path, rh); - double val = rh.value; - packer.AddDouble(val); - break; } - case RES_INT64: { - LongResultHandler rh; - fv.iterateNested(path, rh); - uint64_t val = rh.value; - packer.AddInt64(val); - break; } - case RES_STRING: - case RES_LONG_STRING: - { - StringResultHandler rh(type, packer); - // the string result handler adds the result to the packer - fv.iterateNested(path, rh); - } - break; - case RES_DATA: - case RES_LONG_DATA: - { - RawResultHandler rh(type, packer); - // the raw result handler adds the result to the packer - fv.iterateNested(path, rh); - } - break; - default: - LOG(warning, "Unknown docsum field type: %s", ResultConfig::GetResTypeName(type)); - packer.AddEmpty(); // unhandled output type - break; - } -} - - -void -DocsumFilter::writeSlimeField(const DocsumFieldSpec & fieldSpec, - const Document & docsum, - ResultPacker & packer) -{ - if (fieldSpec.getCommand() == VsmsummaryConfig::Fieldmap::Command::NONE) { - const DocsumFieldSpec::FieldIdentifier & fieldId = fieldSpec.getOutputField(); - const document::FieldValue * fv = docsum.getField(fieldId.getId()); - if (fv != nullptr) { - LOG(debug, "writeSlimeField: About to write field '%d' as Slime: field value = '%s'", - fieldId.getId(), fv->toString().c_str()); - CheckUndefinedValueVisitor check_undefined; - fv->accept(check_undefined); - if (!check_undefined.is_undefined()) { - SlimeFieldWriter writer; - if (! fieldSpec.hasIdentityMapping()) { - writer.setInputFields(fieldSpec.getInputFields()); - } - writer.convert(*fv); - const vespalib::stringref out = writer.out(); - packer.AddLongString(out.data(), out.size()); - } else { - packer.AddEmpty(); - } - } else { - LOG(debug, "writeSlimeField: Field value not set for field '%d'", fieldId.getId()); - packer.AddEmpty(); - } - } else { - LOG(debug, "writeSlimeField: Cannot handle this command"); - packer.AddEmpty(); - } -} - bool DocsumFilter::write_flatten_field(const DocsumFieldSpec& field_spec, const Document& doc) { @@ -484,118 +275,23 @@ DocsumFilter::write_flatten_field(const DocsumFieldSpec& field_spec, const Docum return true; } -void -DocsumFilter::writeFlattenField(const DocsumFieldSpec & fieldSpec, - const Document & docsum, - ResultPacker & packer) -{ - if (!write_flatten_field(fieldSpec, docsum)) { - packer.AddEmpty(); - return; - } - const CharBuffer & buf = _flattenWriter.getResult(); - switch (fieldSpec.getResultType()) { - case RES_STRING: - packer.AddString(buf.getBuffer(), buf.getPos()); - break; - case RES_LONG_STRING: - packer.AddLongString(buf.getBuffer(), buf.getPos()); - break; - default: - break; - } - _flattenWriter.clear(); -} - - -void -DocsumFilter::writeEmpty(ResType type, ResultPacker & packer) -{ - // use the 'notdefined' values when writing numeric values - switch (type) { - case RES_INT: - packer.AddInteger(std::numeric_limits<int32_t>::min()); - break; - case RES_SHORT: - packer.AddShort(std::numeric_limits<int16_t>::min()); - break; - case RES_BYTE: - packer.AddByte(std::numeric_limits<int8_t>::min()); - break; - case RES_FLOAT: - packer.AddFloat(std::numeric_limits<float>::quiet_NaN()); - break; - case RES_DOUBLE: - packer.AddDouble(std::numeric_limits<double>::quiet_NaN()); - break; - case RES_INT64: - packer.AddInt64(std::numeric_limits<int64_t>::min()); - break; - default: - packer.AddEmpty(); - break; - } -} - uint32_t DocsumFilter::getSummaryClassId() const { return _tools->getResultClass() ? _tools->getResultClass()->GetClassID() : ResultConfig::NoClassID(); } -DocsumStoreValue +std::unique_ptr<const IDocsumStoreDocument> DocsumFilter::getMappedDocsum(uint32_t id) { const ResultClass *resClass = _tools->getResultClass(); if (resClass == nullptr) { - return DocsumStoreValue(nullptr, 0); + return {}; } const Document & doc = _docsumCache->getDocSum(id); - _packer.Init(resClass->GetClassID()); - uint32_t entry_idx = 0; - for (FieldSpecList::iterator it(_fields.begin()), end = _fields.end(); it != end; ++it, ++entry_idx) { - if (entry_idx != _packer.get_entry_idx()) { - // Entry is not present in docsum blob - continue; - } - ResType type = it->getResultType(); - if (type == RES_JSONSTRING) { - // this really means 'structured data' - writeSlimeField(*it, doc, _packer); - } else { - if (it->getInputFields().size() == 1 && it->getCommand() == VsmsummaryConfig::Fieldmap::Command::NONE) { - const DocsumFieldSpec::FieldIdentifier & fieldId = it->getInputFields()[0]; - const document::FieldValue * field = doc.getField(fieldId.getId()); - if (field != nullptr) { - CheckUndefinedValueVisitor check_undefined; - field->accept(check_undefined); - if (!check_undefined.is_undefined()) { - writeField(*field, fieldId.getPath(), type, _packer); - } else { - writeEmpty(type, _packer); // void input - } - } else { - writeEmpty(type, _packer); // void input - } - } else if (it->getInputFields().size() == 0 && it->getCommand() == VsmsummaryConfig::Fieldmap::Command::NONE) { - LOG(spam, "0 inputfields for output field %u", it->getOutputField().getId()); - writeEmpty(type, _packer); // no input - } else { - writeFlattenField(*it, doc, _packer); - } - } - } - - const char *buf; - uint32_t buflen; - bool ok = _packer.GetDocsumBlob(&buf, &buflen); - if (ok) { - return DocsumStoreValue(buf, buflen, std::make_unique<DocsumStoreVsmDocument>(*this, doc)); - } else { - return DocsumStoreValue(nullptr, 0); - } + return std::make_unique<DocsumStoreVsmDocument>(*this, doc); } search::docsummary::DocsumStoreFieldValue diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h index bcb89a59a12..17a85f142c8 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h @@ -8,14 +8,11 @@ #include <vespa/vsm/vsm/fieldsearchspec.h> #include <vespa/vsm/vsm/flattendocsumwriter.h> #include <vespa/vsm/vsm/vsm-adapter.h> -#include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/searchsummary/docsummary/docsumstore.h> #include <vespa/searchsummary/docsummary/docsum_store_field_value.h> using search::docsummary::IDocsumStore; -using search::docsummary::DocsumStoreValue; using search::docsummary::ResType; -using search::docsummary::ResultPacker; namespace vsm { @@ -35,7 +32,6 @@ private: DocsumToolsPtr _tools; FieldSpecList _fields; // list of summary fields to generate size_t _highestFieldNo; - ResultPacker _packer; FlattenDocsumWriter _flattenWriter; const FieldModifierMap * _snippetModifiers; document::FieldValue::UP _cachedValue; @@ -48,11 +44,7 @@ private: const document::FieldValue * getFieldValue(const DocsumFieldSpec::FieldIdentifier & fieldId, VsmsummaryConfig::Fieldmap::Command command, const Document & docsum, bool & modified); - void writeField(const document::FieldValue & fv, const FieldPath & path, ResType type, ResultPacker & packer); - void writeSlimeField(const DocsumFieldSpec & fieldSpec, const Document & docsum, ResultPacker & packer); bool write_flatten_field(const DocsumFieldSpec& field_spec, const Document & docsum); - void writeFlattenField(const DocsumFieldSpec & fieldSpec, const Document & docsum, ResultPacker & packer); - void writeEmpty(ResType type, ResultPacker & packer); search::docsummary::DocsumStoreFieldValue get_struct_or_multivalue_summary_field(const DocsumFieldSpec& field_spec, const Document& doc); search::docsummary::DocsumStoreFieldValue get_flattened_summary_field(const DocsumFieldSpec& field_spec, const Document& doc); @@ -87,7 +79,7 @@ public: void setDocSumStore(const IDocSumCache & docsumCache) { _docsumCache = &docsumCache; } // Inherit doc from IDocsumStore - DocsumStoreValue getMappedDocsum(uint32_t id) override; + std::unique_ptr<const search::docsummary::IDocsumStoreDocument> getMappedDocsum(uint32_t id) override; uint32_t getNumDocs() const override; uint32_t getSummaryClassId() const override; diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index 206a73a632e..d4f470d9c47 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -9,7 +9,6 @@ LOG_SETUP(".vsm.vsm-adapter"); using search::docsummary::ResConfigEntry; -using search::docsummary::DocsumBlobEntryFilter; using search::docsummary::KeywordExtractor; using search::MatchingElements; using config::ConfigSnapshot; @@ -136,22 +135,7 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) LOG(debug, "configureVsmSummary(): outputclass='%s'", vsmSummary->outputclass.c_str()); // UlfC: debugging // init result config - DocsumBlobEntryFilter docsum_blob_entry_filter; - docsum_blob_entry_filter.add_skip(search::docsummary::RES_INT); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_SHORT); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_BOOL); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_BYTE); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_FLOAT); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_DOUBLE); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_INT64); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_STRING); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_DATA); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_LONG_STRING); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_LONG_DATA); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_JSONSTRING); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_TENSOR); - docsum_blob_entry_filter.add_skip(search::docsummary::RES_FEATUREDATA); - std::unique_ptr<ResultConfig> resCfg(new ResultConfig(docsum_blob_entry_filter)); + auto resCfg = std::make_unique<ResultConfig>(); if ( ! resCfg->ReadConfig(*summary.get(), _configId.c_str())) { throw std::runtime_error("(re-)configuration of VSM (docsum tools) failed due to bad summary config"); } |