diff options
17 files changed, 769 insertions, 1 deletions
diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 5f6e8881f13..4df636e0219 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -24,6 +24,7 @@ vespa_define_module( TESTS src/tests/docsumformat src/tests/docsummary + src/tests/docsummary/attribute_combiner src/tests/docsummary/slime_summary src/tests/extractkeywords ) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt new file mode 100644 index 00000000000..df323b9c982 --- /dev/null +++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_attribute_combiner_test_app TEST + SOURCES + attribute_combiner_test.cpp + DEPENDS + searchsummary +) +vespa_add_test(NAME searchsummary_attribute_combiner_test_app COMMAND searchsummary_attribute_combiner_test_app) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp new file mode 100644 index 00000000000..97fafd0a446 --- /dev/null +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -0,0 +1,217 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcommon/common/undefinedvalues.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributemanager.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/floatbase.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> +#include <vespa/searchsummary/docsummary/docsumstate.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> +#include <vespa/searchsummary/docsummary/attribute_combiner_dfw.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/log/log.h> +LOG_SETUP("attribute_combiner_test"); + +using search::AttributeFactory; +using search::AttributeManager; +using search::AttributeVector; +using search::IntegerAttribute; +using search::FloatingPointAttribute; +using search::StringAttribute; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::attribute::Config; +using search::attribute::IAttributeVector; +using search::attribute::getUndefined; +using search::docsummary::AttributeCombinerDFW; +using search::docsummary::GetDocsumsState; +using search::docsummary::GetDocsumsStateCallback; +using search::docsummary::IDocsumEnvironment; +using search::docsummary::IDocsumFieldWriter; + +namespace { + +vespalib::string +toCompactJsonString(const vespalib::Slime &slime) +{ + vespalib::SimpleBuffer buf; + vespalib::slime::JsonFormat::encode(slime, buf, true); + return buf.get().make_string(); +} + +struct FieldBlock { + vespalib::string input; + vespalib::Slime slime; + search::RawBuf binary; + vespalib::string json; + + explicit FieldBlock(const vespalib::string &jsonInput) + : input(jsonInput), slime(), binary(1024), json() + { + size_t used = vespalib::slime::JsonFormat::decode(jsonInput, slime); + EXPECT_TRUE(used > 0); + json = toCompactJsonString(slime); + search::SlimeOutputRawBufAdapter adapter(binary); + vespalib::slime::BinaryFormat::encode(slime, adapter); + } + const char *data() const { return binary.GetDrainPos(); } + size_t dataLen() const { return binary.GetUsedLen(); } +}; + +struct AttributeManagerFixture +{ + AttributeManager mgr; + + AttributeManagerFixture(); + + ~AttributeManagerFixture(); + + template <typename AttributeType, typename ValueType> + void + buildAttribute(const vespalib::string &name, + BasicType type, + std::vector<std::vector<ValueType>> values); + + void + buildStringAttribute(const vespalib::string &name, + std::vector<std::vector<vespalib::string>> values); + void + buildFloatAttribute(const vespalib::string &name, + std::vector<std::vector<double>> values); + + void + buildIntegerAttribute(const vespalib::string &name, + BasicType type, + std::vector<std::vector<IAttributeVector::largeint_t>> values); +}; + +AttributeManagerFixture::AttributeManagerFixture() + : mgr() +{ + buildStringAttribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}}); + buildIntegerAttribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}}); + buildFloatAttribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }}); +} + +AttributeManagerFixture::~AttributeManagerFixture() = default; + +template <typename AttributeType, typename ValueType> +void +AttributeManagerFixture::buildAttribute(const vespalib::string &name, + BasicType type, + std::vector<std::vector<ValueType>> values) +{ + Config cfg(type, CollectionType::Type::ARRAY); + auto attrBase = AttributeFactory::createAttribute(name, cfg); + EXPECT_TRUE(attrBase); + auto attr = std::dynamic_pointer_cast<AttributeType>(attrBase); + EXPECT_TRUE(attr); + attr->addReservedDoc(); + for (const auto &docValues : values) { + uint32_t docId = 0; + EXPECT_TRUE(attr->addDoc(docId)); + EXPECT_NOT_EQUAL(0u, docId); + for (const auto &value : docValues) { + attr->append(docId, value, 1); + } + attr->commit(); + } + EXPECT_TRUE(mgr.add(attr)); +} + +void +AttributeManagerFixture::buildStringAttribute(const vespalib::string &name, + std::vector<std::vector<vespalib::string>> values) +{ + buildAttribute<StringAttribute, vespalib::string>(name, BasicType::Type::STRING, std::move(values)); +} + +void +AttributeManagerFixture::buildFloatAttribute(const vespalib::string &name, + std::vector<std::vector<double>> values) +{ + buildAttribute<FloatingPointAttribute, double>(name, BasicType::Type::DOUBLE, std::move(values)); +} + +void +AttributeManagerFixture::buildIntegerAttribute(const vespalib::string &name, + BasicType type, + std::vector<std::vector<IAttributeVector::largeint_t>> values) +{ + buildAttribute<IntegerAttribute, IAttributeVector::largeint_t>(name, type, std::move(values)); +} + + +class DummyStateCallback : public GetDocsumsStateCallback +{ +public: + void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } + void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } + void ParseLocation(GetDocsumsState *) override { } + ~DummyStateCallback() override { } +}; + + +struct Fixture +{ + AttributeManagerFixture attrs; + std::unique_ptr<IDocsumFieldWriter> writer; + DummyStateCallback stateCallback; + GetDocsumsState state; + + Fixture(); + ~Fixture(); + void assertWritten(const vespalib::string &exp, uint32_t docId); +}; + +Fixture::Fixture() + : attrs(), + writer(AttributeCombinerDFW::create("array", attrs.mgr)), + stateCallback(), + state(stateCallback) +{ + EXPECT_TRUE(writer->setFieldWriterStateIndex(0)); + state._attrCtx = attrs.mgr.createContext(); + state._fieldWriterStates.resize(1); +} + +Fixture::~Fixture() +{ +} + +void +Fixture::assertWritten(const vespalib::string &expectedJson, uint32_t docId) +{ + vespalib::Slime target; + vespalib::slime::SlimeInserter inserter(target); + writer->insertField(docId, nullptr, &state, search::docsummary::RES_JSONSTRING, inserter); + search::RawBuf binary(1024); + vespalib::string json = toCompactJsonString(target); + search::SlimeOutputRawBufAdapter adapter(binary); + vespalib::slime::BinaryFormat::encode(target, adapter); + FieldBlock block(expectedJson); + if (!EXPECT_EQUAL(block.dataLen(), binary.GetUsedLen()) || + !EXPECT_EQUAL(0, memcmp(block.data(), binary.GetDrainPos(), block.dataLen()))) { + LOG(error, "Expected '%s'", expectedJson.c_str()); + LOG(error, "Expected normalized '%s'", block.json.c_str()); + LOG(error, "Got '%s'", json.c_str()); + } +} + +TEST_F("require that attributes combiner dfw generates correct slime output for array of struct", Fixture()) +{ + f.assertWritten("[ { fval: 110.0, name: \"n1.1\", val: 10}, { name: \"n1.2\", val: 11}]", 1); + f.assertWritten("[ { fval: 120.0, name: \"n2\", val: 20}, { fval: 121.0, val: 21 }]", 2); + f.assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30}, { fval: 131.0, name: \"n3.2\"} ]", 3); + f.assertWritten("[ { }, { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); +} + +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 9009f0bcbc7..ce54e7b0ea7 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -1,6 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchsummary_docsummary OBJECT SOURCES + array_attribute_combiner_dfw.cpp + attribute_combiner_dfw.cpp + attribute_field_writer.cpp resultclass.cpp resultconfig.cpp resultpacker.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp new file mode 100644 index 00000000000..84e329f159d --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -0,0 +1,89 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "array_attribute_combiner_dfw.h" +#include "docsum_field_writer_state.h" +#include "attribute_field_writer.h" +#include <vespa/searchcommon/attribute/iattributecontext.h> +#include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/vespalib/data/slime/cursor.h> + +using search::attribute::IAttributeContext; +using search::attribute::IAttributeVector; +using vespalib::slime::Cursor; + +namespace search::docsummary { + +namespace { + +class ArrayAttributeFieldWriterState : public DocsumFieldWriterState +{ + std::vector<std::unique_ptr<AttributeFieldWriter>> _writers; + +public: + ArrayAttributeFieldWriterState(const std::vector<vespalib::string> &fieldNames, + const std::vector<vespalib::string> &attributeNames, + IAttributeContext &context); + ~ArrayAttributeFieldWriterState() override; + void insertField(uint32_t docId, vespalib::slime::Inserter &target) override; +}; + +ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector<vespalib::string> &fieldNames, + const std::vector<vespalib::string> &attributeNames, + IAttributeContext &context) + : DocsumFieldWriterState() +{ + size_t fields = fieldNames.size(); + _writers.reserve(fields); + for (uint32_t field = 0; field < fields; ++field) { + const IAttributeVector *attr = context.getAttribute(attributeNames[field]); + if (attr != nullptr) { + _writers.emplace_back(AttributeFieldWriter::create(fieldNames[field], *attr)); + } + } +} + +ArrayAttributeFieldWriterState::~ArrayAttributeFieldWriterState() = default; + +void +ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Inserter &target) +{ + uint32_t elems = 0; + for (auto &writer : _writers) { + writer->fetch(docId); + if (elems < writer->size()) { + elems = writer->size(); + } + } + Cursor &arr = target.insertArray(); + for (uint32_t idx = 0; idx < elems; ++idx) { + Cursor &obj = arr.addObject(); + for (auto &writer : _writers) { + writer->print(idx, obj); + } + } +} + +} + +ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fieldName, + const std::vector<vespalib::string> &fields) + : AttributeCombinerDFW(fieldName), + _fields(fields), + _attributeNames() +{ + _attributeNames.reserve(_fields.size()); + vespalib::string prefix = fieldName + "."; + for (const auto &field : _fields) { + _attributeNames.emplace_back(prefix + field); + } +} + +ArrayAttributeCombinerDFW::~ArrayAttributeCombinerDFW() = default; + +std::unique_ptr<DocsumFieldWriterState> +ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context) +{ + return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context); +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h new file mode 100644 index 00000000000..c02d2bd5da6 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -0,0 +1,29 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "attribute_combiner_dfw.h" + +namespace search::attribute { class IAttributeContext; } + +namespace search::docsummary { + +class DocsumFieldWriterState; + +/* + * This class reads values from multiple struct field attributes and + * inserts them as an array of struct. + */ +class ArrayAttributeCombinerDFW : public AttributeCombinerDFW +{ + std::vector<vespalib::string> _fields; + std::vector<vespalib::string> _attributeNames; + + std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) override; +public: + ArrayAttributeCombinerDFW(const vespalib::string &fieldName, + const std::vector<vespalib::string> &fields); + ~ArrayAttributeCombinerDFW() override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp new file mode 100644 index 00000000000..b532cfb273a --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -0,0 +1,141 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_combiner_dfw.h" +#include "array_attribute_combiner_dfw.h" +#include "docsum_field_writer_state.h" +#include "docsumstate.h" +#include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/iattributemanager.h> +#include <algorithm> + +#include <vespa/log/log.h> +LOG_SETUP(".searchsummary.docsummary.attribute_combiner_dfw"); + +using search::AttributeGuard; +using search::AttributeVector; +using search::attribute::CollectionType; + +namespace search::docsummary { + +namespace { + +class StructFields +{ + std::vector<vespalib::string> _mapFields; + std::vector<vespalib::string> _arrayFields; + bool _hasMapKey; + bool _error; + +public: + StructFields(const vespalib::string &fieldName, const IAttributeManager &attrMgr); + ~StructFields(); + const std::vector<vespalib::string> &getMapFields() const { return _mapFields; } + const std::vector<vespalib::string> &getArrayFields() const { return _arrayFields; } + bool hasMapKey() const { return _hasMapKey; } + bool getError() const { return _error; } +}; + + +StructFields::StructFields(const vespalib::string &fieldName, const IAttributeManager &attrMgr) + : _mapFields(), + _arrayFields(), + _hasMapKey(false), + _error(false) +{ + // Note: Doesn't handle imported attributes + std::vector<AttributeGuard> attrs; + attrMgr.getAttributeList(attrs); + vespalib::string prefix = fieldName + "."; + vespalib::string keyName = prefix + "key"; + vespalib::string valuePrefix = prefix + "value."; + for (const auto &guard : attrs) { + vespalib::string name = guard->getName(); + if (name.substr(0, prefix.size()) != prefix) { + continue; + } + auto collType = guard->getCollectionType(); + if (collType != CollectionType::Type::ARRAY) { + LOG(warning, "Attribute %s is not an array attribute", name.c_str()); + _error = true; + break; + } + if (name.substr(0, valuePrefix.size()) == valuePrefix) { + _mapFields.emplace_back(name.substr(valuePrefix.size())); + } else { + _arrayFields.emplace_back(name.substr(prefix.size())); + if (name == keyName) { + _hasMapKey = true; + } + } + } + if (!_error) { + std::sort(_arrayFields.begin(), _arrayFields.end()); + std::sort(_mapFields.begin(), _mapFields.end()); + if (!_mapFields.empty()) { + if (!_hasMapKey) { + LOG(warning, "Missing key attribute '%s', have value attributes for map", keyName.c_str()); + _error = true; + } else if (_arrayFields.size() != 1u) { + LOG(warning, "Could not determine if field '%s' is array or map of struct", fieldName.c_str()); + _error = true; + } + } + } +} + +StructFields::~StructFields() = default; + +} + +AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName) + : IDocsumFieldWriter(), + _stateIndex(0), + _fieldName(fieldName) +{ +} + +AttributeCombinerDFW::~AttributeCombinerDFW() = default; + +bool +AttributeCombinerDFW::IsGenerated() const +{ + return true; +} + +bool +AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) +{ + _stateIndex = fieldWriterStateIndex; + return true; +} + +std::unique_ptr<IDocsumFieldWriter> +AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManager &attrMgr) +{ + StructFields structFields(fieldName, attrMgr); + if (structFields.getError()) { + return std::unique_ptr<IDocsumFieldWriter>(); + } else if (!structFields.getMapFields().empty()) { + LOG(warning, "map of struct is not yet supported for field '%s'", fieldName.c_str()); + return std::unique_ptr<IDocsumFieldWriter>(); + } + return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields.getArrayFields()); +} + +void +AttributeCombinerDFW::insertField(uint32_t docid, + GeneralResult *, + GetDocsumsState *state, + ResType, + vespalib::slime::Inserter &target) +{ + auto &fieldWriterState = state->_fieldWriterStates[_stateIndex]; + if (!fieldWriterState) { + fieldWriterState = allocFieldWriterState(*state->_attrCtx); + } + fieldWriterState->insertField(docid, target); +} + +} + diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h new file mode 100644 index 00000000000..ef54522a923 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsumfieldwriter.h" + +namespace search::attribute { class IAttributeContext; } + +namespace search::docsummary { + +class DocsumFieldWriterState; +class DynamicDocsumWriter; + +/* + * This class reads values from multiple struct field attributes and + * inserts them as an array of struct or a map of struct. + */ +class AttributeCombinerDFW : public IDocsumFieldWriter +{ +protected: + uint32_t _stateIndex; + vespalib::string _fieldName; + AttributeCombinerDFW(const vespalib::string &fieldName); +protected: + virtual std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) = 0; +public: + ~AttributeCombinerDFW() override; + bool IsGenerated() const override; + bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; + static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, IAttributeManager &attrMgr); + void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, + ResType type, vespalib::slime::Inserter &target) override; +}; + +} + diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp new file mode 100644 index 00000000000..2eebe7137dc --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp @@ -0,0 +1,172 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_field_writer.h" +#include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/common/undefinedvalues.h> +#include <vespa/vespalib/data/slime/cursor.h> +#include <cassert> + +using search::attribute::BasicType; +using search::attribute::IAttributeVector; +using search::attribute::getUndefined; +using vespalib::slime::Cursor; + +namespace search::docsummary { + +AttributeFieldWriter::AttributeFieldWriter(const vespalib::string &fieldName, + const IAttributeVector &attr) + : _fieldName(fieldName), + _attr(attr), + _size(0) +{ +} + +AttributeFieldWriter::~AttributeFieldWriter() = default; + +namespace { + +template <class Content> +class WriteField : public AttributeFieldWriter +{ +protected: + Content _content; + + WriteField(const vespalib::string &fieldName, const IAttributeVector &attr); + ~WriteField() override; +private: + void fetch(uint32_t docId) override; +}; + +class WriteStringField : public WriteField<search::attribute::ConstCharContent> +{ +public: + WriteStringField(const vespalib::string &fieldName, + const IAttributeVector &attr); + ~WriteStringField() override; + void print(uint32_t idx, Cursor &cursor) override; +}; + + +class WriteFloatField : public WriteField<search::attribute::FloatContent> +{ +public: + WriteFloatField(const vespalib::string &fieldName, + const IAttributeVector &attr); + ~WriteFloatField() override; + void print(uint32_t idx, Cursor &cursor) override; +}; + +class WriteIntField : public WriteField<search::attribute::IntegerContent> +{ + IAttributeVector::largeint_t _undefined; +public: + WriteIntField(const vespalib::string &fieldName, + const IAttributeVector &attr, + IAttributeVector::largeint_t undefined); + ~WriteIntField() override; + void print(uint32_t idx, Cursor &cursor) override; +}; + +template <class Content> +WriteField<Content>::WriteField(const vespalib::string &fieldName, const IAttributeVector &attr) + : AttributeFieldWriter(fieldName, attr), + _content() +{ +} + +template <class Content> +WriteField<Content>::~WriteField() = default; + +template <class Content> +void +WriteField<Content>::fetch(uint32_t docId) +{ + _content.fill(_attr, docId); + _size = _content.size(); +} + +WriteStringField::WriteStringField(const vespalib::string &fieldName, + const IAttributeVector &attr) + : WriteField(fieldName, attr) +{ +} + +WriteStringField::~WriteStringField() = default; + +void +WriteStringField::print(uint32_t idx, Cursor &cursor) +{ + if (idx < _size) { + const char *s = _content[idx]; + if (s[0] != '\0') { + cursor.setString(_fieldName, vespalib::Memory(s)); + } + } +} + +WriteFloatField::WriteFloatField(const vespalib::string &fieldName, + const IAttributeVector &attr) + : WriteField(fieldName, attr) +{ +} + +WriteFloatField::~WriteFloatField() = default; + +void +WriteFloatField::print(uint32_t idx, Cursor &cursor) +{ + if (idx < _size) { + double val = _content[idx]; + if (!search::attribute::isUndefined(val)) { + cursor.setDouble(_fieldName, val); + } + } +} + +WriteIntField::WriteIntField(const vespalib::string &fieldName, + const IAttributeVector &attr, + IAttributeVector::largeint_t undefined) + : WriteField(fieldName, attr), + _undefined(undefined) +{ +} + +WriteIntField::~WriteIntField() = default; + +void +WriteIntField::print(uint32_t idx, Cursor &cursor) +{ + if (idx < _size) { + auto val = _content[idx]; + if (val != _undefined) { + cursor.setLong(_fieldName, _content[idx]); + } + } +} + +} + +std::unique_ptr<AttributeFieldWriter> +AttributeFieldWriter::create(const vespalib::string &fieldName, const IAttributeVector &attr) +{ + switch (attr.getBasicType()) { + case BasicType::INT8: + return std::make_unique<WriteIntField>(fieldName, attr, getUndefined<int8_t>()); + case BasicType::INT16: + return std::make_unique<WriteIntField>(fieldName, attr, getUndefined<int16_t>()); + case BasicType::INT32: + return std::make_unique<WriteIntField>(fieldName, attr, getUndefined<int32_t>()); + case BasicType::INT64: + return std::make_unique<WriteIntField>(fieldName, attr, getUndefined<int64_t>()); + case BasicType::FLOAT: + case BasicType::DOUBLE: + return std::make_unique<WriteFloatField>(fieldName, attr); + case BasicType::STRING: + return std::make_unique<WriteStringField>(fieldName, attr); + default: + assert(false); + abort(); + } +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h new file mode 100644 index 00000000000..104455a0e79 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h @@ -0,0 +1,34 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/data/memory.h> + +namespace search::attribute { class IAttributeVector; } +namespace vespalib::slime { class Cursor; } + +namespace search::docsummary { + +/* + * This class reads values from a struct field attribute and inserts + * them into proper position in an array of struct or map of struct. + * If the value to be inserted is considered to be undefined then + * the value is not inserted. + */ +class AttributeFieldWriter +{ +protected: + const vespalib::Memory _fieldName; + const search::attribute::IAttributeVector &_attr; + size_t _size; +public: + AttributeFieldWriter(const vespalib::string &fieldName, + const search::attribute::IAttributeVector &attr); + virtual ~AttributeFieldWriter(); + virtual void fetch(uint32_t docId) = 0; + virtual void print(uint32_t idx, vespalib::slime::Cursor &cursor) = 0; + static std::unique_ptr<AttributeFieldWriter> create(const vespalib::string &fieldName, const search::attribute::IAttributeVector &attr); + uint32_t size() const { return _size; } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h new file mode 100644 index 00000000000..940cfd6ce06 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::slime { class Inserter; } + +namespace search::docsummary { + +/* + * A subclass of this class can be instantiated by a document field writer to + * track extra state during handling of a document summary request and + * insert the field value using that state. + */ +class DocsumFieldWriterState +{ +public: + virtual void insertField(uint32_t docId, vespalib::slime::Inserter &target) = 0; + virtual ~DocsumFieldWriterState() = default; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp index 7b463352155..18e7e471663 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp @@ -21,6 +21,12 @@ using search::common::Location; const vespalib::string IDocsumFieldWriter::_empty(""); +bool +IDocsumFieldWriter::setFieldWriterStateIndex(uint32_t) +{ + return false; // Don't need any field writer state by default +} + //-------------------------------------------------------------------------- EmptyDFW::EmptyDFW() { } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h index abce5c12227..51079f7736e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h @@ -40,6 +40,7 @@ public: } void setIndex(size_t v) { _index = v; } size_t getIndex() const { return _index; } + virtual bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex); private: size_t _index; static const vespalib::string _empty; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 91953612f6a..b0431b6e6ac 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -4,6 +4,7 @@ #include <vespa/juniper/rpinterface.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/location.h> +#include "docsum_field_writer_state.h" namespace search { namespace docsummary { @@ -19,6 +20,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _docSumFieldSpace(_docSumFieldSpaceStore, sizeof(_docSumFieldSpaceStore)), // only alloc buffer if needed _attrCtx(), _attributes(), + _fieldWriterStates(), _jsonStringer(), _parsedLocation(), _summaryFeatures(NULL), diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index 4ffed79043e..fa47d5244eb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -23,6 +23,7 @@ namespace search::docsummary { class GetDocsumsState; class IDocsumEnvironment; class KeywordExtractor; +class DocsumFieldWriterState; class GetDocsumsStateCallback { @@ -70,6 +71,7 @@ public: char _docSumFieldSpaceStore[2048]; std::unique_ptr<search::attribute::IAttributeContext> _attrCtx; std::vector<const search::attribute::IAttributeVector *> _attributes; + std::vector<std::unique_ptr<DocsumFieldWriterState>> _fieldWriterStates; vespalib::JSONStringer _jsonStringer; // used by AbsDistanceDFW diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index bf660b1319b..abd1780b773 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -2,6 +2,7 @@ #include "docsumwriter.h" #include "docsumstate.h" +#include "docsum_field_writer_state.h" #include <vespa/searchlib/common/transport.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/attribute/iattributemanager.h> @@ -77,7 +78,6 @@ DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, uint32_t id) const } } - static void convertEntry(GetDocsumsState *state, const ResConfigEntry *resCfg, const ResEntry *entry, @@ -194,6 +194,7 @@ DynamicDocsumWriter::DynamicDocsumWriter( ResultConfig *config, KeywordExtractor _defaultOutputClass(ResultConfig::NoClassID()), _numClasses(config->GetNumResultClasses()), _numEnumValues(config->GetFieldNameEnum().GetNumEntries()), + _numFieldWriterStates(0), _classInfoTable(nullptr), _overrideTable(nullptr) { @@ -267,6 +268,9 @@ DynamicDocsumWriter::Override(const char *fieldName, IDocsumFieldWriter *writer) writer->setIndex(fieldEnumValue); _overrideTable[fieldEnumValue] = writer; + if (writer->setFieldWriterStateIndex(_numFieldWriterStates)) { + ++_numFieldWriterStates; + } for (ResultConfig::iterator it(_resultConfig->begin()), mt(_resultConfig->end()); it != mt; it++) { @@ -288,6 +292,7 @@ DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *sta state->_kwExtractor = _keywordExtractor; state->_attrCtx = attrMan.createContext(); state->_attributes.resize(_numEnumValues); + state->_fieldWriterStates.resize(_numFieldWriterStates); for (size_t i(0); i < state->_attributes.size(); i++) { const IDocsumFieldWriter *fw = _overrideTable[i]; if (fw) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index 6ef21a71e74..92b26d5cf14 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -54,6 +54,7 @@ private: uint32_t _defaultOutputClass; uint32_t _numClasses; uint32_t _numEnumValues; + uint32_t _numFieldWriterStates; ResultClass::DynamicInfo *_classInfoTable; IDocsumFieldWriter **_overrideTable; |