From 3eccf48b7f63fcfe678a42957aae16d021dccebc Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 6 Jun 2018 21:02:19 +0200 Subject: Add attribute combiner dynamic field writer. --- searchsummary/CMakeLists.txt | 1 + .../docsummary/attribute_combiner/CMakeLists.txt | 8 + .../attribute_combiner/attribute_combiner_test.cpp | 220 +++++++++++++++++++++ .../vespa/searchsummary/docsummary/CMakeLists.txt | 3 + .../docsummary/array_attribute_combiner_dfw.cpp | 89 +++++++++ .../docsummary/array_attribute_combiner_dfw.h | 29 +++ .../docsummary/attribute_combiner_dfw.cpp | 105 ++++++++++ .../docsummary/attribute_combiner_dfw.h | 36 ++++ .../docsummary/attribute_field_writer.cpp | 170 ++++++++++++++++ .../docsummary/attribute_field_writer.h | 34 ++++ .../docsummary/docsum_field_writer_state.h | 21 ++ .../searchsummary/docsummary/docsumfieldwriter.cpp | 6 + .../searchsummary/docsummary/docsumfieldwriter.h | 1 + .../vespa/searchsummary/docsummary/docsumstate.cpp | 2 + .../vespa/searchsummary/docsummary/docsumstate.h | 2 + .../searchsummary/docsummary/docsumwriter.cpp | 7 +- .../vespa/searchsummary/docsummary/docsumwriter.h | 1 + 17 files changed, 734 insertions(+), 1 deletion(-) create mode 100644 searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt create mode 100644 searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp create mode 100644 searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp create mode 100644 searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h create mode 100644 searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp create mode 100644 searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h create mode 100644 searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp create mode 100644 searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h create mode 100644 searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h (limited to 'searchsummary') diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 5f6e8881f13..4df636e0219 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -24,6 +24,7 @@ vespa_define_module( TESTS src/tests/docsumformat src/tests/docsummary + src/tests/docsummary/attribute_combiner src/tests/docsummary/slime_summary src/tests/extractkeywords ) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt new file mode 100644 index 00000000000..5ebafc71b44 --- /dev/null +++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_attribute_combiner_test_app TEST + SOURCES + attribute_combiner_test.cpp + DEPENDS + searchsummary +) +vespa_add_test(NAME searchsummary_attribute_combiner_test_app COMMAND searchsummary_attribute_combiner_test_app) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp new file mode 100644 index 00000000000..8b2f7357982 --- /dev/null +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -0,0 +1,220 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +LOG_SETUP("attribute_combiner_test"); + +using search::AttributeFactory; +using search::AttributeManager; +using search::AttributeVector; +using search::IntegerAttribute; +using search::FloatingPointAttribute; +using search::StringAttribute; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::attribute::Config; +using search::attribute::IAttributeVector; +using search::attribute::getUndefined; +using search::docsummary::AttributeCombinerDFW; +using search::docsummary::GetDocsumsState; +using search::docsummary::GetDocsumsStateCallback; +using search::docsummary::IDocsumEnvironment; +using search::docsummary::IDocsumFieldWriter; + +namespace { + +struct FieldBlock { + vespalib::string input; + vespalib::Slime slime; + search::RawBuf binary; + vespalib::string json; + + explicit FieldBlock(const vespalib::string &jsonInput) + : input(jsonInput), slime(), binary(1024), json() + { + size_t used = vespalib::slime::JsonFormat::decode(jsonInput, slime); + EXPECT_TRUE(used > 0); + { + search::SlimeOutputRawBufAdapter adapter(binary); + vespalib::slime::JsonFormat::encode(slime, adapter, true); + json.assign(binary.GetDrainPos(), binary.GetUsedLen()); + binary.reset(); + } + search::SlimeOutputRawBufAdapter adapter(binary); + vespalib::slime::BinaryFormat::encode(slime, adapter); + } + const char *data() const { return binary.GetDrainPos(); } + size_t dataLen() const { return binary.GetUsedLen(); } +}; + +struct AttributeManagerFixture +{ + AttributeManager mgr; + + AttributeManagerFixture(); + + ~AttributeManagerFixture(); + + template + void + buildAttribute(const vespalib::string &name, + BasicType type, + std::vector> values); + + void + buildStringAttribute(const vespalib::string &name, + std::vector> values); + void + buildFloatAttribute(const vespalib::string &name, + std::vector> values); + + void + buildIntegerAttribute(const vespalib::string &name, + BasicType type, + std::vector> values); +}; + +AttributeManagerFixture::AttributeManagerFixture() + : mgr() +{ + buildStringAttribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}}); + buildIntegerAttribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined(), 41}}); + buildFloatAttribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined(), 141.0 }}); +} + +AttributeManagerFixture::~AttributeManagerFixture() = default; + +template +void +AttributeManagerFixture::buildAttribute(const vespalib::string &name, + BasicType type, + std::vector> values) +{ + Config cfg(type, CollectionType::Type::ARRAY); + auto attrBase = AttributeFactory::createAttribute(name, cfg); + EXPECT_TRUE(attrBase); + auto attr = std::dynamic_pointer_cast(attrBase); + EXPECT_TRUE(attr); + attr->addReservedDoc(); + for (const auto &docValues : values) { + uint32_t docId = 0; + EXPECT_TRUE(attr->addDoc(docId)); + EXPECT_NOT_EQUAL(0u, docId); + for (const auto &value : docValues) { + attr->append(docId, value, 1); + } + attr->commit(); + } + EXPECT_TRUE(mgr.add(attr)); +} + +void +AttributeManagerFixture::buildStringAttribute(const vespalib::string &name, + std::vector> values) +{ + buildAttribute(name, BasicType::Type::STRING, std::move(values)); +} + +void +AttributeManagerFixture::buildFloatAttribute(const vespalib::string &name, + std::vector> values) +{ + buildAttribute(name, BasicType::Type::DOUBLE, std::move(values)); +} + +void +AttributeManagerFixture::buildIntegerAttribute(const vespalib::string &name, + BasicType type, + std::vector> values) +{ + buildAttribute(name, type, std::move(values)); +} + + +class DummyStateCallback : public GetDocsumsStateCallback +{ +public: + void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } + void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } + void ParseLocation(GetDocsumsState *) override { } + ~DummyStateCallback() override { } +}; + + +struct Fixture +{ + AttributeManagerFixture attrs; + std::unique_ptr writer; + DummyStateCallback stateCallback; + GetDocsumsState state; + + Fixture(); + ~Fixture(); + void assertWritten(const vespalib::string &exp, uint32_t docId); +}; + +Fixture::Fixture() + : attrs(), + writer(AttributeCombinerDFW::create("array", attrs.mgr)), + stateCallback(), + state(stateCallback) +{ + EXPECT_TRUE(writer->setFieldWriterStateIndex(0)); + state._attrCtx = attrs.mgr.createContext(); + state._fieldWriterStates.resize(1); +} + +Fixture::~Fixture() +{ +} + +void +Fixture::assertWritten(const vespalib::string &expected, uint32_t docId) +{ + vespalib::Slime target; + vespalib::slime::SlimeInserter inserter(target); + writer->insertField(docId, nullptr, &state, search::docsummary::RES_JSONSTRING, inserter); + search::RawBuf binary(1024); + vespalib::string json; + { + search::SlimeOutputRawBufAdapter adapter(binary); + vespalib::slime::JsonFormat::encode(target, adapter, true); + json.assign(binary.GetDrainPos(), binary.GetUsedLen()); + binary.reset(); + } + search::SlimeOutputRawBufAdapter adapter(binary); + vespalib::slime::BinaryFormat::encode(target, adapter); + FieldBlock block(expected); + if (!EXPECT_EQUAL(block.dataLen(), binary.GetUsedLen()) || + !EXPECT_EQUAL(0, memcmp(block.data(), binary.GetDrainPos(), block.dataLen()))) { + LOG(error, "Expected '%s'", expected.c_str()); + LOG(error, "Expected normalized '%s'", block.json.c_str()); + LOG(error, "Got '%s'", json.c_str()); + } +} + +TEST_F("require that attributes combiner dfw generates correct slime output for array of struct", Fixture()) +{ + f.assertWritten("[ { fval: 110.0, name: \"n1.1\", val: 10}, { name: \"n1.2\", val: 11}]", 1); + f.assertWritten("[ { fval: 120.0, name: \"n2\", val: 20}, { fval: 121.0, val: 21 }]", 2); + f.assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30}, { fval: 131.0, name: \"n3.2\"} ]", 3); + f.assertWritten("[ { }, { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); +} + +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 9009f0bcbc7..ce54e7b0ea7 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -1,6 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchsummary_docsummary OBJECT SOURCES + array_attribute_combiner_dfw.cpp + attribute_combiner_dfw.cpp + attribute_field_writer.cpp resultclass.cpp resultconfig.cpp resultpacker.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp new file mode 100644 index 00000000000..84e329f159d --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -0,0 +1,89 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "array_attribute_combiner_dfw.h" +#include "docsum_field_writer_state.h" +#include "attribute_field_writer.h" +#include +#include +#include + +using search::attribute::IAttributeContext; +using search::attribute::IAttributeVector; +using vespalib::slime::Cursor; + +namespace search::docsummary { + +namespace { + +class ArrayAttributeFieldWriterState : public DocsumFieldWriterState +{ + std::vector> _writers; + +public: + ArrayAttributeFieldWriterState(const std::vector &fieldNames, + const std::vector &attributeNames, + IAttributeContext &context); + ~ArrayAttributeFieldWriterState() override; + void insertField(uint32_t docId, vespalib::slime::Inserter &target) override; +}; + +ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector &fieldNames, + const std::vector &attributeNames, + IAttributeContext &context) + : DocsumFieldWriterState() +{ + size_t fields = fieldNames.size(); + _writers.reserve(fields); + for (uint32_t field = 0; field < fields; ++field) { + const IAttributeVector *attr = context.getAttribute(attributeNames[field]); + if (attr != nullptr) { + _writers.emplace_back(AttributeFieldWriter::create(fieldNames[field], *attr)); + } + } +} + +ArrayAttributeFieldWriterState::~ArrayAttributeFieldWriterState() = default; + +void +ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Inserter &target) +{ + uint32_t elems = 0; + for (auto &writer : _writers) { + writer->fetch(docId); + if (elems < writer->size()) { + elems = writer->size(); + } + } + Cursor &arr = target.insertArray(); + for (uint32_t idx = 0; idx < elems; ++idx) { + Cursor &obj = arr.addObject(); + for (auto &writer : _writers) { + writer->print(idx, obj); + } + } +} + +} + +ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fieldName, + const std::vector &fields) + : AttributeCombinerDFW(fieldName), + _fields(fields), + _attributeNames() +{ + _attributeNames.reserve(_fields.size()); + vespalib::string prefix = fieldName + "."; + for (const auto &field : _fields) { + _attributeNames.emplace_back(prefix + field); + } +} + +ArrayAttributeCombinerDFW::~ArrayAttributeCombinerDFW() = default; + +std::unique_ptr +ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context) +{ + return std::make_unique(_fields, _attributeNames, context); +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h new file mode 100644 index 00000000000..c02d2bd5da6 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -0,0 +1,29 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "attribute_combiner_dfw.h" + +namespace search::attribute { class IAttributeContext; } + +namespace search::docsummary { + +class DocsumFieldWriterState; + +/* + * This class reads values from multiple struct field attributes and + * inserts them as an array of struct. + */ +class ArrayAttributeCombinerDFW : public AttributeCombinerDFW +{ + std::vector _fields; + std::vector _attributeNames; + + std::unique_ptr allocFieldWriterState(search::attribute::IAttributeContext &context) override; +public: + ArrayAttributeCombinerDFW(const vespalib::string &fieldName, + const std::vector &fields); + ~ArrayAttributeCombinerDFW() override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp new file mode 100644 index 00000000000..9383154979f --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -0,0 +1,105 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_combiner_dfw.h" +#include "array_attribute_combiner_dfw.h" +#include "docsum_field_writer_state.h" +#include "docsumstate.h" +#include +#include +#include +#include + +#include +LOG_SETUP(".searchsummary.docsummary.attribute_combiner_dfw"); + +using search::AttributeGuard; +using search::AttributeVector; +using search::attribute::CollectionType; + +namespace search::docsummary { + +AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName) + : IDocsumFieldWriter(), + _stateIndex(0), + _fieldName(fieldName) +{ +} + +AttributeCombinerDFW::~AttributeCombinerDFW() = default; + +bool +AttributeCombinerDFW::IsGenerated() const +{ + return true; +} + +bool +AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) +{ + _stateIndex = fieldWriterStateIndex; + return true; +} + +std::unique_ptr +AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManager &attrMgr) +{ + // Note: Doesn't handle imported attributes + std::vector attrs; + attrMgr.getAttributeList(attrs); + vespalib::string prefix = fieldName + "."; + vespalib::string keyName = prefix + "key"; + vespalib::string valuePrefix = prefix + "value."; + std::vector mapFields; + std::vector arrayFields; + bool foundKey = false; + for (const auto &guard : attrs) { + vespalib::string name = guard->getName(); + if (name.substr(0, prefix.size()) != prefix) { + continue; + } + auto collType = guard->getCollectionType(); + if (collType != CollectionType::Type::ARRAY) { + LOG(warning, "Attribute %s is not an array attribute", name.c_str()); + return std::unique_ptr(); + } + if (name.substr(0, valuePrefix.size()) == valuePrefix) { + mapFields.emplace_back(name.substr(valuePrefix.size())); + } else { + arrayFields.emplace_back(name.substr(prefix.size())); + if (name == keyName) { + foundKey = true; + } + } + } + if (!mapFields.empty()) { + if (!foundKey) { + LOG(warning, "Missing key attribute '%s', have value attributes for map", keyName.c_str()); + return std::unique_ptr(); + } + if (arrayFields.size() != 1u) { + LOG(warning, "Could not determine if field '%s' is array or map of struct", fieldName.c_str()); + return std::unique_ptr(); + } + LOG(warning, "map of struct is not yet supported for field '%s'", fieldName.c_str()); + return std::unique_ptr(); + } + std::sort(arrayFields.begin(), arrayFields.end()); + return std::make_unique(fieldName, arrayFields); +} + +void +AttributeCombinerDFW::insertField(uint32_t docid, + GeneralResult *, + GetDocsumsState *state, + ResType, + vespalib::slime::Inserter &target) +{ + auto &fieldWriterState = state->_fieldWriterStates[_stateIndex]; + if (!fieldWriterState) { + fieldWriterState = allocFieldWriterState(*state->_attrCtx); + } + fieldWriterState->insertField(docid, target); +} + +} + diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h new file mode 100644 index 00000000000..ef54522a923 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsumfieldwriter.h" + +namespace search::attribute { class IAttributeContext; } + +namespace search::docsummary { + +class DocsumFieldWriterState; +class DynamicDocsumWriter; + +/* + * This class reads values from multiple struct field attributes and + * inserts them as an array of struct or a map of struct. + */ +class AttributeCombinerDFW : public IDocsumFieldWriter +{ +protected: + uint32_t _stateIndex; + vespalib::string _fieldName; + AttributeCombinerDFW(const vespalib::string &fieldName); +protected: + virtual std::unique_ptr allocFieldWriterState(search::attribute::IAttributeContext &context) = 0; +public: + ~AttributeCombinerDFW() override; + bool IsGenerated() const override; + bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; + static std::unique_ptr create(const vespalib::string &fieldName, IAttributeManager &attrMgr); + void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, + ResType type, vespalib::slime::Inserter &target) override; +}; + +} + diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp new file mode 100644 index 00000000000..66940f37a13 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp @@ -0,0 +1,170 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_field_writer.h" +#include +#include +#include + +using search::attribute::BasicType; +using search::attribute::IAttributeVector; +using search::attribute::getUndefined; +using vespalib::slime::Cursor; + +namespace search::docsummary { + +AttributeFieldWriter::AttributeFieldWriter(const vespalib::string &fieldName, + const IAttributeVector &attr) + : _fieldName(fieldName), + _attr(attr), + _len(0) +{ +} + +AttributeFieldWriter::~AttributeFieldWriter() = default; + +namespace { + +template +class WriteField : public AttributeFieldWriter +{ +protected: + Content _content; + + WriteField(const vespalib::string &fieldName, const IAttributeVector &attr); + ~WriteField() override; +private: + void fetch(uint32_t docId) override; +}; + +class WriteStringField : public WriteField +{ +public: + WriteStringField(const vespalib::string &fieldName, + const IAttributeVector &attr); + ~WriteStringField() override; + void print(uint32_t idx, Cursor &cursor) override; +}; + + +class WriteFloatField : public WriteField +{ +public: + WriteFloatField(const vespalib::string &fieldName, + const IAttributeVector &attr); + ~WriteFloatField() override; + void print(uint32_t idx, Cursor &cursor) override; +}; + +class WriteIntField : public WriteField +{ + IAttributeVector::largeint_t _undefined; +public: + WriteIntField(const vespalib::string &fieldName, + const IAttributeVector &attr, + IAttributeVector::largeint_t undefined); + ~WriteIntField() override; + void print(uint32_t idx, Cursor &cursor) override; +}; + +template +WriteField::WriteField(const vespalib::string &fieldName, const IAttributeVector &attr) + : AttributeFieldWriter(fieldName, attr), + _content() +{ +} + +template +WriteField::~WriteField() = default; + +template +void +WriteField::fetch(uint32_t docId) +{ + _content.fill(_attr, docId); + _len = _content.size(); +} + +WriteStringField::WriteStringField(const vespalib::string &fieldName, + const IAttributeVector &attr) + : WriteField(fieldName, attr) +{ +} + +WriteStringField::~WriteStringField() = default; + +void +WriteStringField::print(uint32_t idx, Cursor &cursor) +{ + if (idx < _len) { + const char *s = _content[idx]; + if (s[0] != '\0') { + cursor.setString(_fieldName, vespalib::Memory(s)); + } + } +} + +WriteFloatField::WriteFloatField(const vespalib::string &fieldName, + const IAttributeVector &attr) + : WriteField(fieldName, attr) +{ +} + +WriteFloatField::~WriteFloatField() = default; + +void +WriteFloatField::print(uint32_t idx, Cursor &cursor) +{ + if (idx < _len) { + double val = _content[idx]; + if (!search::attribute::isUndefined(val)) { + cursor.setDouble(_fieldName, val); + } + } +} + +WriteIntField::WriteIntField(const vespalib::string &fieldName, + const IAttributeVector &attr, + IAttributeVector::largeint_t undefined) + : WriteField(fieldName, attr), + _undefined(undefined) +{ +} + +WriteIntField::~WriteIntField() = default; + +void +WriteIntField::print(uint32_t idx, Cursor &cursor) +{ + if (idx < _len) { + auto val = _content[idx]; + if (val != _undefined) { + cursor.setLong(_fieldName, _content[idx]); + } + } +} + +} + +std::unique_ptr +AttributeFieldWriter::create(const vespalib::string &fieldName, const IAttributeVector &attr) +{ + switch (attr.getBasicType()) { + case BasicType::INT8: + return std::make_unique(fieldName, attr, getUndefined()); + case BasicType::INT16: + return std::make_unique(fieldName, attr, getUndefined()); + case BasicType::INT32: + return std::make_unique(fieldName, attr, getUndefined()); + case BasicType::INT64: + return std::make_unique(fieldName, attr, getUndefined()); + case BasicType::FLOAT: + case BasicType::DOUBLE: + return std::make_unique(fieldName, attr); + case BasicType::STRING: + return std::make_unique(fieldName, attr); + default: + abort(); + } +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h new file mode 100644 index 00000000000..03eda4b067a --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h @@ -0,0 +1,34 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + +namespace search::attribute { class IAttributeVector; } +namespace vespalib::slime { class Cursor; } + +namespace search::docsummary { + +/* + * This class reads values from a struct field attribute and inserts + * them into proper position in an array of struct or map of struct. + * If the value to be inserted is considered to be undefined then + * the value is not inserted. + */ +class AttributeFieldWriter +{ +protected: + const vespalib::Memory _fieldName; + const search::attribute::IAttributeVector &_attr; + size_t _len; +public: + AttributeFieldWriter(const vespalib::string &fieldName, + const search::attribute::IAttributeVector &attr); + virtual ~AttributeFieldWriter(); + virtual void fetch(uint32_t docId) = 0; + virtual void print(uint32_t idx, vespalib::slime::Cursor &cursor) = 0; + static std::unique_ptr create(const vespalib::string &fieldName, const search::attribute::IAttributeVector &attr); + uint32_t size() const { return _len; } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h new file mode 100644 index 00000000000..940cfd6ce06 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::slime { class Inserter; } + +namespace search::docsummary { + +/* + * A subclass of this class can be instantiated by a document field writer to + * track extra state during handling of a document summary request and + * insert the field value using that state. + */ +class DocsumFieldWriterState +{ +public: + virtual void insertField(uint32_t docId, vespalib::slime::Inserter &target) = 0; + virtual ~DocsumFieldWriterState() = default; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp index 7b463352155..18e7e471663 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp @@ -21,6 +21,12 @@ using search::common::Location; const vespalib::string IDocsumFieldWriter::_empty(""); +bool +IDocsumFieldWriter::setFieldWriterStateIndex(uint32_t) +{ + return false; // Don't need any field writer state by default +} + //-------------------------------------------------------------------------- EmptyDFW::EmptyDFW() { } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h index abce5c12227..51079f7736e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h @@ -40,6 +40,7 @@ public: } void setIndex(size_t v) { _index = v; } size_t getIndex() const { return _index; } + virtual bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex); private: size_t _index; static const vespalib::string _empty; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 91953612f6a..b0431b6e6ac 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -4,6 +4,7 @@ #include #include #include +#include "docsum_field_writer_state.h" namespace search { namespace docsummary { @@ -19,6 +20,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _docSumFieldSpace(_docSumFieldSpaceStore, sizeof(_docSumFieldSpaceStore)), // only alloc buffer if needed _attrCtx(), _attributes(), + _fieldWriterStates(), _jsonStringer(), _parsedLocation(), _summaryFeatures(NULL), diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index 4ffed79043e..fa47d5244eb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -23,6 +23,7 @@ namespace search::docsummary { class GetDocsumsState; class IDocsumEnvironment; class KeywordExtractor; +class DocsumFieldWriterState; class GetDocsumsStateCallback { @@ -70,6 +71,7 @@ public: char _docSumFieldSpaceStore[2048]; std::unique_ptr _attrCtx; std::vector _attributes; + std::vector> _fieldWriterStates; vespalib::JSONStringer _jsonStringer; // used by AbsDistanceDFW diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index bf660b1319b..abd1780b773 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -2,6 +2,7 @@ #include "docsumwriter.h" #include "docsumstate.h" +#include "docsum_field_writer_state.h" #include #include #include @@ -77,7 +78,6 @@ DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, uint32_t id) const } } - static void convertEntry(GetDocsumsState *state, const ResConfigEntry *resCfg, const ResEntry *entry, @@ -194,6 +194,7 @@ DynamicDocsumWriter::DynamicDocsumWriter( ResultConfig *config, KeywordExtractor _defaultOutputClass(ResultConfig::NoClassID()), _numClasses(config->GetNumResultClasses()), _numEnumValues(config->GetFieldNameEnum().GetNumEntries()), + _numFieldWriterStates(0), _classInfoTable(nullptr), _overrideTable(nullptr) { @@ -267,6 +268,9 @@ DynamicDocsumWriter::Override(const char *fieldName, IDocsumFieldWriter *writer) writer->setIndex(fieldEnumValue); _overrideTable[fieldEnumValue] = writer; + if (writer->setFieldWriterStateIndex(_numFieldWriterStates)) { + ++_numFieldWriterStates; + } for (ResultConfig::iterator it(_resultConfig->begin()), mt(_resultConfig->end()); it != mt; it++) { @@ -288,6 +292,7 @@ DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *sta state->_kwExtractor = _keywordExtractor; state->_attrCtx = attrMan.createContext(); state->_attributes.resize(_numEnumValues); + state->_fieldWriterStates.resize(_numFieldWriterStates); for (size_t i(0); i < state->_attributes.size(); i++) { const IDocsumFieldWriter *fw = _overrideTable[i]; if (fw) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index 6ef21a71e74..92b26d5cf14 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -54,6 +54,7 @@ private: uint32_t _defaultOutputClass; uint32_t _numClasses; uint32_t _numEnumValues; + uint32_t _numFieldWriterStates; ResultClass::DynamicInfo *_classInfoTable; IDocsumFieldWriter **_overrideTable; -- cgit v1.2.3