Add attribute combiner dynamic field writer.

17 files changed, 734 insertions, 1 deletions

diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt
index 5f6e8881f13..4df636e0219 100644
--- a/searchsummary/CMakeLists.txt
+++ b/searchsummary/CMakeLists.txt
@@ -24,6 +24,7 @@ vespa_define_module(
     TESTS
     src/tests/docsumformat
     src/tests/docsummary
+    src/tests/docsummary/attribute_combiner
     src/tests/docsummary/slime_summary
     src/tests/extractkeywords
 )
diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt
new file mode 100644
index 00000000000..5ebafc71b44
--- /dev/null
+++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchsummary_attribute_combiner_test_app TEST
+    SOURCES
+    attribute_combiner_test.cpp
+    DEPENDS
+    searchsummary
+)
+vespa_add_test(NAME searchsummary_attribute_combiner_test_app COMMAND searchsummary_attribute_combiner_test_app)
diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp
new file mode 100644
index 00000000000..8b2f7357982
--- /dev/null
+++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp
@@ -0,0 +1,220 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchcommon/common/undefinedvalues.h>
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/attributemanager.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/floatbase.h>
+#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/stringbase.h>
+#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
+#include <vespa/searchsummary/docsummary/docsumstate.h>
+#include <vespa/searchsummary/docsummary/docsum_field_writer_state.h>
+#include <vespa/searchsummary/docsummary/attribute_combiner_dfw.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_combiner_test");
+
+using search::AttributeFactory;
+using search::AttributeManager;
+using search::AttributeVector;
+using search::IntegerAttribute;
+using search::FloatingPointAttribute;
+using search::StringAttribute;
+using search::attribute::BasicType;
+using search::attribute::CollectionType;
+using search::attribute::Config;
+using search::attribute::IAttributeVector;
+using search::attribute::getUndefined;
+using search::docsummary::AttributeCombinerDFW;
+using search::docsummary::GetDocsumsState;
+using search::docsummary::GetDocsumsStateCallback;
+using search::docsummary::IDocsumEnvironment;
+using search::docsummary::IDocsumFieldWriter;
+
+namespace {
+
+struct FieldBlock {
+    vespalib::string input;
+    vespalib::Slime slime;
+    search::RawBuf binary;
+    vespalib::string json;
+
+    explicit FieldBlock(const vespalib::string &jsonInput)
+        : input(jsonInput), slime(), binary(1024), json()
+    {
+        size_t used = vespalib::slime::JsonFormat::decode(jsonInput, slime);
+        EXPECT_TRUE(used > 0);
+        {
+            search::SlimeOutputRawBufAdapter adapter(binary);
+            vespalib::slime::JsonFormat::encode(slime, adapter, true);
+            json.assign(binary.GetDrainPos(), binary.GetUsedLen());
+            binary.reset();
+        }
+        search::SlimeOutputRawBufAdapter adapter(binary);
+        vespalib::slime::BinaryFormat::encode(slime, adapter);
+    }
+    const char *data() const { return binary.GetDrainPos(); }
+    size_t dataLen() const { return binary.GetUsedLen(); }
+};
+
+struct AttributeManagerFixture
+{
+    AttributeManager mgr;
+
+    AttributeManagerFixture();
+
+    ~AttributeManagerFixture();
+
+    template <typename AttributeType, typename ValueType>
+    void
+    buildAttribute(const vespalib::string &name,
+                   BasicType type,
+                   std::vector<std::vector<ValueType>> values);
+
+    void
+    buildStringAttribute(const vespalib::string &name,
+                         std::vector<std::vector<vespalib::string>> values);
+    void
+    buildFloatAttribute(const vespalib::string &name,
+                        std::vector<std::vector<double>> values);
+
+    void
+    buildIntegerAttribute(const vespalib::string &name,
+                          BasicType type,
+                          std::vector<std::vector<IAttributeVector::largeint_t>> values);
+};
+
+AttributeManagerFixture::AttributeManagerFixture()
+    : mgr()
+{
+    buildStringAttribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}});
+    buildIntegerAttribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}});
+    buildFloatAttribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }});
+}
+
+AttributeManagerFixture::~AttributeManagerFixture() = default;
+
+template <typename AttributeType, typename ValueType>
+void
+AttributeManagerFixture::buildAttribute(const vespalib::string &name,
+                                        BasicType type,
+                                        std::vector<std::vector<ValueType>> values)
+{
+    Config cfg(type, CollectionType::Type::ARRAY);
+    auto attrBase = AttributeFactory::createAttribute(name, cfg);
+    EXPECT_TRUE(attrBase);
+    auto attr = std::dynamic_pointer_cast<AttributeType>(attrBase);
+    EXPECT_TRUE(attr);
+    attr->addReservedDoc();
+    for (const auto &docValues : values) {
+        uint32_t docId = 0;
+        EXPECT_TRUE(attr->addDoc(docId));
+        EXPECT_NOT_EQUAL(0u, docId);
+        for (const auto &value : docValues) {
+            attr->append(docId, value, 1);
+        }
+        attr->commit();
+    }
+    EXPECT_TRUE(mgr.add(attr));
+}
+
+void
+AttributeManagerFixture::buildStringAttribute(const vespalib::string &name,
+                                              std::vector<std::vector<vespalib::string>> values)
+{
+    buildAttribute<StringAttribute, vespalib::string>(name, BasicType::Type::STRING, std::move(values));
+}
+
+void
+AttributeManagerFixture::buildFloatAttribute(const vespalib::string &name,
+                                             std::vector<std::vector<double>> values)
+{
+    buildAttribute<FloatingPointAttribute, double>(name, BasicType::Type::DOUBLE, std::move(values));
+}
+
+void
+AttributeManagerFixture::buildIntegerAttribute(const vespalib::string &name,
+                                               BasicType type,
+                                               std::vector<std::vector<IAttributeVector::largeint_t>> values)
+{
+    buildAttribute<IntegerAttribute, IAttributeVector::largeint_t>(name, type, std::move(values));
+}
+
+
+class DummyStateCallback : public GetDocsumsStateCallback
+{
+public:
+    void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { }
+    void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { }
+    void ParseLocation(GetDocsumsState *) override { }
+    ~DummyStateCallback() override { }
+};
+
+
+struct Fixture
+{
+    AttributeManagerFixture             attrs;
+    std::unique_ptr<IDocsumFieldWriter> writer;
+    DummyStateCallback                  stateCallback;
+    GetDocsumsState                     state;
+
+    Fixture();
+    ~Fixture();
+    void assertWritten(const vespalib::string &exp, uint32_t docId);
+};
+
+Fixture::Fixture()
+    : attrs(),
+      writer(AttributeCombinerDFW::create("array", attrs.mgr)),
+      stateCallback(),
+      state(stateCallback)
+{
+    EXPECT_TRUE(writer->setFieldWriterStateIndex(0));
+    state._attrCtx = attrs.mgr.createContext();
+    state._fieldWriterStates.resize(1);
+}
+
+Fixture::~Fixture()
+{
+}
+
+void
+Fixture::assertWritten(const vespalib::string &expected, uint32_t docId)
+{
+    vespalib::Slime target;
+    vespalib::slime::SlimeInserter inserter(target);
+    writer->insertField(docId, nullptr, &state, search::docsummary::RES_JSONSTRING, inserter);
+    search::RawBuf binary(1024);
+    vespalib::string json;
+    {
+        search::SlimeOutputRawBufAdapter adapter(binary);
+        vespalib::slime::JsonFormat::encode(target, adapter, true);
+        json.assign(binary.GetDrainPos(), binary.GetUsedLen());
+        binary.reset();
+    }
+    search::SlimeOutputRawBufAdapter adapter(binary);
+    vespalib::slime::BinaryFormat::encode(target, adapter);
+    FieldBlock block(expected);
+    if (!EXPECT_EQUAL(block.dataLen(), binary.GetUsedLen()) ||
+        !EXPECT_EQUAL(0, memcmp(block.data(), binary.GetDrainPos(), block.dataLen()))) {
+        LOG(error, "Expected '%s'", expected.c_str());
+        LOG(error, "Expected normalized '%s'", block.json.c_str());
+        LOG(error, "Got '%s'", json.c_str());
+    }
+}
+
+TEST_F("require that attributes combiner dfw generates correct slime output for array of struct", Fixture())
+{
+    f.assertWritten("[ { fval: 110.0, name: \"n1.1\", val: 10}, { name: \"n1.2\", val: 11}]", 1);
+    f.assertWritten("[ { fval: 120.0, name: \"n2\", val: 20}, { fval: 121.0, val: 21 }]", 2);
+    f.assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30}, { fval: 131.0, name: \"n3.2\"} ]", 3);
+    f.assertWritten("[ { }, { fval: 141.0, name: \"n4.2\", val:  41} ]", 4);
+}
+
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt
index 9009f0bcbc7..ce54e7b0ea7 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt
+++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt
@@ -1,6 +1,9 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(searchsummary_docsummary OBJECT
     SOURCES
+    array_attribute_combiner_dfw.cpp
+    attribute_combiner_dfw.cpp
+    attribute_field_writer.cpp
     resultclass.cpp
     resultconfig.cpp
     resultpacker.cpp
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp
new file mode 100644
index 00000000000..84e329f159d
--- /dev/null
+++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp
@@ -0,0 +1,89 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "array_attribute_combiner_dfw.h"
+#include "docsum_field_writer_state.h"
+#include "attribute_field_writer.h"
+#include <vespa/searchcommon/attribute/iattributecontext.h>
+#include <vespa/searchcommon/attribute/iattributevector.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+
+using search::attribute::IAttributeContext;
+using search::attribute::IAttributeVector;
+using vespalib::slime::Cursor;
+
+namespace search::docsummary {
+
+namespace {
+
+class ArrayAttributeFieldWriterState : public DocsumFieldWriterState
+{
+    std::vector<std::unique_ptr<AttributeFieldWriter>> _writers;
+
+public:
+    ArrayAttributeFieldWriterState(const std::vector<vespalib::string> &fieldNames,
+                                   const std::vector<vespalib::string> &attributeNames,
+                                   IAttributeContext &context);
+    ~ArrayAttributeFieldWriterState() override;
+    void insertField(uint32_t docId, vespalib::slime::Inserter &target) override;
+};
+
+ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector<vespalib::string> &fieldNames,
+                                                               const std::vector<vespalib::string> &attributeNames,
+                                                               IAttributeContext &context)
+    : DocsumFieldWriterState()
+{
+    size_t fields = fieldNames.size();
+    _writers.reserve(fields);
+    for (uint32_t field = 0; field < fields; ++field) {
+        const IAttributeVector *attr = context.getAttribute(attributeNames[field]);
+        if (attr != nullptr) {
+            _writers.emplace_back(AttributeFieldWriter::create(fieldNames[field], *attr));
+        }
+    }
+}
+
+ArrayAttributeFieldWriterState::~ArrayAttributeFieldWriterState() = default;
+
+void
+ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Inserter &target)
+{
+    uint32_t elems = 0;
+    for (auto &writer : _writers) {
+        writer->fetch(docId);
+        if (elems < writer->size()) {
+            elems = writer->size();
+        }
+    }
+    Cursor &arr = target.insertArray();
+    for (uint32_t idx = 0; idx < elems; ++idx) {
+        Cursor &obj = arr.addObject();
+        for (auto &writer : _writers) {
+            writer->print(idx, obj);
+        }
+    }
+}
+
+}
+
+ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fieldName,
+                                                     const std::vector<vespalib::string> &fields)
+    : AttributeCombinerDFW(fieldName),
+      _fields(fields),
+      _attributeNames()
+{
+    _attributeNames.reserve(_fields.size());
+    vespalib::string prefix = fieldName + ".";
+    for (const auto &field : _fields) {
+        _attributeNames.emplace_back(prefix + field);
+    }
+}
+
+ArrayAttributeCombinerDFW::~ArrayAttributeCombinerDFW() = default;
+
+std::unique_ptr<DocsumFieldWriterState>
+ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context)
+{
+    return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context);
+}
+
+}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h
new file mode 100644
index 00000000000..c02d2bd5da6
--- /dev/null
+++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h
@@ -0,0 +1,29 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "attribute_combiner_dfw.h"
+
+namespace search::attribute { class IAttributeContext; }
+
+namespace search::docsummary {
+
+class DocsumFieldWriterState;
+
+/*
+ * This class reads values from multiple struct field attributes and
+ * inserts them as an array of struct.
+ */
+class ArrayAttributeCombinerDFW : public AttributeCombinerDFW
+{
+    std::vector<vespalib::string> _fields;
+    std::vector<vespalib::string> _attributeNames;
+
+    std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) override;
+public:
+    ArrayAttributeCombinerDFW(const vespalib::string &fieldName,
+                              const std::vector<vespalib::string> &fields);
+    ~ArrayAttributeCombinerDFW() override;
+};
+
+}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp
new file mode 100644
index 00000000000..9383154979f
--- /dev/null
+++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp
@@ -0,0 +1,105 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "attribute_combiner_dfw.h"
+#include "array_attribute_combiner_dfw.h"
+#include "docsum_field_writer_state.h"
+#include "docsumstate.h"
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+#include <algorithm>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".searchsummary.docsummary.attribute_combiner_dfw");
+
+using search::AttributeGuard;
+using search::AttributeVector;
+using search::attribute::CollectionType;
+
+namespace search::docsummary {
+
+AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName)
+    : IDocsumFieldWriter(),
+      _stateIndex(0),
+      _fieldName(fieldName)
+{
+}
+
+AttributeCombinerDFW::~AttributeCombinerDFW() = default;
+
+bool
+AttributeCombinerDFW::IsGenerated() const
+{
+    return true;
+}
+
+bool
+AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex)
+{
+    _stateIndex = fieldWriterStateIndex;
+    return true;
+}
+
+std::unique_ptr<IDocsumFieldWriter>
+AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManager &attrMgr)
+{
+    // Note: Doesn't handle imported attributes
+    std::vector<AttributeGuard> attrs;
+    attrMgr.getAttributeList(attrs);
+    vespalib::string prefix = fieldName + ".";
+    vespalib::string keyName = prefix + "key";
+    vespalib::string valuePrefix = prefix + "value.";
+    std::vector<vespalib::string> mapFields;
+    std::vector<vespalib::string> arrayFields;
+    bool foundKey = false;
+    for (const auto &guard : attrs) {
+        vespalib::string name = guard->getName();
+        if (name.substr(0, prefix.size()) != prefix) {
+            continue;
+        }
+        auto collType = guard->getCollectionType();
+        if (collType != CollectionType::Type::ARRAY) {
+            LOG(warning, "Attribute %s is not an array attribute", name.c_str());
+            return std::unique_ptr<IDocsumFieldWriter>();
+        }
+        if (name.substr(0, valuePrefix.size()) == valuePrefix) {
+            mapFields.emplace_back(name.substr(valuePrefix.size()));
+        } else {
+            arrayFields.emplace_back(name.substr(prefix.size()));
+            if (name == keyName) {
+                foundKey = true;
+            }
+        }
+    }
+    if (!mapFields.empty()) {
+        if (!foundKey) {
+            LOG(warning, "Missing key attribute '%s', have value attributes for map", keyName.c_str());
+            return std::unique_ptr<IDocsumFieldWriter>();
+        }
+        if (arrayFields.size() != 1u) {
+            LOG(warning, "Could not determine if field '%s' is array or map of struct", fieldName.c_str());
+            return std::unique_ptr<IDocsumFieldWriter>();
+        }
+        LOG(warning, "map of struct is not yet supported for field '%s'", fieldName.c_str());
+        return std::unique_ptr<IDocsumFieldWriter>();
+    }
+    std::sort(arrayFields.begin(), arrayFields.end());
+    return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, arrayFields);
+}
+
+void
+AttributeCombinerDFW::insertField(uint32_t docid,
+                                  GeneralResult *,
+                                  GetDocsumsState *state,
+                                  ResType,
+                                  vespalib::slime::Inserter &target)
+{
+    auto &fieldWriterState = state->_fieldWriterStates[_stateIndex];
+    if (!fieldWriterState) {
+        fieldWriterState = allocFieldWriterState(*state->_attrCtx);
+    }
+    fieldWriterState->insertField(docid, target);
+}
+
+}
+
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h
new file mode 100644
index 00000000000..ef54522a923
--- /dev/null
+++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h
@@ -0,0 +1,36 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "docsumfieldwriter.h"
+
+namespace search::attribute { class IAttributeContext; }
+
+namespace search::docsummary {
+
+class DocsumFieldWriterState;
+class DynamicDocsumWriter;
+
+/*
+ * This class reads values from multiple struct field attributes and
+ * inserts them as an array of struct or a map of struct.
+ */
+class AttributeCombinerDFW : public IDocsumFieldWriter
+{
+protected:
+    uint32_t _stateIndex;
+    vespalib::string _fieldName;
+    AttributeCombinerDFW(const vespalib::string &fieldName);
+protected:
+    virtual std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) = 0;
+public:
+    ~AttributeCombinerDFW() override;
+    bool IsGenerated() const override;
+    bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override;
+    static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, IAttributeManager &attrMgr);
+    void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state,
+                     ResType type, vespalib::slime::Inserter &target) override;
+};
+
+}
+
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp
new file mode 100644
index 00000000000..66940f37a13
--- /dev/null
+++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp
@@ -0,0 +1,170 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "attribute_field_writer.h"
+#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/searchcommon/common/undefinedvalues.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+
+using search::attribute::BasicType;
+using search::attribute::IAttributeVector;
+using search::attribute::getUndefined;
+using vespalib::slime::Cursor;
+
+namespace search::docsummary {
+
+AttributeFieldWriter::AttributeFieldWriter(const vespalib::string &fieldName,
+                                           const IAttributeVector &attr)
+    : _fieldName(fieldName),
+      _attr(attr),
+      _len(0)
+{
+}
+
+AttributeFieldWriter::~AttributeFieldWriter() = default;
+
+namespace {
+
+template <class Content>
+class WriteField : public AttributeFieldWriter
+{
+protected:
+    Content _content;
+
+    WriteField(const vespalib::string &fieldName, const IAttributeVector &attr);
+    ~WriteField() override;
+private:
+    void fetch(uint32_t docId) override;
+};
+
+class WriteStringField : public WriteField<search::attribute::ConstCharContent>
+{
+public:
+    WriteStringField(const vespalib::string &fieldName,
+                     const IAttributeVector &attr);
+    ~WriteStringField() override;
+    void print(uint32_t idx, Cursor &cursor) override;
+};
+
+
+class WriteFloatField : public WriteField<search::attribute::FloatContent>
+{
+public:
+    WriteFloatField(const vespalib::string &fieldName,
+                    const IAttributeVector &attr);
+    ~WriteFloatField() override;
+    void print(uint32_t idx, Cursor &cursor) override;
+};
+
+class WriteIntField : public WriteField<search::attribute::IntegerContent>
+{
+    IAttributeVector::largeint_t _undefined;
+public:
+    WriteIntField(const vespalib::string &fieldName,
+                  const IAttributeVector &attr,
+                  IAttributeVector::largeint_t undefined);
+    ~WriteIntField() override;
+    void print(uint32_t idx, Cursor &cursor) override;
+};
+
+template <class Content>
+WriteField<Content>::WriteField(const vespalib::string &fieldName, const IAttributeVector &attr)
+    : AttributeFieldWriter(fieldName, attr),
+      _content()
+{
+}
+
+template <class Content>
+WriteField<Content>::~WriteField() = default;
+
+template <class Content>
+void
+WriteField<Content>::fetch(uint32_t docId)
+{
+    _content.fill(_attr, docId);
+    _len = _content.size();
+}
+
+WriteStringField::WriteStringField(const vespalib::string &fieldName,
+                                   const IAttributeVector &attr)
+    : WriteField(fieldName, attr)
+{
+}
+
+WriteStringField::~WriteStringField() = default;
+
+void
+WriteStringField::print(uint32_t idx, Cursor &cursor)
+{
+    if (idx < _len) {
+        const char *s = _content[idx];
+        if (s[0] != '\0') {
+            cursor.setString(_fieldName, vespalib::Memory(s));
+        }
+    }
+}
+
+WriteFloatField::WriteFloatField(const vespalib::string &fieldName,
+                                 const IAttributeVector &attr)
+    : WriteField(fieldName, attr)
+{
+}
+
+WriteFloatField::~WriteFloatField() = default;
+
+void
+WriteFloatField::print(uint32_t idx, Cursor &cursor)
+{
+    if (idx < _len) {
+        double val = _content[idx];
+        if (!search::attribute::isUndefined(val)) {
+            cursor.setDouble(_fieldName, val);
+        }
+    }
+}
+
+WriteIntField::WriteIntField(const vespalib::string &fieldName,
+                             const IAttributeVector &attr,
+                             IAttributeVector::largeint_t undefined)
+    : WriteField(fieldName, attr),
+      _undefined(undefined)
+{
+}
+
+WriteIntField::~WriteIntField() = default;
+
+void
+WriteIntField::print(uint32_t idx, Cursor &cursor)
+{
+    if (idx < _len) {
+        auto val = _content[idx];
+        if (val != _undefined) {
+            cursor.setLong(_fieldName, _content[idx]);
+        }
+    }
+}
+
+}
+
+std::unique_ptr<AttributeFieldWriter>
+AttributeFieldWriter::create(const vespalib::string &fieldName, const IAttributeVector &attr)
+{
+    switch (attr.getBasicType()) {
+    case BasicType::INT8:
+        return std::make_unique<WriteIntField>(fieldName, attr, getUndefined<int8_t>());
+    case BasicType::INT16:
+        return std::make_unique<WriteIntField>(fieldName, attr, getUndefined<int16_t>());
+    case BasicType::INT32:
+        return std::make_unique<WriteIntField>(fieldName, attr, getUndefined<int32_t>());
+    case BasicType::INT64:
+        return std::make_unique<WriteIntField>(fieldName, attr, getUndefined<int64_t>());
+    case BasicType::FLOAT:
+    case BasicType::DOUBLE:
+        return std::make_unique<WriteFloatField>(fieldName, attr);
+    case BasicType::STRING:
+        return std::make_unique<WriteStringField>(fieldName, attr);
+    default:
+        abort();
+    }
+}
+
+}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h
new file mode 100644
index 00000000000..03eda4b067a
--- /dev/null
+++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h
@@ -0,0 +1,34 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/data/memory.h>
+
+namespace search::attribute { class IAttributeVector; }
+namespace vespalib::slime { class Cursor; }
+
+namespace search::docsummary {
+
+/*
+ * This class reads values from a struct field attribute and inserts
+ * them into proper position in an array of struct or map of struct.
+ * If the value to be inserted is considered to be undefined then
+ * the value is not inserted.
+ */
+class AttributeFieldWriter
+{
+protected:
+    const vespalib::Memory                     _fieldName;
+    const search::attribute::IAttributeVector &_attr;
+    size_t                                     _len;
+public:
+    AttributeFieldWriter(const vespalib::string &fieldName,
+                         const search::attribute::IAttributeVector &attr);
+    virtual ~AttributeFieldWriter();
+    virtual void fetch(uint32_t docId) = 0;
+    virtual void print(uint32_t idx, vespalib::slime::Cursor &cursor) = 0;
+    static std::unique_ptr<AttributeFieldWriter> create(const vespalib::string &fieldName, const search::attribute::IAttributeVector &attr);
+    uint32_t size() const { return _len; }
+};
+
+}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h
new file mode 100644
index 00000000000..940cfd6ce06
--- /dev/null
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_state.h
@@ -0,0 +1,21 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace vespalib::slime { class Inserter; }
+
+namespace search::docsummary {
+
+/*
+ * A subclass of this class can be instantiated by a document field writer to
+ * track extra state during handling of a document summary request and
+ * insert the field value using that state.
+ */
+class DocsumFieldWriterState
+{
+public:
+    virtual void insertField(uint32_t docId, vespalib::slime::Inserter &target) = 0;
+    virtual ~DocsumFieldWriterState() = default;
+};
+
+}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
index 7b463352155..18e7e471663 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
@@ -21,6 +21,12 @@ using search::common::Location;
 
 const vespalib::string IDocsumFieldWriter::_empty("");
 
+bool
+IDocsumFieldWriter::setFieldWriterStateIndex(uint32_t)
+{
+    return false; // Don't need any field writer state by default
+}
+
 //--------------------------------------------------------------------------
 
 EmptyDFW::EmptyDFW() { }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h
index abce5c12227..51079f7736e 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h
@@ -40,6 +40,7 @@ public:
     }
     void setIndex(size_t v) { _index = v; }
     size_t getIndex() const { return _index; }
+    virtual bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex);
 private:
     size_t _index;
     static const vespalib::string _empty;
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp
index 91953612f6a..b0431b6e6ac 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp
@@ -4,6 +4,7 @@
 #include <vespa/juniper/rpinterface.h>
 #include <vespa/searchcommon/attribute/iattributecontext.h>
 #include <vespa/searchlib/common/location.h>
+#include "docsum_field_writer_state.h"
 
 namespace search {
 namespace docsummary {
@@ -19,6 +20,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback)
       _docSumFieldSpace(_docSumFieldSpaceStore, sizeof(_docSumFieldSpaceStore)), // only alloc buffer if needed
       _attrCtx(),
       _attributes(),
+      _fieldWriterStates(),
       _jsonStringer(),
       _parsedLocation(),
       _summaryFeatures(NULL),
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h
index 4ffed79043e..fa47d5244eb 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h
@@ -23,6 +23,7 @@ namespace search::docsummary {
 class GetDocsumsState;
 class IDocsumEnvironment;
 class KeywordExtractor;
+class DocsumFieldWriterState;
 
 class GetDocsumsStateCallback
 {
@@ -70,6 +71,7 @@ public:
     char                         _docSumFieldSpaceStore[2048];
     std::unique_ptr<search::attribute::IAttributeContext> _attrCtx;
     std::vector<const search::attribute::IAttributeVector *> _attributes;
+    std::vector<std::unique_ptr<DocsumFieldWriterState>> _fieldWriterStates;
     vespalib::JSONStringer        _jsonStringer;
 
     // used by AbsDistanceDFW
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
index bf660b1319b..abd1780b773 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
@@ -2,6 +2,7 @@
 
 #include "docsumwriter.h"
 #include "docsumstate.h"
+#include "docsum_field_writer_state.h"
 #include <vespa/searchlib/common/transport.h>
 #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
 #include <vespa/searchlib/attribute/iattributemanager.h>
@@ -77,7 +78,6 @@ DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, uint32_t id) const
     }
 }
 
-
 static void convertEntry(GetDocsumsState *state,
                          const ResConfigEntry *resCfg,
                          const ResEntry *entry,
@@ -194,6 +194,7 @@ DynamicDocsumWriter::DynamicDocsumWriter( ResultConfig *config, KeywordExtractor
       _defaultOutputClass(ResultConfig::NoClassID()),
       _numClasses(config->GetNumResultClasses()),
       _numEnumValues(config->GetFieldNameEnum().GetNumEntries()),
+      _numFieldWriterStates(0),
       _classInfoTable(nullptr),
       _overrideTable(nullptr)
 {
@@ -267,6 +268,9 @@ DynamicDocsumWriter::Override(const char *fieldName, IDocsumFieldWriter *writer)
 
     writer->setIndex(fieldEnumValue);
     _overrideTable[fieldEnumValue] = writer;
+    if (writer->setFieldWriterStateIndex(_numFieldWriterStates)) {
+        ++_numFieldWriterStates;
+    }
 
     for (ResultConfig::iterator it(_resultConfig->begin()), mt(_resultConfig->end()); it != mt; it++) {
 
@@ -288,6 +292,7 @@ DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *sta
     state->_kwExtractor = _keywordExtractor;
     state->_attrCtx = attrMan.createContext();
     state->_attributes.resize(_numEnumValues);
+    state->_fieldWriterStates.resize(_numFieldWriterStates);
     for (size_t i(0); i < state->_attributes.size(); i++) {
         const IDocsumFieldWriter *fw = _overrideTable[i];
         if (fw) {
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h
index 6ef21a71e74..92b26d5cf14 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h
@@ -54,6 +54,7 @@ private:
     uint32_t             _defaultOutputClass;
     uint32_t             _numClasses;
     uint32_t             _numEnumValues;
+    uint32_t             _numFieldWriterStates;
     ResultClass::DynamicInfo *_classInfoTable;
     IDocsumFieldWriter **_overrideTable;