Add imported attribute fields to schema.

4 files changed, 91 insertions, 39 deletions

diff --git a/searchcommon/src/tests/schema/.gitignore b/searchcommon/src/tests/schema/.gitignore
index ae6d9596934..e000f0ca2c8 100644
--- a/searchcommon/src/tests/schema/.gitignore
+++ b/searchcommon/src/tests/schema/.gitignore
@@ -2,6 +2,7 @@
 /Makefile
 /schema_test
 searchcommon_schema_test_app
+/schema-no-imported-fields.txt
 /schema-with-timestamps.txt
 /schema-without-timestamps.txt
 /schema.txt
diff --git a/searchcommon/src/tests/schema/schema_test.cpp b/searchcommon/src/tests/schema/schema_test.cpp
index 2c05bdc453e..142a5f25184 100644
--- a/searchcommon/src/tests/schema/schema_test.cpp
+++ b/searchcommon/src/tests/schema/schema_test.cpp
@@ -55,6 +55,12 @@ void assertSchema(const Schema & exp, const Schema & act) {
     for (size_t i = 0; i < exp.getNumFieldSets(); ++i) {
         assertSet(exp.getFieldSet(i), act.getFieldSet(i));
     }
+    const auto &expImported = exp.getImportedAttributeFields();
+    const auto &actImported = act.getImportedAttributeFields();
+    ASSERT_EQUAL(expImported.size(), actImported.size());
+    for (size_t i = 0; i < expImported.size(); ++i) {
+        assertField(expImported[i], actImported[i]);
+    }
 }
 
 TEST("testBasic") {
@@ -62,6 +68,7 @@ TEST("testBasic") {
     EXPECT_EQUAL(0u, s.getNumIndexFields());
     EXPECT_EQUAL(0u, s.getNumAttributeFields());
     EXPECT_EQUAL(0u, s.getNumSummaryFields());
+    EXPECT_EQUAL(0u, s.getNumImportedAttributeFields());
 
     s.addIndexField(Schema::IndexField("foo", schema::STRING));
     s.addIndexField(Schema::IndexField("bar", schema::INT32));
@@ -75,8 +82,9 @@ TEST("testBasic") {
     s.addSummaryField(Schema::SummaryField("cox", schema::STRING));
     s.addSummaryField(Schema::SummaryField("fox", schema::RAW));
 
-    s.addFieldSet(Schema::FieldSet("default").
-                  addField("foo").addField("bar"));
+    s.addFieldSet(Schema::FieldSet("default").addField("foo").addField("bar"));
+
+    s.addImportedAttributeField(Schema::ImportedAttributeField("imported", schema::INT32));
 
     EXPECT_EQUAL(2u, s.getNumIndexFields());
     {
@@ -145,12 +153,21 @@ TEST("testBasic") {
         EXPECT_EQUAL("foo", s.getFieldSet(0).getFields()[0]);
         EXPECT_EQUAL("bar", s.getFieldSet(0).getFields()[1]);
     }
+    EXPECT_EQUAL(1u, s.getNumImportedAttributeFields());
+    {
+        const auto &imported = s.getImportedAttributeFields();
+        EXPECT_EQUAL(1u, imported.size());
+        EXPECT_EQUAL("imported", imported[0].getName());
+        EXPECT_EQUAL(schema::INT32, imported[0].getDataType());
+        EXPECT_EQUAL(schema::SINGLE, imported[0].getCollectionType());
+    }
 }
 
 TEST("testLoadAndSave") {
-    typedef Schema::IndexField SIF;
-    typedef Schema::AttributeField SAF;
-    typedef Schema::SummaryField SSF;
+    using SIF = Schema::IndexField;
+    using SAF = Schema::AttributeField;
+    using SSF = Schema::SummaryField;
+    using SIAF = Schema::ImportedAttributeField;
     using SDT = schema::DataType;
     using SCT = schema::CollectionType;
     typedef Schema::FieldSet SFS;
@@ -203,6 +220,7 @@ TEST("testLoadAndSave") {
         EXPECT_TRUE(s3.loadFromFile("schema.txt"));
         assertSchema(s, s3); // test that saved file is loaded correctly
         s3.addIndexField(SIF("foo", SDT::STRING));
+        s3.addImportedAttributeField(SIAF("imported", schema::INT32));
         EXPECT_TRUE(s3.loadFromFile("schema.txt")); // load should clear the current content
         assertSchema(s, s3);
     }
@@ -211,6 +229,7 @@ TEST("testLoadAndSave") {
         EXPECT_TRUE(s.saveToFile("schema2.txt"));
         Schema s2;
         s2.addIndexField(SIF("foo", SDT::STRING));
+        s2.addImportedAttributeField(SIAF("imported", schema::INT32));
         EXPECT_TRUE(s2.loadFromFile("schema2.txt"));
         assertSchema(s, s2);
     }
@@ -377,6 +396,21 @@ TEST("require that incompatible fields are removed from intersection") {
     EXPECT_FALSE(schema->isIndexField(name));
 }
 
+TEST("require that imported attribute fields are not saved to disk")
+{
+    const vespalib::string fileName = "schema-no-imported-fields.txt";
+    {
+        Schema s;
+        s.addImportedAttributeField(Schema::ImportedAttributeField("imported", schema::INT32));
+        s.saveToFile(fileName);
+    }
+    {
+        Schema s;
+        s.loadFromFile(fileName);
+        EXPECT_EQUAL(0u, s.getNumImportedAttributeFields());
+    }
+}
+
 }  // namespace index
 }  // namespace search
 
diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp
index f024814037d..334fb424e77 100644
--- a/searchcommon/src/vespa/searchcommon/common/schema.cpp
+++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp
@@ -111,7 +111,6 @@ Schema::Field::write(vespalib::asciistream & os, const vespalib::stringref & pre
     }
 }
 
-
 bool
 Schema::Field::operator==(const Field &rhs) const
 {
@@ -121,7 +120,6 @@ Schema::Field::operator==(const Field &rhs) const
       _timestamp == rhs._timestamp;
 }
 
-
 bool
 Schema::Field::operator!=(const Field &rhs) const
 {
@@ -131,7 +129,6 @@ Schema::Field::operator!=(const Field &rhs) const
       _timestamp != rhs._timestamp;
 }
 
-
 Schema::IndexField::IndexField(const vespalib::stringref &name, DataType dt)
     : Field(name, dt),
       _prefix(false),
@@ -160,7 +157,6 @@ Schema::IndexField::IndexField(const std::vector<vespalib::string> &lines)
 {
 }
 
-
 void
 Schema::IndexField::write(vespalib::asciistream & os, const vespalib::stringref & prefix) const
 {
@@ -171,7 +167,6 @@ Schema::IndexField::write(vespalib::asciistream & os, const vespalib::stringref
     os << prefix << "averageelementlen " << static_cast<int32_t>(_avgElemLen) << "\n";
 }
 
-
 bool
 Schema::IndexField::operator==(const IndexField &rhs) const
 {
@@ -182,7 +177,6 @@ Schema::IndexField::operator==(const IndexField &rhs) const
               _avgElemLen == rhs._avgElemLen;
 }
 
-
 bool
 Schema::IndexField::operator!=(const IndexField &rhs) const
 {
@@ -193,7 +187,6 @@ Schema::IndexField::operator!=(const IndexField &rhs) const
               _avgElemLen != rhs._avgElemLen;
 }
 
-
 Schema::FieldSet::FieldSet(const std::vector<vespalib::string> & lines) :
     _name(ConfigParser::parse<vespalib::string>("name", lines)),
     _fields()
@@ -221,12 +214,15 @@ Schema::FieldSet::operator!=(const FieldSet &rhs) const
 }
 
 void
-Schema::writeToStream(vespalib::asciistream &os) const
+Schema::writeToStream(vespalib::asciistream &os, bool saveToDisk) const
 {
     writeFields(os, "attributefield", _attributeFields);
     writeFields(os, "summaryfield", _summaryFields);
     writeFieldSets(os, "fieldset", _fieldSets);
     writeFields(os, "indexfield", _indexFields);
+    if (!saveToDisk) {
+        writeFields(os, "importedattributefields", _importedAttributeFields);
+    }
 }
 
 Schema::Schema()
@@ -234,10 +230,12 @@ Schema::Schema()
       _attributeFields(),
       _summaryFields(),
       _fieldSets(),
+      _importedAttributeFields(),
       _indexIds(),
       _attributeIds(),
       _summaryIds(),
-      _fieldSetIds()
+      _fieldSetIds(),
+      _importedAttributeIds()
 {
 }
 
@@ -261,6 +259,7 @@ Schema::loadFromFile(const vespalib::stringref & fileName)
     _attributeFields = ConfigParser::parseArray<AttributeField>("attributefield", lines);
     _summaryFields = ConfigParser::parseArray<SummaryField>("summaryfield", lines);
     _fieldSets = ConfigParser::parseArray<FieldSet>("fieldset", lines);
+    _importedAttributeFields.clear(); // NOTE: these are not persisted to disk
     _indexIds.clear();
     for (size_t i(0), m(_indexFields.size()); i < m; i++) {
         _indexIds[_indexFields[i].getName()] = i;
@@ -277,6 +276,7 @@ Schema::loadFromFile(const vespalib::stringref & fileName)
     for (size_t i(0), m(_fieldSets.size()); i < m; i++) {
         _fieldSetIds[_fieldSets[i].getName()] = i;
     }
+    _importedAttributeIds.clear();
     return true;
 }
 
@@ -284,7 +284,7 @@ bool
 Schema::saveToFile(const vespalib::stringref & fileName) const
 {
     vespalib::asciistream os;
-    writeToStream(os);
+    writeToStream(os, true);
     std::ofstream file(fileName.c_str());
     if (!file) {
         LOG(warning, "Could not open output file '%s' as part of saveToFile()", fileName.c_str());
@@ -321,7 +321,7 @@ vespalib::string
 Schema::toString() const
 {
     vespalib::asciistream os;
-    writeToStream(os);
+    writeToStream(os, false);
     return os.str();
 }
 
@@ -383,6 +383,12 @@ Schema::addSummaryField(const SummaryField &field)
 }
 
 Schema &
+Schema::addImportedAttributeField(const ImportedAttributeField &field)
+{
+    return addField(field, *this, _importedAttributeFields, _importedAttributeIds);
+}
+
+Schema &
 Schema::addFieldSet(const FieldSet &fieldSet)
 {
     return addField(fieldSet, *this, _fieldSets, _fieldSetIds);
@@ -406,14 +412,12 @@ Schema::getSummaryFieldId(const vespalib::stringref & name) const
     return getFieldId(name, _summaryIds);
 }
 
-
 uint32_t
 Schema::getFieldSetId(const vespalib::stringref &name) const
 {
     return getFieldId(name, _fieldSetIds);
 }
 
-
 void
 Schema::swap(Schema &rhs)
 {
@@ -421,13 +425,14 @@ Schema::swap(Schema &rhs)
     _attributeFields.swap(rhs._attributeFields);
     _summaryFields.swap(rhs._summaryFields);
     _fieldSets.swap(rhs._fieldSets);
+    _importedAttributeFields.swap(rhs._importedAttributeFields);
     _indexIds.swap(rhs._indexIds);
     _attributeIds.swap(rhs._attributeIds);
     _summaryIds.swap(rhs._summaryIds);
     _fieldSetIds.swap(rhs._fieldSetIds);
+    _importedAttributeIds.swap(rhs._importedAttributeIds);
 }
 
-
 void
 Schema::clear()
 {
@@ -435,13 +440,14 @@ Schema::clear()
     _attributeFields.clear();
     _summaryFields.clear();
     _fieldSets.clear();
+    _importedAttributeFields.clear();
     _indexIds.clear();
     _attributeIds.clear();
     _summaryIds.clear();
     _fieldSetIds.clear();
+    _importedAttributeIds.clear();
 }
 
-
 namespace {
 // Helper class allowing the is_matching specialization to access the schema.
 struct IntersectHelper {
@@ -579,31 +585,31 @@ bool
 Schema::operator==(const Schema &rhs) const
 {
     return _indexFields == rhs._indexFields &&
-       _attributeFields == rhs._attributeFields &&
-         _summaryFields == rhs._summaryFields &&
-             _fieldSets == rhs._fieldSets;
+            _attributeFields == rhs._attributeFields &&
+            _summaryFields == rhs._summaryFields &&
+            _fieldSets == rhs._fieldSets &&
+            _importedAttributeFields == rhs._importedAttributeFields;
 }
 
-
 bool
 Schema::operator!=(const Schema &rhs) const
 {
     return _indexFields != rhs._indexFields ||
-       _attributeFields != rhs._attributeFields ||
-         _summaryFields != rhs._summaryFields ||
-             _fieldSets != rhs._fieldSets;
+            _attributeFields != rhs._attributeFields ||
+            _summaryFields != rhs._summaryFields ||
+            _fieldSets != rhs._fieldSets ||
+            _importedAttributeFields != rhs._importedAttributeFields;
 }
 
-
 bool
 Schema::empty() const
 {
     return _indexFields.empty() &&
-        _attributeFields.empty() &&
-        _summaryFields.empty() &&
-        _fieldSets.empty();
+            _attributeFields.empty() &&
+            _summaryFields.empty() &&
+            _fieldSets.empty() &&
+            _importedAttributeFields.empty();
 }
 
-
 } // namespace search::index
 } // namespace search
diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h
index f3ab4e745b4..0870cb43f8c 100644
--- a/searchcommon/src/vespa/searchcommon/common/schema.h
+++ b/searchcommon/src/vespa/searchcommon/common/schema.h
@@ -21,12 +21,11 @@ namespace index {
 class Schema
 {
 public:
-    typedef std::unique_ptr<Schema> UP;
-    typedef std::shared_ptr<Schema> SP;
-    typedef vespalib::PtrHolder<Schema> PH;
+    using UP = std::unique_ptr<Schema>;
+    using SP = std::shared_ptr<Schema>;
+    using PH = vespalib::PtrHolder<Schema>;
 
     using DataType = schema::DataType;
-
     using CollectionType = schema::CollectionType;
 
     /**
@@ -72,6 +71,7 @@ public:
         bool operator!=(const Field &rhs) const;
     };
 
+
     /**
      * A representation of an index field with extra information on
      * how the index should be generated.
@@ -111,8 +111,9 @@ public:
         bool operator!=(const IndexField &rhs) const;
     };
 
-    typedef Field AttributeField;
-    typedef Field SummaryField;
+    using AttributeField = Field;
+    using SummaryField = Field;
+    using ImportedAttributeField = Field;
 
     /**
      * A field collection has a name and a list of index field names,
@@ -153,13 +154,15 @@ private:
     std::vector<AttributeField>  _attributeFields;
     std::vector<SummaryField>    _summaryFields;
     std::vector<FieldSet> _fieldSets;
+    std::vector<ImportedAttributeField> _importedAttributeFields;
     using Name2IdMap = vespalib::hash_map<vespalib::string, uint32_t>;
     Name2IdMap _indexIds;
     Name2IdMap _attributeIds;
     Name2IdMap _summaryIds;
     Name2IdMap _fieldSetIds;
+    Name2IdMap _importedAttributeIds;
 
-    void writeToStream(vespalib::asciistream &os) const;
+    void writeToStream(vespalib::asciistream &os, bool saveToDisk) const;
 
 public:
     /**
@@ -224,6 +227,8 @@ public:
     Schema &
     addFieldSet(const FieldSet &collection);
 
+    Schema &addImportedAttributeField(const ImportedAttributeField &field);
+
     /**
      * Obtain the number of index fields in this schema.
      *
@@ -252,6 +257,8 @@ public:
      **/
     uint32_t getNumFieldSets() const { return _fieldSets.size(); }
 
+    size_t getNumImportedAttributeFields() const { return _importedAttributeFields.size(); }
+
     /**
      * Get information about a specific index field using the given fieldId.
      *
@@ -389,6 +396,10 @@ public:
     uint32_t
     getFieldSetId(const vespalib::stringref &name) const;
 
+    const std::vector<ImportedAttributeField> &getImportedAttributeFields() const {
+        return _importedAttributeFields;
+    }
+
     void swap(Schema &rhs);
     void clear();