diff options
-rw-r--r-- | searchcommon/src/tests/schema/.gitignore | 1 | ||||
-rw-r--r-- | searchcommon/src/tests/schema/schema_test.cpp | 44 | ||||
-rw-r--r-- | searchcommon/src/vespa/searchcommon/common/schema.cpp | 60 | ||||
-rw-r--r-- | searchcommon/src/vespa/searchcommon/common/schema.h | 25 |
4 files changed, 91 insertions, 39 deletions
diff --git a/searchcommon/src/tests/schema/.gitignore b/searchcommon/src/tests/schema/.gitignore index ae6d9596934..e000f0ca2c8 100644 --- a/searchcommon/src/tests/schema/.gitignore +++ b/searchcommon/src/tests/schema/.gitignore @@ -2,6 +2,7 @@ /Makefile /schema_test searchcommon_schema_test_app +/schema-no-imported-fields.txt /schema-with-timestamps.txt /schema-without-timestamps.txt /schema.txt diff --git a/searchcommon/src/tests/schema/schema_test.cpp b/searchcommon/src/tests/schema/schema_test.cpp index 2c05bdc453e..142a5f25184 100644 --- a/searchcommon/src/tests/schema/schema_test.cpp +++ b/searchcommon/src/tests/schema/schema_test.cpp @@ -55,6 +55,12 @@ void assertSchema(const Schema & exp, const Schema & act) { for (size_t i = 0; i < exp.getNumFieldSets(); ++i) { assertSet(exp.getFieldSet(i), act.getFieldSet(i)); } + const auto &expImported = exp.getImportedAttributeFields(); + const auto &actImported = act.getImportedAttributeFields(); + ASSERT_EQUAL(expImported.size(), actImported.size()); + for (size_t i = 0; i < expImported.size(); ++i) { + assertField(expImported[i], actImported[i]); + } } TEST("testBasic") { @@ -62,6 +68,7 @@ TEST("testBasic") { EXPECT_EQUAL(0u, s.getNumIndexFields()); EXPECT_EQUAL(0u, s.getNumAttributeFields()); EXPECT_EQUAL(0u, s.getNumSummaryFields()); + EXPECT_EQUAL(0u, s.getNumImportedAttributeFields()); s.addIndexField(Schema::IndexField("foo", schema::STRING)); s.addIndexField(Schema::IndexField("bar", schema::INT32)); @@ -75,8 +82,9 @@ TEST("testBasic") { s.addSummaryField(Schema::SummaryField("cox", schema::STRING)); s.addSummaryField(Schema::SummaryField("fox", schema::RAW)); - s.addFieldSet(Schema::FieldSet("default"). - addField("foo").addField("bar")); + s.addFieldSet(Schema::FieldSet("default").addField("foo").addField("bar")); + + s.addImportedAttributeField(Schema::ImportedAttributeField("imported", schema::INT32)); EXPECT_EQUAL(2u, s.getNumIndexFields()); { @@ -145,12 +153,21 @@ TEST("testBasic") { EXPECT_EQUAL("foo", s.getFieldSet(0).getFields()[0]); EXPECT_EQUAL("bar", s.getFieldSet(0).getFields()[1]); } + EXPECT_EQUAL(1u, s.getNumImportedAttributeFields()); + { + const auto &imported = s.getImportedAttributeFields(); + EXPECT_EQUAL(1u, imported.size()); + EXPECT_EQUAL("imported", imported[0].getName()); + EXPECT_EQUAL(schema::INT32, imported[0].getDataType()); + EXPECT_EQUAL(schema::SINGLE, imported[0].getCollectionType()); + } } TEST("testLoadAndSave") { - typedef Schema::IndexField SIF; - typedef Schema::AttributeField SAF; - typedef Schema::SummaryField SSF; + using SIF = Schema::IndexField; + using SAF = Schema::AttributeField; + using SSF = Schema::SummaryField; + using SIAF = Schema::ImportedAttributeField; using SDT = schema::DataType; using SCT = schema::CollectionType; typedef Schema::FieldSet SFS; @@ -203,6 +220,7 @@ TEST("testLoadAndSave") { EXPECT_TRUE(s3.loadFromFile("schema.txt")); assertSchema(s, s3); // test that saved file is loaded correctly s3.addIndexField(SIF("foo", SDT::STRING)); + s3.addImportedAttributeField(SIAF("imported", schema::INT32)); EXPECT_TRUE(s3.loadFromFile("schema.txt")); // load should clear the current content assertSchema(s, s3); } @@ -211,6 +229,7 @@ TEST("testLoadAndSave") { EXPECT_TRUE(s.saveToFile("schema2.txt")); Schema s2; s2.addIndexField(SIF("foo", SDT::STRING)); + s2.addImportedAttributeField(SIAF("imported", schema::INT32)); EXPECT_TRUE(s2.loadFromFile("schema2.txt")); assertSchema(s, s2); } @@ -377,6 +396,21 @@ TEST("require that incompatible fields are removed from intersection") { EXPECT_FALSE(schema->isIndexField(name)); } +TEST("require that imported attribute fields are not saved to disk") +{ + const vespalib::string fileName = "schema-no-imported-fields.txt"; + { + Schema s; + s.addImportedAttributeField(Schema::ImportedAttributeField("imported", schema::INT32)); + s.saveToFile(fileName); + } + { + Schema s; + s.loadFromFile(fileName); + EXPECT_EQUAL(0u, s.getNumImportedAttributeFields()); + } +} + } // namespace index } // namespace search diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp index f024814037d..334fb424e77 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp @@ -111,7 +111,6 @@ Schema::Field::write(vespalib::asciistream & os, const vespalib::stringref & pre } } - bool Schema::Field::operator==(const Field &rhs) const { @@ -121,7 +120,6 @@ Schema::Field::operator==(const Field &rhs) const _timestamp == rhs._timestamp; } - bool Schema::Field::operator!=(const Field &rhs) const { @@ -131,7 +129,6 @@ Schema::Field::operator!=(const Field &rhs) const _timestamp != rhs._timestamp; } - Schema::IndexField::IndexField(const vespalib::stringref &name, DataType dt) : Field(name, dt), _prefix(false), @@ -160,7 +157,6 @@ Schema::IndexField::IndexField(const std::vector<vespalib::string> &lines) { } - void Schema::IndexField::write(vespalib::asciistream & os, const vespalib::stringref & prefix) const { @@ -171,7 +167,6 @@ Schema::IndexField::write(vespalib::asciistream & os, const vespalib::stringref os << prefix << "averageelementlen " << static_cast<int32_t>(_avgElemLen) << "\n"; } - bool Schema::IndexField::operator==(const IndexField &rhs) const { @@ -182,7 +177,6 @@ Schema::IndexField::operator==(const IndexField &rhs) const _avgElemLen == rhs._avgElemLen; } - bool Schema::IndexField::operator!=(const IndexField &rhs) const { @@ -193,7 +187,6 @@ Schema::IndexField::operator!=(const IndexField &rhs) const _avgElemLen != rhs._avgElemLen; } - Schema::FieldSet::FieldSet(const std::vector<vespalib::string> & lines) : _name(ConfigParser::parse<vespalib::string>("name", lines)), _fields() @@ -221,12 +214,15 @@ Schema::FieldSet::operator!=(const FieldSet &rhs) const } void -Schema::writeToStream(vespalib::asciistream &os) const +Schema::writeToStream(vespalib::asciistream &os, bool saveToDisk) const { writeFields(os, "attributefield", _attributeFields); writeFields(os, "summaryfield", _summaryFields); writeFieldSets(os, "fieldset", _fieldSets); writeFields(os, "indexfield", _indexFields); + if (!saveToDisk) { + writeFields(os, "importedattributefields", _importedAttributeFields); + } } Schema::Schema() @@ -234,10 +230,12 @@ Schema::Schema() _attributeFields(), _summaryFields(), _fieldSets(), + _importedAttributeFields(), _indexIds(), _attributeIds(), _summaryIds(), - _fieldSetIds() + _fieldSetIds(), + _importedAttributeIds() { } @@ -261,6 +259,7 @@ Schema::loadFromFile(const vespalib::stringref & fileName) _attributeFields = ConfigParser::parseArray<AttributeField>("attributefield", lines); _summaryFields = ConfigParser::parseArray<SummaryField>("summaryfield", lines); _fieldSets = ConfigParser::parseArray<FieldSet>("fieldset", lines); + _importedAttributeFields.clear(); // NOTE: these are not persisted to disk _indexIds.clear(); for (size_t i(0), m(_indexFields.size()); i < m; i++) { _indexIds[_indexFields[i].getName()] = i; @@ -277,6 +276,7 @@ Schema::loadFromFile(const vespalib::stringref & fileName) for (size_t i(0), m(_fieldSets.size()); i < m; i++) { _fieldSetIds[_fieldSets[i].getName()] = i; } + _importedAttributeIds.clear(); return true; } @@ -284,7 +284,7 @@ bool Schema::saveToFile(const vespalib::stringref & fileName) const { vespalib::asciistream os; - writeToStream(os); + writeToStream(os, true); std::ofstream file(fileName.c_str()); if (!file) { LOG(warning, "Could not open output file '%s' as part of saveToFile()", fileName.c_str()); @@ -321,7 +321,7 @@ vespalib::string Schema::toString() const { vespalib::asciistream os; - writeToStream(os); + writeToStream(os, false); return os.str(); } @@ -383,6 +383,12 @@ Schema::addSummaryField(const SummaryField &field) } Schema & +Schema::addImportedAttributeField(const ImportedAttributeField &field) +{ + return addField(field, *this, _importedAttributeFields, _importedAttributeIds); +} + +Schema & Schema::addFieldSet(const FieldSet &fieldSet) { return addField(fieldSet, *this, _fieldSets, _fieldSetIds); @@ -406,14 +412,12 @@ Schema::getSummaryFieldId(const vespalib::stringref & name) const return getFieldId(name, _summaryIds); } - uint32_t Schema::getFieldSetId(const vespalib::stringref &name) const { return getFieldId(name, _fieldSetIds); } - void Schema::swap(Schema &rhs) { @@ -421,13 +425,14 @@ Schema::swap(Schema &rhs) _attributeFields.swap(rhs._attributeFields); _summaryFields.swap(rhs._summaryFields); _fieldSets.swap(rhs._fieldSets); + _importedAttributeFields.swap(rhs._importedAttributeFields); _indexIds.swap(rhs._indexIds); _attributeIds.swap(rhs._attributeIds); _summaryIds.swap(rhs._summaryIds); _fieldSetIds.swap(rhs._fieldSetIds); + _importedAttributeIds.swap(rhs._importedAttributeIds); } - void Schema::clear() { @@ -435,13 +440,14 @@ Schema::clear() _attributeFields.clear(); _summaryFields.clear(); _fieldSets.clear(); + _importedAttributeFields.clear(); _indexIds.clear(); _attributeIds.clear(); _summaryIds.clear(); _fieldSetIds.clear(); + _importedAttributeIds.clear(); } - namespace { // Helper class allowing the is_matching specialization to access the schema. struct IntersectHelper { @@ -579,31 +585,31 @@ bool Schema::operator==(const Schema &rhs) const { return _indexFields == rhs._indexFields && - _attributeFields == rhs._attributeFields && - _summaryFields == rhs._summaryFields && - _fieldSets == rhs._fieldSets; + _attributeFields == rhs._attributeFields && + _summaryFields == rhs._summaryFields && + _fieldSets == rhs._fieldSets && + _importedAttributeFields == rhs._importedAttributeFields; } - bool Schema::operator!=(const Schema &rhs) const { return _indexFields != rhs._indexFields || - _attributeFields != rhs._attributeFields || - _summaryFields != rhs._summaryFields || - _fieldSets != rhs._fieldSets; + _attributeFields != rhs._attributeFields || + _summaryFields != rhs._summaryFields || + _fieldSets != rhs._fieldSets || + _importedAttributeFields != rhs._importedAttributeFields; } - bool Schema::empty() const { return _indexFields.empty() && - _attributeFields.empty() && - _summaryFields.empty() && - _fieldSets.empty(); + _attributeFields.empty() && + _summaryFields.empty() && + _fieldSets.empty() && + _importedAttributeFields.empty(); } - } // namespace search::index } // namespace search diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h index f3ab4e745b4..0870cb43f8c 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.h +++ b/searchcommon/src/vespa/searchcommon/common/schema.h @@ -21,12 +21,11 @@ namespace index { class Schema { public: - typedef std::unique_ptr<Schema> UP; - typedef std::shared_ptr<Schema> SP; - typedef vespalib::PtrHolder<Schema> PH; + using UP = std::unique_ptr<Schema>; + using SP = std::shared_ptr<Schema>; + using PH = vespalib::PtrHolder<Schema>; using DataType = schema::DataType; - using CollectionType = schema::CollectionType; /** @@ -72,6 +71,7 @@ public: bool operator!=(const Field &rhs) const; }; + /** * A representation of an index field with extra information on * how the index should be generated. @@ -111,8 +111,9 @@ public: bool operator!=(const IndexField &rhs) const; }; - typedef Field AttributeField; - typedef Field SummaryField; + using AttributeField = Field; + using SummaryField = Field; + using ImportedAttributeField = Field; /** * A field collection has a name and a list of index field names, @@ -153,13 +154,15 @@ private: std::vector<AttributeField> _attributeFields; std::vector<SummaryField> _summaryFields; std::vector<FieldSet> _fieldSets; + std::vector<ImportedAttributeField> _importedAttributeFields; using Name2IdMap = vespalib::hash_map<vespalib::string, uint32_t>; Name2IdMap _indexIds; Name2IdMap _attributeIds; Name2IdMap _summaryIds; Name2IdMap _fieldSetIds; + Name2IdMap _importedAttributeIds; - void writeToStream(vespalib::asciistream &os) const; + void writeToStream(vespalib::asciistream &os, bool saveToDisk) const; public: /** @@ -224,6 +227,8 @@ public: Schema & addFieldSet(const FieldSet &collection); + Schema &addImportedAttributeField(const ImportedAttributeField &field); + /** * Obtain the number of index fields in this schema. * @@ -252,6 +257,8 @@ public: **/ uint32_t getNumFieldSets() const { return _fieldSets.size(); } + size_t getNumImportedAttributeFields() const { return _importedAttributeFields.size(); } + /** * Get information about a specific index field using the given fieldId. * @@ -389,6 +396,10 @@ public: uint32_t getFieldSetId(const vespalib::stringref &name) const; + const std::vector<ImportedAttributeField> &getImportedAttributeFields() const { + return _importedAttributeFields; + } + void swap(Schema &rhs); void clear(); |