diff options
Diffstat (limited to 'searchlib')
6 files changed, 46 insertions, 228 deletions
diff --git a/searchlib/src/tests/searchcommon/schema/old-schema-with-summary-fields.txt b/searchlib/src/tests/searchcommon/schema/old-schema-with-summary-fields.txt new file mode 100644 index 00000000000..eba1170cf48 --- /dev/null +++ b/searchlib/src/tests/searchcommon/schema/old-schema-with-summary-fields.txt @@ -0,0 +1,32 @@ +attributefield[2] +attributefield[0].name afoo +attributefield[0].datatype STRING +attributefield[0].collectiontype SINGLE +attributefield[1].name abar +attributefield[1].datatype INT32 +attributefield[1].collectiontype SINGLE +summaryfield[2] +summaryfield[0].name sfoo +summaryfield[0].datatype STRING +summaryfield[0].collectiontype SINGLE +summaryfield[1].name sbar +summaryfield[1].datatype INT32 +summaryfield[1].collectiontype SINGLE +fieldset[0] +indexfield[2] +indexfield[0].name ifoo +indexfield[0].datatype STRING +indexfield[0].collectiontype SINGLE +indexfield[0].averageelementlen 512 +indexfield[0].interleavedfeatures false +indexfield[0].prefix false +indexfield[0].phrases false +indexfield[0].positions true +indexfield[1].name ibar +indexfield[1].datatype INT32 +indexfield[1].collectiontype SINGLE +indexfield[1].averageelementlen 512 +indexfield[1].interleavedfeatures false +indexfield[1].prefix false +indexfield[1].phrases false +indexfield[1].positions true diff --git a/searchlib/src/tests/searchcommon/schema/schema_test.cpp b/searchlib/src/tests/searchcommon/schema/schema_test.cpp index 09a7359bac7..a2f16b661c6 100644 --- a/searchlib/src/tests/searchcommon/schema/schema_test.cpp +++ b/searchlib/src/tests/searchcommon/schema/schema_test.cpp @@ -57,10 +57,6 @@ assertSchema(const Schema& exp, const Schema& act) for (size_t i = 0; i < exp.getNumAttributeFields(); ++i) { assertField(exp.getAttributeField(i), act.getAttributeField(i)); } - ASSERT_EQ(exp.getNumSummaryFields(), act.getNumSummaryFields()); - for (size_t i = 0; i < exp.getNumSummaryFields(); ++i) { - assertField(exp.getSummaryField(i), act.getSummaryField(i)); - } ASSERT_EQ(exp.getNumFieldSets(), act.getNumFieldSets()); for (size_t i = 0; i < exp.getNumFieldSets(); ++i) { assertSet(exp.getFieldSet(i), act.getFieldSet(i)); @@ -78,7 +74,6 @@ TEST(SchemaTest, test_basic) Schema s; EXPECT_EQ(0u, s.getNumIndexFields()); EXPECT_EQ(0u, s.getNumAttributeFields()); - EXPECT_EQ(0u, s.getNumSummaryFields()); EXPECT_EQ(0u, s.getNumImportedAttributeFields()); s.addIndexField(Schema::IndexField("foo", DataType::STRING)); @@ -88,11 +83,6 @@ TEST(SchemaTest, test_basic) s.addAttributeField(Schema::AttributeField("bar", DataType::INT32, CollectionType::WEIGHTEDSET)); s.addAttributeField(Schema::AttributeField("cox", DataType::STRING)); - s.addSummaryField(Schema::SummaryField("foo", DataType::STRING, CollectionType::ARRAY)); - s.addSummaryField(Schema::SummaryField("bar", DataType::INT32, CollectionType::WEIGHTEDSET)); - s.addSummaryField(Schema::SummaryField("cox", DataType::STRING)); - s.addSummaryField(Schema::SummaryField("fox", DataType::RAW)); - s.addFieldSet(Schema::FieldSet("default").addField("foo").addField("bar")); s.addImportedAttributeField(SIAF("imported", DataType::INT32)); @@ -130,30 +120,6 @@ TEST(SchemaTest, test_basic) EXPECT_EQ(2u, s.getAttributeFieldId("cox")); EXPECT_EQ(Schema::UNKNOWN_FIELD_ID, s.getIndexFieldId("fox")); } - ASSERT_EQ(4u, s.getNumSummaryFields()); - { - EXPECT_EQ("foo", s.getSummaryField(0).getName()); - EXPECT_EQ(DataType::STRING, s.getSummaryField(0).getDataType()); - EXPECT_EQ(CollectionType::ARRAY, s.getSummaryField(0).getCollectionType()); - - EXPECT_EQ("bar", s.getSummaryField(1).getName()); - EXPECT_EQ(DataType::INT32, s.getSummaryField(1).getDataType()); - EXPECT_EQ(CollectionType::WEIGHTEDSET, s.getSummaryField(1).getCollectionType()); - - EXPECT_EQ("cox", s.getSummaryField(2).getName()); - EXPECT_EQ(DataType::STRING, s.getSummaryField(2).getDataType()); - EXPECT_EQ(CollectionType::SINGLE, s.getSummaryField(2).getCollectionType()); - - EXPECT_EQ("fox", s.getSummaryField(3).getName()); - EXPECT_EQ(DataType::RAW, s.getSummaryField(3).getDataType()); - EXPECT_EQ(CollectionType::SINGLE, s.getSummaryField(3).getCollectionType()); - - EXPECT_EQ(0u, s.getSummaryFieldId("foo")); - EXPECT_EQ(1u, s.getSummaryFieldId("bar")); - EXPECT_EQ(2u, s.getSummaryFieldId("cox")); - EXPECT_EQ(3u, s.getSummaryFieldId("fox")); - EXPECT_EQ(Schema::UNKNOWN_FIELD_ID, s.getSummaryFieldId("not")); - } ASSERT_EQ(1u, s.getNumFieldSets()); { EXPECT_EQ("default", s.getFieldSet(0).getName()); @@ -172,7 +138,6 @@ TEST(SchemaTest, test_basic) TEST(SchemaTest, test_load_and_save) { using SAF = Schema::AttributeField; - using SSF = Schema::SummaryField; using SDT = schema::DataType; using SCT = schema::CollectionType; using SFS = Schema::FieldSet; @@ -198,20 +163,6 @@ TEST(SchemaTest, test_load_and_save) assertField(SAF("h", SDT::BOOLEANTREE), s.getAttributeField(7)); assertField(SAF("i", SDT::TENSOR), s.getAttributeField(8)); - EXPECT_EQ(12u, s.getNumSummaryFields()); - assertField(SSF("a", SDT::INT8), s.getSummaryField(0)); - assertField(SSF("b", SDT::INT16), s.getSummaryField(1)); - assertField(SSF("c", SDT::INT32), s.getSummaryField(2)); - assertField(SSF("d", SDT::INT64), s.getSummaryField(3)); - assertField(SSF("e", SDT::FLOAT), s.getSummaryField(4)); - assertField(SSF("f", SDT::DOUBLE), s.getSummaryField(5)); - assertField(SSF("g", SDT::STRING), s.getSummaryField(6)); - assertField(SSF("h", SDT::STRING), s.getSummaryField(7)); - assertField(SSF("i", SDT::STRING), s.getSummaryField(8)); - assertField(SSF("j", SDT::STRING), s.getSummaryField(9)); - assertField(SSF("k", SDT::RAW), s.getSummaryField(10)); - assertField(SSF("l", SDT::RAW), s.getSummaryField(11)); - EXPECT_EQ(1u, s.getNumFieldSets()); assertSet(SFS("default").addField("a").addField("c"), s.getFieldSet(0)); @@ -252,9 +203,6 @@ addAllFieldTypes(const string& name, Schema& schema) Schema::AttributeField attribute_field(name, DataType::STRING); schema.addAttributeField(attribute_field); - Schema::SummaryField summary_field(name, DataType::STRING); - schema.addSummaryField(summary_field); - schema.addFieldSet(Schema::FieldSet(name)); } @@ -278,11 +226,6 @@ TEST(SchemaTest, require_that_schemas_can_be_added) sum->getAttributeField(sum->getAttributeFieldId(name1))); EXPECT_TRUE(s2.getAttributeField(0) == sum->getAttributeField(sum->getAttributeFieldId(name2))); - ASSERT_EQ(2u, sum->getNumSummaryFields()); - EXPECT_TRUE(s1.getSummaryField(0) == - sum->getSummaryField(sum->getSummaryFieldId(name1))); - EXPECT_TRUE(s2.getSummaryField(0) == - sum->getSummaryField(sum->getSummaryFieldId(name2))); ASSERT_EQ(2u, sum->getNumFieldSets()); EXPECT_TRUE(s1.getFieldSet(0) == sum->getFieldSet(sum->getFieldSetId(name1))); @@ -391,6 +334,19 @@ TEST(SchemaTest, require_that_index_field_is_loaded_with_default_values_when_pro assertIndexField(SIF("foo", DataType::STRING, CollectionType::SINGLE), index_fields[0]); } +TEST(SchemaTest, test_load_from_saved_schema_with_summary_fields) +{ + vespalib::string schema_name("old-schema-with-summary-fields.txt"); + Schema s; + s.addIndexField(Schema::IndexField("ifoo", DataType::STRING)); + s.addIndexField(Schema::IndexField("ibar", DataType::INT32)); + s.addAttributeField(Schema::AttributeField("afoo", DataType::STRING)); + s.addAttributeField(Schema::AttributeField("abar", DataType::INT32)); + Schema s2; + EXPECT_TRUE(s2.loadFromFile(schema_name)); + assertSchema(s, s2); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchcommon/common/schema.cpp b/searchlib/src/vespa/searchcommon/common/schema.cpp index c6a2a4532a3..0cc9d3cb2d8 100644 --- a/searchlib/src/vespa/searchcommon/common/schema.cpp +++ b/searchlib/src/vespa/searchcommon/common/schema.cpp @@ -213,7 +213,6 @@ void Schema::writeToStream(vespalib::asciistream &os, bool saveToDisk) const { writeFields(os, "attributefield", _attributeFields); - writeFields(os, "summaryfield", _summaryFields); writeFieldSets(os, "fieldset", _fieldSets); writeFields(os, "indexfield", _indexFields); if (!saveToDisk) { @@ -245,7 +244,6 @@ Schema::loadFromFile(const vespalib::string & fileName) } _indexFields = ConfigParser::parseArray<std::vector<IndexField>>("indexfield", lines); _attributeFields = ConfigParser::parseArray<std::vector<AttributeField>>("attributefield", lines); - _summaryFields = ConfigParser::parseArray<std::vector<SummaryField>>("summaryfield", lines); _fieldSets = ConfigParser::parseArray<std::vector<FieldSet>>("fieldset", lines); _importedAttributeFields.clear(); // NOTE: these are not persisted to disk _indexIds.clear(); @@ -256,10 +254,6 @@ Schema::loadFromFile(const vespalib::string & fileName) for (size_t i(0), m(_attributeFields.size()); i < m; i++) { _attributeIds[_attributeFields[i].getName()] = i; } - _summaryIds.clear(); - for (size_t i(0), m(_summaryFields.size()); i < m; i++) { - _summaryIds[_summaryFields[i].getName()] = i; - } _fieldSetIds.clear(); for (size_t i(0), m(_fieldSets.size()); i < m; i++) { _fieldSetIds[_fieldSets[i].getName()] = i; @@ -357,12 +351,6 @@ Schema::addAttributeField(const AttributeField &field) } Schema & -Schema::addSummaryField(const SummaryField &field) -{ - return addField(field, *this, _summaryFields, _summaryIds); -} - -Schema & Schema::addImportedAttributeField(const ImportedAttributeField &field) { return addField(field, *this, _importedAttributeFields, _importedAttributeIds); @@ -387,12 +375,6 @@ Schema::getAttributeFieldId(vespalib::stringref name) const } uint32_t -Schema::getSummaryFieldId(vespalib::stringref name) const -{ - return getFieldId(name, _summaryIds); -} - -uint32_t Schema::getFieldSetId(vespalib::stringref name) const { return getFieldId(name, _fieldSetIds); @@ -405,12 +387,6 @@ Schema::isIndexField(vespalib::stringref name) const } bool -Schema::isSummaryField(vespalib::stringref name) const -{ - return _summaryIds.find(name) != _summaryIds.end(); -} - -bool Schema::isAttributeField(vespalib::stringref name) const { return _attributeIds.find(name) != _attributeIds.end(); @@ -422,12 +398,10 @@ Schema::swap(Schema &rhs) { _indexFields.swap(rhs._indexFields); _attributeFields.swap(rhs._attributeFields); - _summaryFields.swap(rhs._summaryFields); _fieldSets.swap(rhs._fieldSets); _importedAttributeFields.swap(rhs._importedAttributeFields); _indexIds.swap(rhs._indexIds); _attributeIds.swap(rhs._attributeIds); - _summaryIds.swap(rhs._summaryIds); _fieldSetIds.swap(rhs._fieldSetIds); _importedAttributeIds.swap(rhs._importedAttributeIds); } @@ -437,12 +411,10 @@ Schema::clear() { _indexFields.clear(); _attributeFields.clear(); - _summaryFields.clear(); _fieldSets.clear(); _importedAttributeFields.clear(); _indexIds.clear(); _attributeIds.clear(); - _summaryIds.clear(); _fieldSetIds.clear(); _importedAttributeIds.clear(); } @@ -515,8 +487,6 @@ Schema::intersect(const Schema &lhs, const Schema &rhs) h.schema->_indexFields, h.schema->_indexIds); h.intersect(lhs._attributeFields, rhs._attributeFields, rhs._attributeIds, h.schema->_attributeFields, h.schema->_attributeIds); - h.intersect(lhs._summaryFields, rhs._summaryFields, rhs._summaryIds, - h.schema->_summaryFields, h.schema->_summaryIds); h.intersect(lhs._fieldSets, rhs._fieldSets, rhs._fieldSetIds, h.schema->_fieldSets, h.schema->_fieldSetIds); return std::move(h.schema); @@ -528,7 +498,6 @@ Schema::make_union(const Schema &lhs, const Schema &rhs) Schema::UP schema(new Schema(lhs)); addEntries(rhs._indexFields, schema->_indexFields, schema->_indexIds); addEntries(rhs._attributeFields, schema->_attributeFields, schema->_attributeIds); - addEntries(rhs._summaryFields, schema->_summaryFields, schema->_summaryIds); addEntries(rhs._fieldSets, schema->_fieldSets, schema->_fieldSetIds); return schema; } @@ -541,8 +510,6 @@ Schema::set_difference(const Schema &lhs, const Schema &rhs) schema->_indexFields, schema->_indexIds); difference(lhs._attributeFields, rhs._attributeIds, schema->_attributeFields, schema->_attributeIds); - difference(lhs._summaryFields, rhs._summaryIds, - schema->_summaryFields, schema->_summaryIds); difference(lhs._fieldSets, rhs._fieldSetIds, schema->_fieldSets, schema->_fieldSetIds); return schema; @@ -553,7 +520,6 @@ Schema::operator==(const Schema &rhs) const { return _indexFields == rhs._indexFields && _attributeFields == rhs._attributeFields && - _summaryFields == rhs._summaryFields && _fieldSets == rhs._fieldSets && _importedAttributeFields == rhs._importedAttributeFields; } @@ -563,7 +529,6 @@ Schema::operator!=(const Schema &rhs) const { return _indexFields != rhs._indexFields || _attributeFields != rhs._attributeFields || - _summaryFields != rhs._summaryFields || _fieldSets != rhs._fieldSets || _importedAttributeFields != rhs._importedAttributeFields; } @@ -573,7 +538,6 @@ Schema::empty() const { return _indexFields.empty() && _attributeFields.empty() && - _summaryFields.empty() && _fieldSets.empty() && _importedAttributeFields.empty(); } diff --git a/searchlib/src/vespa/searchcommon/common/schema.h b/searchlib/src/vespa/searchcommon/common/schema.h index 3a9bcbdd904..42291e04634 100644 --- a/searchlib/src/vespa/searchcommon/common/schema.h +++ b/searchlib/src/vespa/searchcommon/common/schema.h @@ -25,7 +25,7 @@ public: /** * A single field has a name, data type and collection - * type. Various aspects (index/attribute/summary) may have + * type. Various aspects (index/attribute) may have * limitations on what types are supported in the back-end. **/ class Field @@ -109,7 +109,6 @@ public: }; using AttributeField = Field; - using SummaryField = Field; using ImportedAttributeField = Field; /** @@ -154,13 +153,11 @@ public: private: std::vector<IndexField> _indexFields; std::vector<AttributeField> _attributeFields; - std::vector<SummaryField> _summaryFields; std::vector<FieldSet> _fieldSets; std::vector<ImportedAttributeField> _importedAttributeFields; using Name2IdMap = vespalib::hash_map<vespalib::string, uint32_t>; Name2IdMap _indexIds; Name2IdMap _attributeIds; - Name2IdMap _summaryIds; Name2IdMap _fieldSetIds; Name2IdMap _importedAttributeIds; @@ -218,14 +215,6 @@ public: addAttributeField(const AttributeField &field); /** - * Add a summary field to this schema - * - * @param field the field to add - **/ - Schema & - addSummaryField(const SummaryField &field); - - /** * Add a field set to this schema. * * @param collection the field set to add. @@ -250,13 +239,6 @@ public: uint32_t getNumAttributeFields() const { return _attributeFields.size(); } /** - * Obtain the number of summary fields in this schema. - * - * @return number of fields - **/ - uint32_t getNumSummaryFields() const { return _summaryFields.size(); } - - /** * Obtain the number of field sets in this schema. * * @return number of field sets. @@ -301,14 +283,6 @@ public: bool isIndexField(vespalib::stringref name) const; /** - * Check if a field is a summary field - * - * @return true if field is an summary field. - * @param name the name of the field. - **/ - bool isSummaryField(vespalib::stringref name) const; - - /** * Check if a field is a attribute field * * @return true if field is an attribute field. @@ -344,33 +318,6 @@ public: uint32_t getAttributeFieldId(vespalib::stringref name) const; /** - * Get information about a specific summary field using the given fieldId. - * - * @return the field - * @param idx an index in the range [0, size - 1] - **/ - const SummaryField & - getSummaryField(uint32_t fieldId) const - { - return _summaryFields[fieldId]; - } - - /** - * Returns const view of the summary fields. - */ - const std::vector<SummaryField> &getSummaryFields() const { - return _summaryFields; - } - - /** - * Get the field id for the summary field with the given name. - * - * @return the field id or UNKNOWN_FIELD_ID if not found. - * @param name the name of the field. - **/ - uint32_t getSummaryFieldId(vespalib::stringref name) const; - - /** * Get information about a specific field set * * @return the field set. diff --git a/searchlib/src/vespa/searchcommon/common/schemaconfigurer.cpp b/searchlib/src/vespa/searchcommon/common/schemaconfigurer.cpp index 8fbebe80b4b..fd23b699f01 100644 --- a/searchlib/src/vespa/searchcommon/common/schemaconfigurer.cpp +++ b/searchlib/src/vespa/searchcommon/common/schemaconfigurer.cpp @@ -103,37 +103,6 @@ convertCollectionType(const ConfigType &type) return CollectionType::SINGLE; } - -Schema::DataType -convertSummaryType(const vespalib::string &type) -{ - if (type == "byte") { - return DataType::INT8; - } else if (type == "short") { - return DataType::INT16; - } else if (type == "integer") { - return DataType::INT32; - } else if (type == "int64") { - return DataType::INT64; - } else if (type == "float") { - return DataType::FLOAT; - } else if (type == "double") { - return DataType::DOUBLE; - } else if (type == "string" || - type == "longstring" || - type == "xmlstring" || - type == "featuredata" || - type == "jsonstring") - { - return DataType::STRING; - } else if (type == "data" || - type == "longdata") - { - return DataType::RAW; - } - return DataType::RAW; -} - } void @@ -175,36 +144,6 @@ SchemaBuilder::build(const AttributesConfig &cfg, Schema &schema) void -SchemaBuilder::build(const SummaryConfig &cfg, Schema &schema) -{ - for (size_t i = 0; i < cfg.classes.size(); ++i) { - LOG(debug, "class with index %lu has id %d (default has id %d)", - i, cfg.classes[i].id, cfg.defaultsummaryid); - } - for (size_t i = 0; i < cfg.classes.size(); ++i) { - // use the default summary class that has all fields - if (cfg.classes[i].id == cfg.defaultsummaryid) { - for (size_t j = 0; j < cfg.classes[i].fields.size(); ++j) { - const SummaryConfig::Classes::Fields & f = - cfg.classes[i].fields[j]; - schema.addSummaryField(Schema::Field(f.name, - convertSummaryType(f.type))); - } - return; - } - } - if (cfg.classes.empty()) { - LOG(debug, - "No summary class configured that match the default summary id %d", - cfg.defaultsummaryid); - } else { - LOG(warning, - "No summary class configured that match the default summary id %d", - cfg.defaultsummaryid); - } -} - -void SchemaConfigurer::configure(const IndexschemaConfig &cfg) { SchemaBuilder::build(cfg, _schema); @@ -216,12 +155,6 @@ SchemaConfigurer::configure(const AttributesConfig &cfg) SchemaBuilder::build(cfg, _schema); } -void -SchemaConfigurer::configure(const SummaryConfig & cfg) -{ - SchemaBuilder::build(cfg, _schema); -} - SchemaConfigurer::SchemaConfigurer(Schema &schema, const vespalib::string &configId) : _schema(schema) { @@ -229,11 +162,8 @@ SchemaConfigurer::SchemaConfigurer(Schema &schema, const vespalib::string &confi indexSchemaSubscriber(*this, &SchemaConfigurer::configure); search::SubscriptionProxyNg<SchemaConfigurer, AttributesConfig> attributesSubscriber(*this, &SchemaConfigurer::configure); - search::SubscriptionProxyNg<SchemaConfigurer, SummaryConfig> - summarySubscriber(*this, &SchemaConfigurer::configure); indexSchemaSubscriber.subscribe(configId.c_str()); attributesSubscriber.subscribe(configId.c_str()); - summarySubscriber.subscribe(configId.c_str()); } } diff --git a/searchlib/src/vespa/searchcommon/common/schemaconfigurer.h b/searchlib/src/vespa/searchcommon/common/schemaconfigurer.h index 925aefcfa25..30deb01456c 100644 --- a/searchlib/src/vespa/searchcommon/common/schemaconfigurer.h +++ b/searchlib/src/vespa/searchcommon/common/schemaconfigurer.h @@ -7,7 +7,6 @@ namespace vespa::config::search::internal { class InternalIndexschemaType; class InternalAttributesType; - class InternalSummaryType; } namespace search::index { @@ -23,7 +22,6 @@ class SchemaBuilder public: using IndexschemaConfig = const vespa::config::search::internal::InternalIndexschemaType; using AttributesConfig = const vespa::config::search::internal::InternalAttributesType; - using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; /** * Build from indexschema config. * @@ -36,13 +34,6 @@ public: * @param attributeCfg AttributesConfig to use **/ static void build(const AttributesConfig &cfg, Schema &schema); - /** - * Build from summary config. - * - * @param summaryCfg SummaryConfig to use - **/ - static void build(const SummaryConfig &cfg, Schema &schema); - }; class SchemaConfigurer @@ -50,11 +41,9 @@ class SchemaConfigurer private: using IndexschemaConfig = SchemaBuilder::IndexschemaConfig; using AttributesConfig = SchemaBuilder::AttributesConfig; - using SummaryConfig = SchemaBuilder::SummaryConfig; Schema & _schema; void configure(const IndexschemaConfig & cfg); void configure(const AttributesConfig & cfg); - void configure(const SummaryConfig & cfg); public: /** |