diff options
55 files changed, 693 insertions, 841 deletions
diff --git a/config/src/vespa/config/common/exceptions.h b/config/src/vespa/config/common/exceptions.h index c3fafab1287..6e70947135d 100644 --- a/config/src/vespa/config/common/exceptions.h +++ b/config/src/vespa/config/common/exceptions.h @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <string> -#include <stdexcept> #include <vespa/vespalib/util/exception.h> namespace config { diff --git a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt index a70a71772f5..fd8e5419ccd 100644 --- a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt +++ b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcommon_searchcommon_common OBJECT SOURCES + datatype.cpp schema.cpp schemaconfigurer.cpp DEPENDS diff --git a/searchcommon/src/vespa/searchcommon/common/datatype.cpp b/searchcommon/src/vespa/searchcommon/common/datatype.cpp new file mode 100644 index 00000000000..f5aa9e50f3a --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/common/datatype.cpp @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "datatype.h" +#include <vespa/config/common/exceptions.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/arraysize.h> + +namespace search { +namespace index { +namespace schema { + +using config::InvalidConfigException; + +DataType +dataTypeFromName(const vespalib::stringref &name) { + if (name == "UINT1") { return UINT1; } + else if (name == "UINT2") { return UINT2; } + else if (name == "UINT4") { return UINT4; } + else if (name == "INT8") { return INT8; } + else if (name == "INT16") { return INT16; } + else if (name == "INT32") { return INT32; } + else if (name == "INT64") { return INT64; } + else if (name == "FLOAT") { return FLOAT; } + else if (name == "DOUBLE") { return DOUBLE; } + else if (name == "STRING") { return STRING; } + else if (name == "RAW") { return RAW; } + else if (name == "BOOLEANTREE") { return BOOLEANTREE; } + else if (name == "TENSOR") { return TENSOR; } + else { + throw InvalidConfigException("Illegal enum value '" + name + "'"); + } +} + +const char *datatype_str[] = { "UINT1", + "UINT2", + "UINT4", + "INT8", + "INT16", + "INT32", + "INT64", + "FLOAT", + "DOUBLE", + "STRING", + "RAW", + "FEATURE_NOTUSED", + "BOOLEANTREE", + "TENSOR" }; + +vespalib::string +getTypeName(DataType type) { + if (type > vespalib::arraysize(datatype_str)) { + vespalib::asciistream ost; + ost << "UNKNOWN(" << type << ")"; + return ost.str(); + } + return datatype_str[type]; +} + +CollectionType +collectionTypeFromName(const vespalib::stringref &name) { + if (name == "SINGLE") { return SINGLE; } + else if (name == "ARRAY") { return ARRAY; } + else if (name == "WEIGHTEDSET") { return WEIGHTEDSET; } + else { + throw InvalidConfigException("Illegal enum value '" + name + "'"); + } +} + +const char *collectiontype_str[] = { "SINGLE", + "ARRAY", + "WEIGHTEDSET" }; + +vespalib::string +getTypeName(CollectionType type) { + if (type > vespalib::arraysize(collectiontype_str)) { + vespalib::asciistream ost; + ost << "UNKNOWN(" << type << ")"; + return ost.str(); + } + return collectiontype_str[type]; +} + + +} +} +} diff --git a/searchcommon/src/vespa/searchcommon/common/datatype.h b/searchcommon/src/vespa/searchcommon/common/datatype.h new file mode 100644 index 00000000000..de442e139db --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/common/datatype.h @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace search { +namespace index { +namespace schema { + +/** + * Basic data type for a field. + **/ +enum DataType { UINT1 = 0, + UINT2 = 1, + UINT4 = 2, + INT8 = 3, + INT16 = 4, + INT32 = 5, + INT64 = 6, + FLOAT = 7, + DOUBLE = 8, + STRING = 9, + RAW = 10, + //FEATURE = 11, + BOOLEANTREE = 12, + TENSOR = 13 +}; + +/** + * Collection type for a field. + **/ +enum CollectionType { SINGLE = 0, + ARRAY = 1, + WEIGHTEDSET = 2 +}; + +DataType dataTypeFromName(const vespalib::stringref &name); +vespalib::string getTypeName(DataType type); +CollectionType collectionTypeFromName(const vespalib::stringref &n); +vespalib::string getTypeName(CollectionType type); + +} +} +} diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp index 673d78fbc89..88403f49d61 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> +#include "schema.h" #include <fstream> #include <vespa/config/common/configparser.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/arraysize.h> -#include "schema.h" +#include <vespa/log/log.h> LOG_SETUP(".index.schema"); using namespace config; @@ -69,76 +68,10 @@ namespace index { const uint32_t Schema::UNKNOWN_FIELD_ID(std::numeric_limits<uint32_t>::max()); -Schema::DataType Schema::dataTypeFromName(const vespalib::stringref &name) { - if (name == "UINT1") { return UINT1; } - else if (name == "UINT2") { return UINT2; } - else if (name == "UINT4") { return UINT4; } - else if (name == "INT8") { return INT8; } - else if (name == "INT16") { return INT16; } - else if (name == "INT32") { return INT32; } - else if (name == "INT64") { return INT64; } - else if (name == "FLOAT") { return FLOAT; } - else if (name == "DOUBLE") { return DOUBLE; } - else if (name == "STRING") { return STRING; } - else if (name == "RAW") { return RAW; } - else if (name == "BOOLEANTREE") { return BOOLEANTREE; } - else if (name == "TENSOR") { return TENSOR; } - else { - throw InvalidConfigException("Illegal enum value '" + name + "'"); - } -} - -const char *datatype_str[] = { "UINT1", - "UINT2", - "UINT4", - "INT8", - "INT16", - "INT32", - "INT64", - "FLOAT", - "DOUBLE", - "STRING", - "RAW", - "FEATURE_NOTUSED", - "BOOLEANTREE", - "TENSOR" }; - -vespalib::string Schema::getTypeName(DataType type) { - if (type > vespalib::arraysize(datatype_str)) { - vespalib::asciistream ost; - ost << "UNKNOWN(" << type << ")"; - return ost.str(); - } - return datatype_str[type]; -} - -Schema::CollectionType Schema::collectionTypeFromName( - const vespalib::stringref &name) { - if (name == "SINGLE") { return SINGLE; } - else if (name == "ARRAY") { return ARRAY; } - else if (name == "WEIGHTEDSET") { return WEIGHTEDSET; } - else { - throw InvalidConfigException("Illegal enum value '" + name + "'"); - } -} - -const char *collectiontype_str[] = { "SINGLE", - "ARRAY", - "WEIGHTEDSET" }; - -vespalib::string Schema::getTypeName(CollectionType type) { - if (type > vespalib::arraysize(collectiontype_str)) { - vespalib::asciistream ost; - ost << "UNKNOWN(" << type << ")"; - return ost.str(); - } - return collectiontype_str[type]; -} - Schema::Field::Field(const vespalib::stringref &n, DataType dt) : _name(n), _dataType(dt), - _collectionType(SINGLE), + _collectionType(schema::SINGLE), _timestamp(0) { } @@ -155,10 +88,10 @@ Schema::Field::Field(const vespalib::stringref &n, // XXX: Resource leak if exception is thrown. Schema::Field::Field(const std::vector<vespalib::string> & lines) : _name(ConfigParser::parse<vespalib::string>("name", lines)), - _dataType(dataTypeFromName(ConfigParser::parse<vespalib::string>( + _dataType(schema::dataTypeFromName(ConfigParser::parse<vespalib::string>( "datatype", lines))), _collectionType( - collectionTypeFromName(ConfigParser::parse<vespalib::string>( + schema::collectionTypeFromName(ConfigParser::parse<vespalib::string>( "collectiontype", lines))), _timestamp(ConfigParser::parse<int64_t>("timestamp", lines, 0)) { diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h index dcc559f6293..f3ab4e745b4 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.h +++ b/searchcommon/src/vespa/searchcommon/common/schema.h @@ -3,11 +3,14 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/ptrholder.h> #include <vector> +#include "datatype.h" +namespace vespalib { + class asciistream; +} namespace search { namespace index { @@ -22,34 +25,9 @@ public: typedef std::shared_ptr<Schema> SP; typedef vespalib::PtrHolder<Schema> PH; - /** - * Basic data type for a field. - **/ - enum DataType { UINT1 = 0, - UINT2 = 1, - UINT4 = 2, - INT8 = 3, - INT16 = 4, - INT32 = 5, - INT64 = 6, - FLOAT = 7, - DOUBLE = 8, - STRING = 9, - RAW = 10, - //FEATURE = 11, - BOOLEANTREE = 12, - TENSOR = 13}; - static DataType dataTypeFromName(const vespalib::stringref &name); - static vespalib::string getTypeName(DataType type); + using DataType = schema::DataType; - /** - * Collection type for a field. - **/ - enum CollectionType { SINGLE = 0, - ARRAY = 1, - WEIGHTEDSET = 2 }; - static CollectionType collectionTypeFromName(const vespalib::stringref &n); - static vespalib::string getTypeName(CollectionType type); + using CollectionType = schema::CollectionType; /** * A single field has a name, data type and collection @@ -107,8 +85,7 @@ public: public: IndexField(const vespalib::stringref &name, DataType dt); - IndexField(const vespalib::stringref &name, DataType dt, - CollectionType ct); + IndexField(const vespalib::stringref &name, DataType dt, CollectionType ct); /** * Create this index field based on the given config lines. **/ @@ -176,7 +153,7 @@ private: std::vector<AttributeField> _attributeFields; std::vector<SummaryField> _summaryFields; std::vector<FieldSet> _fieldSets; - typedef vespalib::hash_map<vespalib::string, uint32_t> Name2IdMap; + using Name2IdMap = vespalib::hash_map<vespalib::string, uint32_t>; Name2IdMap _indexIds; Name2IdMap _attributeIds; Name2IdMap _summaryIds; diff --git a/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp b/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp index 34071e241d7..56acb3d8ec2 100644 --- a/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp @@ -19,13 +19,13 @@ SchemaBuilder::convert(const IndexschemaConfig::Indexfield::Datatype &type) { switch (type) { case IndexschemaConfig::Indexfield::STRING: - return Schema::STRING; + return schema::STRING; case IndexschemaConfig::Indexfield::INT64: - return Schema::INT64; + return schema::INT64; case IndexschemaConfig::Indexfield::BOOLEANTREE: - return Schema::BOOLEANTREE; + return schema::BOOLEANTREE; } - return Schema::STRING; + return schema::STRING; } @@ -34,13 +34,13 @@ SchemaBuilder::convert(const IndexschemaConfig::Indexfield::Collectiontype & typ { switch (type) { case IndexschemaConfig::Indexfield::SINGLE: - return Schema::SINGLE; + return schema::SINGLE; case IndexschemaConfig::Indexfield::ARRAY: - return Schema::ARRAY; + return schema::ARRAY; case IndexschemaConfig::Indexfield::WEIGHTEDSET: - return Schema::WEIGHTEDSET; + return schema::WEIGHTEDSET; } - return Schema::SINGLE; + return schema::SINGLE; } @@ -49,34 +49,34 @@ SchemaBuilder::convert(const AttributesConfig::Attribute::Datatype &type) { switch (type) { case AttributesConfig::Attribute::STRING: - return Schema::STRING; + return schema::STRING; case AttributesConfig::Attribute::UINT1: - return Schema::UINT1; + return schema::UINT1; case AttributesConfig::Attribute::UINT2: - return Schema::UINT2; + return schema::UINT2; case AttributesConfig::Attribute::UINT4: - return Schema::UINT4; + return schema::UINT4; case AttributesConfig::Attribute::INT8: - return Schema::INT8; + return schema::INT8; case AttributesConfig::Attribute::INT16: - return Schema::INT16; + return schema::INT16; case AttributesConfig::Attribute::INT32: - return Schema::INT32; + return schema::INT32; case AttributesConfig::Attribute::INT64: - return Schema::INT64; + return schema::INT64; case AttributesConfig::Attribute::FLOAT: - return Schema::FLOAT; + return schema::FLOAT; case AttributesConfig::Attribute::DOUBLE: - return Schema::DOUBLE; + return schema::DOUBLE; case AttributesConfig::Attribute::PREDICATE: - return Schema::BOOLEANTREE; + return schema::BOOLEANTREE; case AttributesConfig::Attribute::TENSOR: - return Schema::TENSOR; + return schema::TENSOR; default: break; } // TODO: exception? - return Schema::STRING; + return schema::STRING; } @@ -85,13 +85,13 @@ SchemaBuilder::convert(const AttributesConfig::Attribute::Collectiontype &type) { switch (type) { case AttributesConfig::Attribute::SINGLE: - return Schema::SINGLE; + return schema::SINGLE; case AttributesConfig::Attribute::ARRAY: - return Schema::ARRAY; + return schema::ARRAY; case AttributesConfig::Attribute::WEIGHTEDSET: - return Schema::WEIGHTEDSET; + return schema::WEIGHTEDSET; } - return Schema::SINGLE; + return schema::SINGLE; } @@ -99,30 +99,30 @@ Schema::DataType SchemaBuilder::convertSummaryType(const vespalib::string & type) { if (type == "byte") { - return Schema::INT8; + return schema::INT8; } else if (type == "short") { - return Schema::INT16; + return schema::INT16; } else if (type == "integer") { - return Schema::INT32; + return schema::INT32; } else if (type == "int64") { - return Schema::INT64; + return schema::INT64; } else if (type == "float") { - return Schema::FLOAT; + return schema::FLOAT; } else if (type == "double") { - return Schema::DOUBLE; + return schema::DOUBLE; } else if (type == "string" || type == "longstring" || type == "xmlstring" || type == "featuredata" || type == "jsonstring") { - return Schema::STRING; + return schema::STRING; } else if (type == "data" || type == "longdata") { - return Schema::RAW; + return schema::RAW; } - return Schema::RAW; + return schema::RAW; } diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp index df7c82f191d..292f0e2e787 100644 --- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp +++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp @@ -445,11 +445,11 @@ void Test::checkArray(const char *(&str)[N], const FieldValue *value) { } void Test::setSummaryField(const string &field) { - _schema->addSummaryField(Schema::Field(field, Schema::STRING)); + _schema->addSummaryField(Schema::Field(field, search::index::schema::STRING)); } void Test::setAttributeField(const string &field) { - _schema->addAttributeField(Schema::Field(field, Schema::STRING)); + _schema->addAttributeField(Schema::Field(field, search::index::schema::STRING)); } void Test::requireThatSummaryIsAnUnmodifiedString() { diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index ccd80f6bdf3..3b863561188 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -96,8 +96,7 @@ const uint32_t field_id = 1; Schema getSchema() { Schema schema; - schema.addIndexField(Schema::IndexField(field_name, - Schema::STRING)); + schema.addIndexField(Schema::IndexField(field_name, search::index::schema::STRING)); return schema; } diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 8cace3a7bb4..bd1b0a5db6b 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -15,8 +15,8 @@ const std::string gen_dir("generated"); const char *valid_feature = "value(0)"; const char *invalid_feature = "invalid_feature_name and format"; -using search::index::Schema; using namespace search::fef::indexproperties; +using namespace search::index; struct Writer { FILE *file; @@ -53,17 +53,17 @@ struct Model { Model() : indexes(), attributes(), properties(), extra_profiles() { verify_dir(); } - void index(const std::string &name, Schema::DataType data_type, - Schema::CollectionType collection_type) + void index(const std::string &name, schema::DataType data_type, + schema::CollectionType collection_type) { - indexes[name].first = Schema::getTypeName(data_type); - indexes[name].second = Schema::getTypeName(collection_type); + indexes[name].first = schema::getTypeName(data_type); + indexes[name].second = schema::getTypeName(collection_type); } - void attribute(const std::string &name, Schema::DataType data_type, - Schema::CollectionType collection_type) + void attribute(const std::string &name, schema::DataType data_type, + schema::CollectionType collection_type) { - attributes[name].first = Schema::getTypeName(data_type); - attributes[name].second = Schema::getTypeName(collection_type); + attributes[name].first = schema::getTypeName(data_type); + attributes[name].second = schema::getTypeName(collection_type); } void property(const std::string &name, const std::string &val) { properties[name] = val; @@ -162,18 +162,18 @@ struct EmptyModel : Model {}; struct SimpleModel : Model { SimpleModel() : Model() { - index("title", Schema::STRING, Schema::SINGLE); - index("list", Schema::STRING, Schema::ARRAY); - index("keywords", Schema::STRING, Schema::WEIGHTEDSET); - attribute("date", Schema::INT32, Schema::SINGLE); + index("title", schema::STRING, schema::SINGLE); + index("list", schema::STRING, schema::ARRAY); + index("keywords", schema::STRING, schema::WEIGHTEDSET); + attribute("date", schema::INT32, schema::SINGLE); constants["my_tensor"] = "tensor(x{},y{})"; } }; struct ShadowModel : Model { ShadowModel() : Model() { - index("both", Schema::STRING, Schema::SINGLE); - attribute("both", Schema::STRING, Schema::SINGLE); + index("both", schema::STRING, schema::SINGLE); + attribute("both", schema::STRING, schema::SINGLE); } }; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp index f95bf7dabf4..74a6d97366d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.attribute.document_field_retriever"); #include "document_field_retriever.h" - #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> @@ -12,6 +9,9 @@ LOG_SETUP(".proton.attribute.document_field_retriever"); #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/vespalib/tensor/tensor.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.attribute.document_field_retriever"); + using search::DocumentIdT; using document::ArrayFieldValue; using document::Document; @@ -19,7 +19,7 @@ using document::Field; using document::FieldValue; using document::TensorFieldValue; using document::WeightedSetFieldValue; -using search::index::Schema; +using namespace search::index; using search::attribute::AttributeContent; using search::attribute::IAttributeVector; using search::attribute::WeightedType; @@ -38,7 +38,7 @@ setValue(DocumentIdT lid, const IAttributeVector &attr) { switch (field.getCollectionType()) { - case Schema::SINGLE: + case schema::SINGLE: { if ( ! attr.isUndefined(lid) ) { AttributeContent<T> content; @@ -49,7 +49,7 @@ setValue(DocumentIdT lid, } break; } - case Schema::ARRAY: + case schema::ARRAY: { AttributeContent<T> content; content.fill(attr, lid); @@ -69,7 +69,7 @@ setValue(DocumentIdT lid, doc.setValue(f, *fv); break; } - case Schema::WEIGHTEDSET: + case schema::WEIGHTEDSET: { AttributeContent<WeightedType<T> > content; content.fill(attr, lid); @@ -107,34 +107,34 @@ DocumentFieldRetriever::populate(DocumentIdT lid, bool isIndexField) { switch (field.getDataType()) { - case Schema::UINT1: - case Schema::UINT2: - case Schema::UINT4: - case Schema::INT8: - case Schema::INT16: - case Schema::INT32: - case Schema::INT64: + case schema::UINT1: + case schema::UINT2: + case schema::UINT4: + case schema::INT8: + case schema::INT16: + case schema::INT32: + case schema::INT64: setValue<IAttributeVector::largeint_t>( lid, doc, field, attr); break; - case Schema::FLOAT: - case Schema::DOUBLE: + case schema::FLOAT: + case schema::DOUBLE: setValue<double>(lid, doc, field, attr); break; - case Schema::STRING: + case schema::STRING: // If it is a stringfield we also need to check if // it is an index field. In that case we shall // keep the original in order to preserve annotations. if (isIndexField) { break; } - case Schema::RAW: + case schema::RAW: setValue<const char *>(lid, doc, field, attr); break; - case Schema::BOOLEANTREE: + case schema::BOOLEANTREE: // Predicate attribute doesn't store documents, it only indexes them. break; - case Schema::TENSOR: + case schema::TENSOR: // Tensor attribute is not authorative. Partial updates must update // document store. break; diff --git a/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp b/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp index 4175925357d..da7e89d8f59 100644 --- a/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> +#include "schemautil.h" + #include <vespa/log/log.h> LOG_SETUP(".proton.common.schemautil"); -#include "schemautil.h" - -using search::index::Schema; +using namespace search::index; namespace proton { @@ -205,24 +205,24 @@ SchemaUtil::listSchema(const Schema &schema, const Schema::AttributeField &field = schema.getAttributeField(i); quads.push_back( FieldQuad(field.getName(), - Schema::getTypeName(field.getDataType()), - Schema::getTypeName(field.getCollectionType()), + schema::getTypeName(field.getDataType()), + schema::getTypeName(field.getCollectionType()), "a")); } for (uint32_t i = 0; i < schema.getNumIndexFields(); ++i) { const Schema::IndexField &field = schema.getIndexField(i); quads.push_back( FieldQuad(field.getName(), - Schema::getTypeName(field.getDataType()), - Schema::getTypeName(field.getCollectionType()), + schema::getTypeName(field.getDataType()), + schema::getTypeName(field.getCollectionType()), "i")); } for (uint32_t i = 0; i < schema.getNumSummaryFields(); ++i) { const Schema::SummaryField &field = schema.getSummaryField(i); quads.push_back( FieldQuad(field.getName(), - Schema::getTypeName(field.getDataType()), - Schema::getTypeName(field.getCollectionType()), + schema::getTypeName(field.getDataType()), + schema::getTypeName(field.getCollectionType()), "s")); } std::sort(quads.begin(), quads.end()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp index 062bc7bdd27..7cec043a162 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.matching.indexenvironment"); #include "indexenvironment.h" #include <vespa/searchlib/fef/functiontablefactory.h> @@ -18,9 +16,9 @@ namespace { search::fef::CollectionType convertCollectionType(search::index::Schema::CollectionType type) { switch (type) { - case search::index::Schema::SINGLE: return search::fef::CollectionType::SINGLE; - case search::index::Schema::ARRAY: return search::fef::CollectionType::ARRAY; - case search::index::Schema::WEIGHTEDSET: return search::fef::CollectionType::WEIGHTEDSET; + case search::index::schema::SINGLE: return search::fef::CollectionType::SINGLE; + case search::index::schema::ARRAY: return search::fef::CollectionType::ARRAY; + case search::index::schema::WEIGHTEDSET: return search::fef::CollectionType::WEIGHTEDSET; default: abort(); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index 0cbb00efd85..fb78f25cc41 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -19,6 +19,7 @@ #include <vespa/vespalib/util/clock.h> #include <vespa/vespalib/util/closure.h> #include <vespa/vespalib/util/thread_bundle.h> +#include <vespa/searchcore/grouping/groupingmanager.h> #include <vespa/log/log.h> #include <cmath> @@ -389,7 +390,7 @@ MatchThread::run() search::ResultSet::UP result = findMatches(*matchTools); match_time.stop(); match_time_s = match_time.elapsed().sec(); - resultContext = resultProcessor.createThreadContext(matchTools->getHardDoom(), thread_id); + resultContext = resultProcessor.createThreadContext(matchTools->getHardDoom(), thread_id, _distributionKey); { WaitTimer get_token_timer(wait_time_s); QueryLimiter::Token::UP processToken( diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h index 7264057c223..f998a8aa92e 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h +++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h @@ -6,8 +6,6 @@ #include <vespa/vespalib/util/dual_merge_director.h> #include <vespa/searchlib/common/rankedhit.h> #include <vector> -#include <algorithm> -#include <memory> namespace proton { namespace matching { diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp index 37dfbe3faf5..787178de76e 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp @@ -1,13 +1,23 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.matching.result_processor"); #include "result_processor.h" -#include <vespa/searchlib/common/sortresults.h> #include <vespa/searchlib/common/docstamp.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchlib/engine/searchreply.h> +#include <vespa/searchcore/grouping/groupingcontext.h> +#include <vespa/searchcore/grouping/groupingmanager.h> +#include <vespa/searchcore/grouping/groupingsession.h> +#include "partial_result.h" +#include "sessionmanager.h" + +#include <vespa/log/log.h> +LOG_SETUP(".proton.matching.result_processor"); + +using search::attribute::IAttributeContext; +using search::grouping::GroupingSession; +using search::grouping::GroupingContext; +using search::grouping::SessionId; namespace proton { namespace matching { @@ -19,22 +29,40 @@ ResultProcessor::Result::Result(std::unique_ptr<search::engine::SearchReply> rep ResultProcessor::Result::~Result() { } -ResultProcessor::Sort::Sort(const search::IDocumentMetaStore & metaStore, const vespalib::Doom & doom, - search::attribute::IAttributeContext &ac, const vespalib::string &ss) +ResultProcessor::Sort::Sort(uint32_t partitionId, const vespalib::Doom & doom, IAttributeContext &ac, const vespalib::string &ss) : sorter(FastS_DefaultResultSorter::instance()), _ucaFactory(std::make_unique<search::uca::UcaConverterFactory>()), - sortSpec(&metaStore, doom, *_ucaFactory) + sortSpec(partitionId, doom, *_ucaFactory) { if (!ss.empty() && sortSpec.Init(ss.c_str(), ac)) { sorter = &sortSpec; } } -ResultProcessor::ResultProcessor(search::attribute::IAttributeContext &attrContext, +ResultProcessor::Context::Context(Sort::UP s, PartialResult::LP r, GroupingContext::UP g) + : sort(std::move(s)), + result(r), + grouping(std::move(g)), + groupingSource(grouping.get()) +{ } + +ResultProcessor::Context::~Context() { } + +void +ResultProcessor::GroupingSource::merge(Source &s) { + GroupingSource &rhs = static_cast<GroupingSource&>(s); + assert((ctx == 0) == (rhs.ctx == 0)); + if (ctx != 0) { + search::grouping::GroupingManager man(*ctx); + man.merge(*rhs.ctx); + } +} + +ResultProcessor::ResultProcessor(IAttributeContext &attrContext, const search::IDocumentMetaStore &metaStore, SessionManager &sessionMgr, - search::grouping::GroupingContext &groupingContext, - const search::grouping::SessionId &sessionId, + GroupingContext &groupingContext, + const vespalib::string &sessionId, const vespalib::string &sortSpec, size_t offset, size_t hits) : _attrContext(attrContext), @@ -49,11 +77,12 @@ ResultProcessor::ResultProcessor(search::attribute::IAttributeContext &attrConte _wasMerged(false) { if (!_groupingContext.empty()) { - _groupingSession.reset(new search::grouping::GroupingSession(sessionId, - _groupingContext, attrContext)); + _groupingSession.reset(new GroupingSession(sessionId, _groupingContext, attrContext)); } } +ResultProcessor::~ResultProcessor() { } + void ResultProcessor::prepareThreadContextCreation(size_t num_threads) { @@ -66,9 +95,9 @@ ResultProcessor::prepareThreadContextCreation(size_t num_threads) } ResultProcessor::Context::UP -ResultProcessor::createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id) +ResultProcessor::createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id, uint32_t distributionKey) { - Sort::UP sort(new Sort(_metaStore, hardDoom, _attrContext, _sortSpec)); + Sort::UP sort(new Sort(distributionKey, hardDoom, _attrContext, _sortSpec)); PartialResult::LP result(new PartialResult((_offset + _hits), sort->hasSortData())); if (thread_id == 0) { _result = result; diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h index 1d7036237db..0ce87ceb7c6 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h @@ -2,42 +2,45 @@ #pragma once -#include "partial_result.h" -#include "result_processor.h" -#include "sessionmanager.h" -#include <vespa/searchcore/grouping/groupingcontext.h> -#include <vespa/searchcore/grouping/groupingmanager.h> -#include <vespa/searchcore/grouping/groupingsession.h> -#include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/common/sortresults.h> -#include <vespa/searchlib/common/idocumentmetastore.h> -#include <vespa/searchlib/common/resultset.h> #include <vespa/vespalib/util/dual_merge_director.h> -#include <vespa/vespalib/util/noncopyable.hpp> namespace search { -namespace engine { - class SearchReply; -} + namespace engine { + class SearchReply; + } + namespace grouping { + class GroupingContext; + class GroupingSession; + } + class IDocumentMetaStore; } namespace proton { namespace matching { +class SessionManager; +class PartialResult; + class ResultProcessor { + using GroupingContext = search::grouping::GroupingContext; + using GroupingSession = search::grouping::GroupingSession; + using IAttributeContext = search::attribute::IAttributeContext; + using PartialResultLP = vespalib::LinkedPtr<PartialResult>; public: /** * Sorter selection and owner of additional data needed for * multi-level sorting. **/ - struct Sort : vespalib::noncopyable { + struct Sort { typedef std::unique_ptr<Sort> UP; FastS_IResultSorter *sorter; std::unique_ptr<search::common::ConverterFactory> _ucaFactory; FastS_SortSpec sortSpec; - Sort(const search::IDocumentMetaStore & metaStore, const vespalib::Doom & doom, - search::attribute::IAttributeContext &ac, const vespalib::string &ss); + Sort(const Sort &) = delete; + Sort & operator = (const Sort &) = delete; + Sort(uint32_t partitionId, const vespalib::Doom & doom, IAttributeContext &ac, const vespalib::string &ss); bool hasSortData() const { return (sorter == (const FastS_IResultSorter *) &sortSpec); } @@ -47,69 +50,62 @@ public: * Adapter to use grouping contexts as merging sources. **/ struct GroupingSource : vespalib::DualMergeDirector::Source { - search::grouping::GroupingContext *ctx; - GroupingSource(search::grouping::GroupingContext *g) : ctx(g) {} - virtual void merge(Source &s) { - GroupingSource &rhs = static_cast<GroupingSource&>(s); - assert((ctx == 0) == (rhs.ctx == 0)); - if (ctx != 0) { - search::grouping::GroupingManager man(*ctx); - man.merge(*rhs.ctx); - } - } + GroupingContext *ctx; + GroupingSource(GroupingContext *g) : ctx(g) {} + void merge(Source &s) override; }; /** * Context per thread used for result processing. **/ struct Context { - typedef std::unique_ptr<Context> UP; - - Sort::UP sort; - PartialResult::LP result; - search::grouping::GroupingContext::UP grouping; - GroupingSource groupingSource; - - Context(Sort::UP s, PartialResult::LP r, - search::grouping::GroupingContext::UP g) - : sort(std::move(s)), result(r), grouping(std::move(g)), - groupingSource(grouping.get()) - { } + using UP = std::unique_ptr<Context>; + using GroupingContextUP = std::unique_ptr<GroupingContext>; + + Sort::UP sort; + PartialResultLP result; + GroupingContextUP grouping; + GroupingSource groupingSource; + + Context(Sort::UP s, PartialResultLP r, GroupingContextUP g); + ~Context(); }; struct Result { - typedef std::unique_ptr<Result> UP; - Result(std::unique_ptr<search::engine::SearchReply> reply, size_t numFs4Hits); + using UP = std::unique_ptr<Result>; + using SearchReply = search::engine::SearchReply; + Result(std::unique_ptr<SearchReply> reply, size_t numFs4Hits); ~Result(); - std::unique_ptr<search::engine::SearchReply> _reply; + std::unique_ptr<SearchReply> _reply; size_t _numFs4Hits; }; private: - search::attribute::IAttributeContext &_attrContext; + IAttributeContext &_attrContext; const search::IDocumentMetaStore &_metaStore; SessionManager &_sessionMgr; - search::grouping::GroupingContext &_groupingContext; - search::grouping::GroupingSession::UP _groupingSession; + GroupingContext &_groupingContext; + std::unique_ptr<GroupingSession> _groupingSession; const vespalib::string &_sortSpec; size_t _offset; size_t _hits; - PartialResult::LP _result; + PartialResultLP _result; bool _wasMerged; public: - ResultProcessor(search::attribute::IAttributeContext &attrContext, - const search::IDocumentMetaStore &metaStore, - SessionManager &sessionMgr, - search::grouping::GroupingContext &groupingContext, - const search::grouping::SessionId &sessionId, - const vespalib::string &sortSpec, + ResultProcessor(IAttributeContext &attrContext, + const search::IDocumentMetaStore & metaStore, + SessionManager & sessionMgr, + GroupingContext & groupingContext, + const vespalib::string & sessionId, + const vespalib::string & sortSpec, size_t offset, size_t hits); + ~ResultProcessor(); size_t countFS4Hits(); void prepareThreadContextCreation(size_t num_threads); - Context::UP createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id); - Result::UP makeReply(); + Context::UP createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id, uint32_t distributionKey); + std::unique_ptr<Result> makeReply(); }; } // namespace proton::matching diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp index 3d7473d779a..d1cb05292c1 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp @@ -1,15 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.reprocessing.attribute_reprocessing_initializer"); - #include "attribute_reprocessing_initializer.h" #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> -using search::index::Schema; +#include <vespa/log/log.h> +LOG_SETUP(".proton.reprocessing.attribute_reprocessing_initializer"); + +using namespace search::index; using search::AttributeGuard; using search::AttributeVector; using search::SerialNum; @@ -28,9 +28,9 @@ toStr(bool value) return (value ? "true" : "false"); } -bool fastPartialUpdateAttribute(const Schema::DataType &attrType) { +bool fastPartialUpdateAttribute(const schema::DataType &attrType) { // Partial update to tensor or predicate attribute must update document - return ((attrType != Schema::BOOLEANTREE) && (attrType != Schema::TENSOR)); + return ((attrType != schema::BOOLEANTREE) && (attrType != schema::TENSOR)); } @@ -95,7 +95,7 @@ getFieldsToPopulate(const ARIConfig &newCfg, bool inNewSchema = newCfg.getInspector()->hasField(name); // NOTE: If it is a string and index field we shall // keep the original in order to preserve annotations. - bool isStringIndexField = attrField.getDataType() == Schema::STRING && + bool isStringIndexField = attrField.getDataType() == schema::STRING && newCfg.getSchema().isIndexField(name); bool populateField = !inNewAttrMgr && inNewSchema && !isStringIndexField && fastPartialUpdateAttribute(attrType); @@ -103,7 +103,7 @@ getFieldsToPopulate(const ARIConfig &newCfg, "isStringIndexField=%s, dataType=%s, populate=%s", name.c_str(), toStr(inNewAttrMgr), toStr(inNewSchema), toStr(isStringIndexField), - Schema::getTypeName(attrType).c_str(), + schema::getTypeName(attrType).c_str(), toStr(populateField)); if (populateField) { fieldsToPopulate.push_back(IReprocessingRewriter::SP diff --git a/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp b/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp index b1cf96f81ef..fa54ac8ca66 100644 --- a/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp +++ b/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("vespa-index-inspect"); #include <vespa/searchlib/index/dictionaryfile.h> #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/bitcompression/compression.h> @@ -18,6 +16,8 @@ LOG_SETUP("vespa-index-inspect"); #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <iostream> +#include <vespa/log/log.h> +LOG_SETUP("vespa-index-inspect"); using search::index::Schema; using search::index::SchemaUtil; @@ -38,6 +38,7 @@ using search::diskindex::WordNumMapping; using search::diskindex::FieldReader; using search::diskindex::PageDict4FileSeqRead; using search::TuneFileSeqRead; +using namespace search::index; namespace { @@ -410,7 +411,7 @@ ShowPostingListSubApp::readDocIdLimit(const Schema &schema) uint32_t numIndexFields = schema.getNumIndexFields(); for (uint32_t fieldId = 0; fieldId < numIndexFields; ++fieldId) { const Schema::IndexField &field = schema.getIndexField(fieldId); - if (field.getDataType() == Schema::STRING) { + if (field.getDataType() == schema::STRING) { FieldReader fr; if (!fr.open(_indexDir + "/" + field.getName() + "/", tuneFileRead)) diff --git a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp index bf95e3d56a6..33800480492 100644 --- a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp @@ -191,7 +191,7 @@ Test::Test() : _schema(), _indexId(0) { - _schema.addIndexField(Schema::IndexField("f1", Schema::STRING)); + _schema.addIndexField(Schema::IndexField("f1", schema::STRING)); } int diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index ab6be2e0801..7bc4958ac4d 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("fieldwriter_test"); #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/resultset.h> #include <vespa/searchlib/util/rand48.h> @@ -22,6 +20,8 @@ LOG_SETUP("fieldwriter_test"); #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/searchlib/diskindex/pagedict4file.h> #include <vespa/searchlib/diskindex/pagedict4randread.h> +#include <vespa/log/log.h> +LOG_SETUP("fieldwriter_test"); using search::ResultSet; @@ -47,12 +47,12 @@ using search::diskindex::FieldReader; using search::diskindex::DocIdMapping; using search::diskindex::WordNumMapping; using search::diskindex::PageDict4RandRead; +using namespace search::index; // needed to resolve external symbol from httpd.h on AIX -void FastS_block_usr2() {} +void FastS_block_usr2() { } -namespace fieldwriter -{ +namespace fieldwriter { uint32_t minSkipDocs = 64; uint32_t minChunkDocs = 262144; @@ -198,8 +198,8 @@ WrappedFieldWriter::WrappedFieldWriter(const vespalib::string &namepref, _schema(), _indexId() { - Schema::CollectionType ct(Schema::SINGLE); - _schema.addIndexField(Schema::IndexField("field1", Schema::STRING, ct)); + schema::CollectionType ct(schema::SINGLE); + _schema.addIndexField(Schema::IndexField("field1", schema::STRING, ct)); _indexId = _schema.getIndexFieldId("field1"); } @@ -334,12 +334,12 @@ WrappedFieldReader::WrappedFieldReader(const vespalib::string &namepref, _oldSchema(), _schema() { - Schema::CollectionType ct(Schema::SINGLE); + Schema::CollectionType ct(schema::SINGLE); _oldSchema.addIndexField(Schema::IndexField("field1", - Schema::STRING, + schema::STRING, ct)); _schema.addIndexField(Schema::IndexField("field1", - Schema::STRING, + schema::STRING, ct)); } diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index 4191a8f8d2b..ca303cad99b 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -1,7 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("fusion_test"); #include <vespa/searchlib/diskindex/checkpointfile.h> #include <vespa/searchlib/diskindex/fusion.h> #include <vespa/searchlib/diskindex/indexbuilder.h> @@ -27,26 +25,20 @@ LOG_SETUP("fusion_test"); #include <vespa/searchlib/util/filekit.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> -namespace search -{ +namespace search { using document::Document; using fef::FieldPositionsIterator; using fef::TermFieldMatchData; using fef::TermFieldMatchDataArray; -using index::DocBuilder; -using index::DocIdAndFeatures; -using index::Schema; -using index::SchemaUtil; +using namespace index; using search::common::FileHeaderContext; -using search::index::DummyFileHeaderContext; using memoryindex::Dictionary; using memoryindex::DocumentInverter; using queryeval::SearchIterator; -namespace diskindex -{ +namespace diskindex { class Test : public vespalib::TestApp @@ -59,7 +51,6 @@ private: requireThatFusionIsWorking(const vespalib::string &prefix, bool directio, bool readmmap); - public: Test(); int Main(); @@ -269,20 +260,19 @@ Test::requireThatFusionIsWorking(const vespalib::string &prefix, schema.addIndexField(Schema::IndexField(iField.getName(), iField.getDataType(), iField.getCollectionType())); - if (iField.getCollectionType() == Schema::WEIGHTEDSET) + if (iField.getCollectionType() == schema::WEIGHTEDSET) schema2.addIndexField(Schema::IndexField(iField.getName(), iField.getDataType(), - Schema::ARRAY)); + schema::ARRAY)); else schema2.addIndexField(Schema::IndexField(iField.getName(), iField.getDataType(), iField.getCollectionType())); schema3.addIndexField(Schema::IndexField(iField.getName(), iField.getDataType(), - Schema::SINGLE)); + schema::SINGLE)); } - schema3.addIndexField(Schema::IndexField("f4", - Schema::STRING)); + schema3.addIndexField(Schema::IndexField("f4", schema::STRING)); schema.addFieldSet(Schema::FieldSet("nc0"). addField("f0").addField("f1")); schema2.addFieldSet(Schema::FieldSet("nc0"). @@ -467,19 +457,15 @@ Test::requireThatFusionIsWorking(const vespalib::string &prefix, } while (0); } - Test::Test() : _schema() { - _schema.addIndexField(Schema::IndexField("f0", Schema::STRING)); - _schema.addIndexField(Schema::IndexField("f1", Schema::STRING)); - _schema.addIndexField(Schema::IndexField("f2", Schema::STRING, - Schema::ARRAY)); - _schema.addIndexField(Schema::IndexField("f3", Schema::STRING, - Schema::WEIGHTEDSET)); + _schema.addIndexField(Schema::IndexField("f0", schema::STRING)); + _schema.addIndexField(Schema::IndexField("f1", schema::STRING)); + _schema.addIndexField(Schema::IndexField("f2", schema::STRING, schema::ARRAY)); + _schema.addIndexField(Schema::IndexField("f3", schema::STRING, schema::WEIGHTEDSET)); } - int Test::Main() { @@ -499,8 +485,6 @@ Test::Main() } - } - -TEST_APPHOOK(search::diskindex::Test); +TEST_APPHOOK(search::diskindex::Test);
\ No newline at end of file diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp index d8ad1698c9d..37fc78df460 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("pagedict4test"); #include <vespa/searchlib/bitcompression/compression.h> #include <vector> #include <vespa/searchlib/util/rand48.h> @@ -15,6 +13,8 @@ LOG_SETUP("pagedict4test"); #include <vespa/searchlib/diskindex/pagedict4file.h> #include <vespa/searchlib/diskindex/pagedict4randread.h> #include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/log/log.h> +LOG_SETUP("pagedict4test"); using search::bitcompression::PostingListCountFileEncodeContext; using search::bitcompression::PostingListCountFileDecodeContext; @@ -37,6 +37,7 @@ using search::diskindex::PageDict4FileSeqRead; using search::diskindex::PageDict4FileSeqWrite; using search::diskindex::PageDict4RandRead; using search::index::DummyFileHeaderContext; +using namespace search::index; typedef search::bitcompression::PageDict4StartOffset StartOffset; @@ -644,8 +645,8 @@ testWords(const std::string &logname, fn << "f0"; schema.addIndexField(Schema:: IndexField(fn.str(), - Schema::STRING, - Schema::SINGLE)); + schema::STRING, + schema::SINGLE)); indexes.push_back(0); } { diff --git a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp index 460037e0736..446e782f092 100644 --- a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp +++ b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp @@ -46,54 +46,38 @@ void Test::testBuilder() { Schema s; - s.addIndexField(Schema::IndexField("ia", Schema::STRING)); - s.addIndexField(Schema::IndexField("ib", Schema::STRING, Schema::ARRAY)); - s.addIndexField(Schema::IndexField("ic", Schema::STRING, Schema::WEIGHTEDSET)); - s.addUriIndexFields(Schema::IndexField("iu", Schema::STRING)); - s.addUriIndexFields(Schema::IndexField("iau", - Schema::STRING, - Schema::ARRAY)); - s.addUriIndexFields(Schema::IndexField("iwu", - Schema::STRING, - Schema::WEIGHTEDSET)); - s.addAttributeField(Schema::AttributeField("aa", Schema::INT32)); - s.addAttributeField(Schema::AttributeField("ab", Schema::FLOAT)); - s.addAttributeField(Schema::AttributeField("ac", Schema::STRING)); - s.addAttributeField(Schema::AttributeField("ad", Schema::INT32, Schema::ARRAY)); - s.addAttributeField(Schema::AttributeField("ae", Schema::FLOAT, Schema::ARRAY)); - s.addAttributeField(Schema::AttributeField("af", Schema::STRING, Schema::ARRAY)); - s.addAttributeField(Schema::AttributeField("ag", Schema::INT32, Schema::WEIGHTEDSET)); - s.addAttributeField(Schema::AttributeField("ah", Schema::FLOAT, Schema::WEIGHTEDSET)); - s.addAttributeField(Schema::AttributeField("ai", Schema::STRING, Schema::WEIGHTEDSET)); - s.addAttributeField(Schema::AttributeField("asp1", - Schema::INT32)); - s.addAttributeField(Schema::AttributeField("asp2", - Schema::INT64)); - s.addAttributeField(Schema::AttributeField("aap1", - Schema::INT32, - Schema::ARRAY)); - s.addAttributeField(Schema::AttributeField("aap2", - Schema::INT64, - Schema::ARRAY)); - s.addAttributeField(Schema::AttributeField("awp1", - Schema::INT32, - Schema::WEIGHTEDSET)); - s.addAttributeField(Schema::AttributeField("awp2", - Schema::INT64, - Schema::WEIGHTEDSET)); + s.addIndexField(Schema::IndexField("ia", schema::STRING)); + s.addIndexField(Schema::IndexField("ib", schema::STRING, schema::ARRAY)); + s.addIndexField(Schema::IndexField("ic", schema::STRING, schema::WEIGHTEDSET)); + s.addUriIndexFields(Schema::IndexField("iu", schema::STRING)); + s.addUriIndexFields(Schema::IndexField("iau", schema::STRING, schema::ARRAY)); + s.addUriIndexFields(Schema::IndexField("iwu", schema::STRING, schema::WEIGHTEDSET)); + s.addAttributeField(Schema::AttributeField("aa", schema::INT32)); + s.addAttributeField(Schema::AttributeField("ab", schema::FLOAT)); + s.addAttributeField(Schema::AttributeField("ac", schema::STRING)); + s.addAttributeField(Schema::AttributeField("ad", schema::INT32, schema::ARRAY)); + s.addAttributeField(Schema::AttributeField("ae", schema::FLOAT, schema::ARRAY)); + s.addAttributeField(Schema::AttributeField("af", schema::STRING, schema::ARRAY)); + s.addAttributeField(Schema::AttributeField("ag", schema::INT32, schema::WEIGHTEDSET)); + s.addAttributeField(Schema::AttributeField("ah", schema::FLOAT, schema::WEIGHTEDSET)); + s.addAttributeField(Schema::AttributeField("ai", schema::STRING, schema::WEIGHTEDSET)); + s.addAttributeField(Schema::AttributeField("asp1", schema::INT32)); + s.addAttributeField(Schema::AttributeField("asp2", schema::INT64)); + s.addAttributeField(Schema::AttributeField("aap1", schema::INT32, schema::ARRAY)); + s.addAttributeField(Schema::AttributeField("aap2", schema::INT64, schema::ARRAY)); + s.addAttributeField(Schema::AttributeField("awp1", schema::INT32, schema::WEIGHTEDSET)); + s.addAttributeField(Schema::AttributeField("awp2", schema::INT64, schema::WEIGHTEDSET)); - s.addSummaryField(Schema::SummaryField("sa", Schema::INT8)); - s.addSummaryField(Schema::SummaryField("sb", Schema::INT16)); - s.addSummaryField(Schema::SummaryField("sc", Schema::INT32)); - s.addSummaryField(Schema::SummaryField("sd", Schema::INT64)); - s.addSummaryField(Schema::SummaryField("se", Schema::FLOAT)); - s.addSummaryField(Schema::SummaryField("sf", Schema::DOUBLE)); - s.addSummaryField(Schema::SummaryField("sg", Schema::STRING)); - s.addSummaryField(Schema::SummaryField("sh", Schema::RAW)); - s.addSummaryField(Schema::SummaryField("si", Schema::RAW, - Schema::ARRAY)); - s.addSummaryField(Schema::SummaryField("sj", Schema::RAW, - Schema::WEIGHTEDSET)); + s.addSummaryField(Schema::SummaryField("sa", schema::INT8)); + s.addSummaryField(Schema::SummaryField("sb", schema::INT16)); + s.addSummaryField(Schema::SummaryField("sc", schema::INT32)); + s.addSummaryField(Schema::SummaryField("sd", schema::INT64)); + s.addSummaryField(Schema::SummaryField("se", schema::FLOAT)); + s.addSummaryField(Schema::SummaryField("sf", schema::DOUBLE)); + s.addSummaryField(Schema::SummaryField("sg", schema::STRING)); + s.addSummaryField(Schema::SummaryField("sh", schema::RAW)); + s.addSummaryField(Schema::SummaryField("si", schema::RAW, schema::ARRAY)); + s.addSummaryField(Schema::SummaryField("sj", schema::RAW, schema::WEIGHTEDSET)); DocBuilder b(s); Document::UP doc; diff --git a/searchlib/src/tests/index/doctypebuilder/doctypebuilder_test.cpp b/searchlib/src/tests/index/doctypebuilder/doctypebuilder_test.cpp index 86a1b8f3d23..9f04563ad8b 100644 --- a/searchlib/src/tests/index/doctypebuilder/doctypebuilder_test.cpp +++ b/searchlib/src/tests/index/doctypebuilder/doctypebuilder_test.cpp @@ -1,7 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("doctypebuilder_test"); #include <vespa/document/repo/documenttyperepo.h> #include <vespa/searchlib/index/doctypebuilder.h> #include <vespa/vespalib/testkit/testapp.h> @@ -13,26 +11,17 @@ namespace index { TEST("testSearchDocType") { Schema s; - s.addIndexField(Schema::IndexField("ia", Schema::STRING)); - s.addIndexField(Schema::IndexField("ib", Schema::STRING, Schema::ARRAY)); - s.addIndexField(Schema::IndexField("ic", Schema::STRING, Schema::WEIGHTEDSET)); - s.addUriIndexFields(Schema::IndexField("iu", Schema::STRING)); - s.addUriIndexFields(Schema::IndexField("iau", - Schema::STRING, - Schema::ARRAY)); - s.addUriIndexFields(Schema::IndexField("iwu", - Schema::STRING, - Schema::WEIGHTEDSET)); - s.addAttributeField(Schema::AttributeField("aa", Schema::INT32)); - s.addAttributeField(Schema::AttributeField("spos", - Schema::INT64)); - s.addAttributeField(Schema::AttributeField("apos", - Schema::INT64, - Schema::ARRAY)); - s.addAttributeField(Schema::AttributeField("wpos", - Schema::INT64, - Schema::WEIGHTEDSET)); - s.addSummaryField(Schema::SummaryField("sa", Schema::STRING)); + s.addIndexField(Schema::IndexField("ia", schema::STRING)); + s.addIndexField(Schema::IndexField("ib", schema::STRING, schema::ARRAY)); + s.addIndexField(Schema::IndexField("ic", schema::STRING, schema::WEIGHTEDSET)); + s.addUriIndexFields(Schema::IndexField("iu", schema::STRING)); + s.addUriIndexFields(Schema::IndexField("iau", schema::STRING, schema::ARRAY)); + s.addUriIndexFields(Schema::IndexField("iwu", schema::STRING, schema::WEIGHTEDSET)); + s.addAttributeField(Schema::AttributeField("aa", schema::INT32)); + s.addAttributeField(Schema::AttributeField("spos", schema::INT64)); + s.addAttributeField(Schema::AttributeField("apos", schema::INT64, schema::ARRAY)); + s.addAttributeField(Schema::AttributeField("wpos", schema::INT64, schema::WEIGHTEDSET)); + s.addSummaryField(Schema::SummaryField("sa", schema::STRING)); DocTypeBuilder docTypeBuilder(s); document::DocumenttypesConfig config = docTypeBuilder.makeConfig(); @@ -63,10 +52,8 @@ TEST("testSearchDocType") { TEST("require that multiple fields can have the same type") { Schema s; - s.addIndexField(Schema::IndexField("array1", Schema::STRING, - Schema::ARRAY)); - s.addIndexField(Schema::IndexField("array2", Schema::STRING, - Schema::ARRAY)); + s.addIndexField(Schema::IndexField("array1", schema::STRING, schema::ARRAY)); + s.addIndexField(Schema::IndexField("array2", schema::STRING, schema::ARRAY)); DocTypeBuilder docTypeBuilder(s); document::DocumenttypesConfig config = docTypeBuilder.makeConfig(); DocumentTypeRepo repo(config); diff --git a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp b/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp index 87a9ab87cb1..c13aeaa1053 100644 --- a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp @@ -217,10 +217,8 @@ Test::requireThatAddFeaturesTriggersChangeOfBuffer(void) Test::Test() : _schema() { - _schema.addIndexField(Schema::IndexField("f0", Schema::STRING)); - _schema.addIndexField(Schema::IndexField("f1", - Schema::STRING, - Schema::WEIGHTEDSET)); + _schema.addIndexField(Schema::IndexField("f0", schema::STRING)); + _schema.addIndexField(Schema::IndexField("f1", schema::STRING, schema::WEIGHTEDSET)); } diff --git a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp b/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp index 907e983420c..1e0213e5fae 100644 --- a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp +++ b/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp @@ -41,8 +41,7 @@ LOG_SETUP("dictionary_test"); -namespace search -{ +namespace search { using namespace btree; using namespace datastore; @@ -54,8 +53,7 @@ using diskindex::CheckPointFile; using vespalib::GenerationHandler; using test::InitRangeVerifier; -namespace memoryindex -{ +namespace memoryindex { typedef Dictionary::PostingList PostingList; typedef PostingList::Iterator PostingItr; @@ -602,12 +600,10 @@ struct Fixture { Schema _schema; Fixture() : _schema() { - _schema.addIndexField(Schema::IndexField("f0", Schema::STRING)); - _schema.addIndexField(Schema::IndexField("f1", Schema::STRING)); - _schema.addIndexField(Schema::IndexField("f2", Schema::STRING, - Schema::ARRAY)); - _schema.addIndexField(Schema::IndexField("f3", Schema::STRING, - Schema::WEIGHTEDSET)); + _schema.addIndexField(Schema::IndexField("f0", schema::STRING)); + _schema.addIndexField(Schema::IndexField("f1", schema::STRING)); + _schema.addIndexField(Schema::IndexField("f2", schema::STRING, schema::ARRAY)); + _schema.addIndexField(Schema::IndexField("f3", schema::STRING, schema::WEIGHTEDSET)); } const Schema & getSchema() const { return _schema; } }; @@ -1167,14 +1163,9 @@ public: UriFixture() : _schema() { - _schema.addUriIndexFields(Schema::IndexField("iu", - Schema::STRING)); - _schema.addUriIndexFields(Schema::IndexField("iau", - Schema::STRING, - Schema::ARRAY)); - _schema.addUriIndexFields(Schema::IndexField("iwu", - Schema::STRING, - Schema::WEIGHTEDSET)); + _schema.addUriIndexFields(Schema::IndexField("iu", schema::STRING)); + _schema.addUriIndexFields(Schema::IndexField("iau", schema::STRING, schema::ARRAY)); + _schema.addUriIndexFields(Schema::IndexField("iwu", schema::STRING, schema::WEIGHTEDSET)); } const Schema & getSchema() const { return _schema; } }; @@ -1380,7 +1371,7 @@ public: SingleFieldFixture() : _schema() { - _schema.addIndexField(Schema::IndexField("i", Schema::STRING)); + _schema.addIndexField(Schema::IndexField("i", schema::STRING)); } const Schema & getSchema() const { return _schema; } }; diff --git a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp b/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp index d3ad1f54e95..d69780b9516 100644 --- a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp @@ -3,8 +3,6 @@ #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("documentinverter_test"); #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/documentinverter.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> @@ -13,21 +11,17 @@ LOG_SETUP("documentinverter_test"); #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/vespalib/testkit/testapp.h> -namespace search -{ +namespace search { using document::Document; using index::DocBuilder; using index::Schema; +using namespace index; -namespace memoryindex -{ - - -namespace -{ +namespace memoryindex { +namespace { Document::UP makeDoc10(DocBuilder &b) @@ -109,12 +103,10 @@ struct Fixture makeSchema() { Schema schema; - schema.addIndexField(Schema::IndexField("f0", Schema::STRING)); - schema.addIndexField(Schema::IndexField("f1", Schema::STRING)); - schema.addIndexField(Schema::IndexField("f2", Schema::STRING, - Schema::ARRAY)); - schema.addIndexField(Schema::IndexField("f3", Schema::STRING, - Schema::WEIGHTEDSET)); + schema.addIndexField(Schema::IndexField("f0", schema::STRING)); + schema.addIndexField(Schema::IndexField("f1", schema::STRING)); + schema.addIndexField(Schema::IndexField("f2", schema::STRING, schema::ARRAY)); + schema.addIndexField(Schema::IndexField("f3", schema::STRING, schema::WEIGHTEDSET)); return schema; } diff --git a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp b/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp index 6216ba9eb3c..eb86015bb27 100644 --- a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* -*- mode: C++; coding: utf-8; -*- */ - #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("fieldinverter_test"); #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> #include <vespa/vespalib/objects/nbostream.h> @@ -12,20 +8,18 @@ LOG_SETUP("fieldinverter_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/repo/fixedtyperepo.h> -namespace search -{ +namespace search { using document::Document; using index::DocBuilder; using index::Schema; +using namespace index; -namespace memoryindex -{ +namespace memoryindex { -namespace -{ +namespace { Document::UP @@ -117,12 +111,10 @@ struct Fixture makeSchema() { Schema schema; - schema.addIndexField(Schema::IndexField("f0", Schema::STRING)); - schema.addIndexField(Schema::IndexField("f1", Schema::STRING)); - schema.addIndexField(Schema::IndexField("f2", Schema::STRING, - Schema::ARRAY)); - schema.addIndexField(Schema::IndexField("f3", Schema::STRING, - Schema::WEIGHTEDSET)); + schema.addIndexField(Schema::IndexField("f0", schema::STRING)); + schema.addIndexField(Schema::IndexField("f1", schema::STRING)); + schema.addIndexField(Schema::IndexField("f2", schema::STRING, schema::ARRAY)); + schema.addIndexField(Schema::IndexField("f3", schema::STRING, schema::WEIGHTEDSET)); return schema; } diff --git a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp b/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp index 93ae4d9636b..57e638d26a5 100644 --- a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp +++ b/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp @@ -37,8 +37,7 @@ using namespace search::queryeval; struct Setup { Schema schema; Setup &field(const std::string &name) { - schema.addIndexField(Schema::IndexField(name, - Schema::STRING)); + schema.addIndexField(Schema::IndexField(name, schema::STRING)); return *this; } }; diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp index 30b5883f153..9e6f401cd4c 100644 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp @@ -3,8 +3,6 @@ #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("urlfieldinverter_test"); #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> #include <vespa/searchlib/memoryindex/urlfieldinverter.h> @@ -13,25 +11,16 @@ LOG_SETUP("urlfieldinverter_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/repo/fixedtyperepo.h> -namespace search -{ - +namespace search { using document::Document; -using index::DocBuilder; -using index::DocTypeBuilder; -using index::Schema; +using namespace index; -namespace memoryindex -{ + +namespace memoryindex { namespace { const vespalib::string url = "url"; -} - - -namespace -{ Document::UP makeDoc10Single(DocBuilder &b) @@ -200,8 +189,7 @@ struct Fixture makeSchema(Schema::CollectionType collectionType) { Schema schema; - schema.addUriIndexFields(Schema::IndexField("url", Schema::STRING, - collectionType)); + schema.addUriIndexFields(Schema::IndexField("url", index::schema::STRING, collectionType)); return schema; } @@ -258,7 +246,7 @@ struct Fixture }; -TEST_F("requireThatSingleUrlFieldWorks", Fixture(Schema::SINGLE)) +TEST_F("requireThatSingleUrlFieldWorks", Fixture(schema::SINGLE)) { f.invertDocument(10, *makeDoc10Single(f._b)); f.pushDocuments(); @@ -297,7 +285,7 @@ TEST_F("requireThatSingleUrlFieldWorks", Fixture(Schema::SINGLE)) } -TEST_F("requireThatArrayUrlFieldWorks", Fixture(Schema::ARRAY)) +TEST_F("requireThatArrayUrlFieldWorks", Fixture(schema::ARRAY)) { f.invertDocument(10, *makeDoc10Array(f._b)); f.pushDocuments(); @@ -340,7 +328,7 @@ TEST_F("requireThatArrayUrlFieldWorks", Fixture(Schema::ARRAY)) f._inserter.toStr()); } -TEST_F("requireThatWeightedSetFieldWorks", Fixture(Schema::WEIGHTEDSET)) +TEST_F("requireThatWeightedSetFieldWorks", Fixture(schema::WEIGHTEDSET)) { f.invertDocument(10, *makeDoc10WeightedSet(f._b)); f.pushDocuments(); @@ -385,7 +373,7 @@ TEST_F("requireThatWeightedSetFieldWorks", Fixture(Schema::WEIGHTEDSET)) f._inserter.toStr()); } -TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(Schema::SINGLE)) +TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(schema::SINGLE)) { f.enableAnnotations(); f.invertDocument(10, *makeDoc10Single(f._b)); @@ -426,7 +414,7 @@ TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(Schema::SINGLE)) } -TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(Schema::ARRAY)) +TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(schema::ARRAY)) { f.enableAnnotations(); f.invertDocument(10, *makeDoc10Array(f._b)); @@ -472,7 +460,7 @@ TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(Schema::ARRAY)) } TEST_F("requireThatAnnotatedWeightedSetFieldWorks", - Fixture(Schema::WEIGHTEDSET)) + Fixture(schema::WEIGHTEDSET)) { f.enableAnnotations(); f._inserter.setVerbose(); @@ -521,15 +509,14 @@ TEST_F("requireThatAnnotatedWeightedSetFieldWorks", } -TEST_F("requireThatEmptySingleFieldWorks", Fixture(Schema::SINGLE)) +TEST_F("requireThatEmptySingleFieldWorks", Fixture(schema::SINGLE)) { f.invertDocument(10, *makeDoc10Empty(f._b)); f.pushDocuments(); - EXPECT_EQUAL("", - f._inserter.toStr()); + EXPECT_EQUAL("", f._inserter.toStr()); } -TEST_F("requireThatEmptyArrayFieldWorks", Fixture(Schema::ARRAY)) +TEST_F("requireThatEmptyArrayFieldWorks", Fixture(schema::ARRAY)) { f.invertDocument(10, *makeDoc10Empty(f._b)); f.pushDocuments(); @@ -537,40 +524,35 @@ TEST_F("requireThatEmptyArrayFieldWorks", Fixture(Schema::ARRAY)) f._inserter.toStr()); } -TEST_F("requireThatEmptyWeightedSetFieldWorks", Fixture(Schema::WEIGHTEDSET)) +TEST_F("requireThatEmptyWeightedSetFieldWorks", Fixture(schema::WEIGHTEDSET)) { f.invertDocument(10, *makeDoc10Empty(f._b)); f.pushDocuments(); - EXPECT_EQUAL("", - f._inserter.toStr()); + EXPECT_EQUAL("", f._inserter.toStr()); } -TEST_F("requireThatAnnotatedEmptySingleFieldWorks", Fixture(Schema::SINGLE)) +TEST_F("requireThatAnnotatedEmptySingleFieldWorks", Fixture(schema::SINGLE)) { f.enableAnnotations(); f.invertDocument(10, *makeDoc10Empty(f._b)); f.pushDocuments(); - EXPECT_EQUAL("", - f._inserter.toStr()); + EXPECT_EQUAL("", f._inserter.toStr()); } -TEST_F("requireThatAnnotatedEmptyArrayFieldWorks", Fixture(Schema::ARRAY)) +TEST_F("requireThatAnnotatedEmptyArrayFieldWorks", Fixture(schema::ARRAY)) { f.enableAnnotations(); f.invertDocument(10, *makeDoc10Empty(f._b)); f.pushDocuments(); - EXPECT_EQUAL("", - f._inserter.toStr()); + EXPECT_EQUAL("", f._inserter.toStr()); } -TEST_F("requireThatAnnotatedEmptyWeightedSetFieldWorks", - Fixture(Schema::WEIGHTEDSET)) +TEST_F("requireThatAnnotatedEmptyWeightedSetFieldWorks", Fixture(schema::WEIGHTEDSET)) { f.enableAnnotations(); f.invertDocument(10, *makeDoc10Empty(f._b)); f.pushDocuments(); - EXPECT_EQUAL("", - f._inserter.toStr()); + EXPECT_EQUAL("", f._inserter.toStr()); } } // namespace memoryindex diff --git a/searchlib/src/tests/postinglistbm/postinglistbm.cpp b/searchlib/src/tests/postinglistbm/postinglistbm.cpp index fc93eb42dcd..14ff86d4db0 100644 --- a/searchlib/src/tests/postinglistbm/postinglistbm.cpp +++ b/searchlib/src/tests/postinglistbm/postinglistbm.cpp @@ -3,8 +3,6 @@ // Copyright (C) 2003 Overture Services Norway AS #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("postinglistbm"); #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/resultset.h> #include <vespa/searchlib/util/rand48.h> @@ -19,7 +17,7 @@ using search::ResultSet; using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataArray; using search::queryeval::SearchIterator; -using search::index::Schema; +using namespace search::index; using namespace search::fakedata; // needed to resolve external symbol from httpd.h on AIX @@ -374,8 +372,8 @@ PostingListBM::Main(void) do { Schema schema; Schema::IndexField indexField("field0", - Schema::STRING, - Schema::SINGLE); + schema::STRING, + schema::SINGLE); schema.addIndexField(indexField); std::unique_ptr<FPFactory> ff(getFPFactory(optArg, schema)); if (ff.get() == NULL) { diff --git a/searchlib/src/tests/sortspec/multilevelsort.cpp b/searchlib/src/tests/sortspec/multilevelsort.cpp index 5f35cf65711..697c434c473 100644 --- a/searchlib/src/tests/sortspec/multilevelsort.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort.cpp @@ -253,7 +253,7 @@ MultilevelSortTest::sortAndCheck(const std::vector<Spec> &spec, uint32_t num, vespalib::Clock clock; vespalib::Doom doom(clock, std::numeric_limits<long>::max()); search::uca::UcaConverterFactory ucaFactory; - FastS_SortSpec sorter(nullptr, doom, ucaFactory, _sortMethod); + FastS_SortSpec sorter(7, doom, ucaFactory, _sortMethod); // init sorter with sort data for(uint32_t i = 0; i < spec.size(); ++i) { AttributeGuard ag; diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp index ebf4ff59889..68b7b7513b0 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp @@ -3,8 +3,6 @@ // Copyright (C) 2003 Overture Services Norway AS #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".posocccompression"); #include "compression.h" #include "posocccompression.h" #include <vespa/searchlib/index/schemautil.h> @@ -12,6 +10,8 @@ LOG_SETUP(".posocccompression"); #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/log/log.h> +LOG_SETUP(".posocccompression"); using search::index::DocIdAndFeatures; using search::index::WordDocElementFeatures; @@ -21,29 +21,25 @@ using search::index::SchemaUtil; using search::index::Schema; using search::fef::TermFieldMatchData; using vespalib::GenericHeader; +using namespace search::index; -namespace -{ +namespace { vespalib::string PosOccId = "PosOcc.3"; - vespalib::string PosOccIdCooked = "PosOcc.3.Cooked"; } -namespace -{ +namespace { vespalib::string EG64PosOccId = "EG64PosOcc.3"; // Dynamic k values vespalib::string EG64PosOccId2 = "EG64PosOcc.2"; // Fixed k values } -namespace search -{ +namespace search { -namespace bitcompression -{ +namespace bitcompression { PosOccFieldParams::PosOccFieldParams(void) @@ -53,8 +49,7 @@ PosOccFieldParams::PosOccFieldParams(void) _avgElemLen(512), _collectionType(SINGLE), _name() -{ -} +{ } bool @@ -136,24 +131,23 @@ PosOccFieldParams::setSchemaParams(const Schema &schema, uint32_t fieldId) assert(fieldId < schema.getNumIndexFields()); const Schema::IndexField &field = schema.getIndexField(fieldId); switch (field.getCollectionType()) { - case Schema::SINGLE: + case schema::SINGLE: _collectionType = SINGLE; _hasElements = false; _hasElementWeights = false; break; - case Schema::ARRAY: + case schema::ARRAY: _collectionType = ARRAY; _hasElements = true; _hasElementWeights = false; break; - case Schema::WEIGHTEDSET: + case schema::WEIGHTEDSET: _collectionType = WEIGHTEDSET; _hasElements = true; _hasElementWeights = true; break; default: - LOG(error, - "Bad collection type"); + LOG(error, "Bad collection type"); abort(); } _avgElemLen = field.getAvgElemLen(); @@ -169,27 +163,25 @@ PosOccFieldParams::readHeader(const vespalib::GenericHeader &header, vespalib::string collKey(prefix + "collectionType"); vespalib::string avgElemLenKey(prefix + "avgElemLen"); _name = header.getTag(nameKey).asString(); - Schema::CollectionType ct = - Schema::collectionTypeFromName(header.getTag(collKey).asString()); + Schema::CollectionType ct = schema::collectionTypeFromName(header.getTag(collKey).asString()); switch (ct) { - case Schema::SINGLE: + case schema::SINGLE: _collectionType = SINGLE; _hasElements = false; _hasElementWeights = false; break; - case Schema::ARRAY: + case schema::ARRAY: _collectionType = ARRAY; _hasElements = true; _hasElementWeights = false; break; - case Schema::WEIGHTEDSET: + case schema::WEIGHTEDSET: _collectionType = WEIGHTEDSET; _hasElements = true; _hasElementWeights = true; break; default: - LOG(error, - "Bad collection type when reading field param in header"); + LOG(error, "Bad collection type when reading field param in header"); abort(); } _avgElemLen = header.getTag(avgElemLenKey).asInteger(); @@ -204,23 +196,23 @@ PosOccFieldParams::writeHeader(vespalib::GenericHeader &header, vespalib::string collKey(prefix + "collectionType"); vespalib::string avgElemLenKey(prefix + "avgElemLen"); header.putTag(GenericHeader::Tag(nameKey, _name)); - Schema::CollectionType ct(Schema::SINGLE); + Schema::CollectionType ct(schema::SINGLE); switch (_collectionType) { case SINGLE: - ct = Schema::SINGLE; + ct = schema::SINGLE; break; case ARRAY: - ct = Schema::ARRAY; + ct = schema::ARRAY; break; case WEIGHTEDSET: - ct = Schema::WEIGHTEDSET; + ct = schema::WEIGHTEDSET; break; default: LOG(error, "Bad collection type when writing field param in header"); abort(); } - header.putTag(GenericHeader::Tag(collKey, Schema::getTypeName(ct))); + header.putTag(GenericHeader::Tag(collKey, schema::getTypeName(ct))); header.putTag(GenericHeader::Tag(avgElemLenKey, _avgElemLen)); } diff --git a/searchlib/src/vespa/searchlib/common/resultset.h b/searchlib/src/vespa/searchlib/common/resultset.h index 46129b1fb57..1ea32cc9cee 100644 --- a/searchlib/src/vespa/searchlib/common/resultset.h +++ b/searchlib/src/vespa/searchlib/common/resultset.h @@ -5,10 +5,11 @@ #pragma once #include <vespa/searchlib/common/rankedhit.h> -#include <vespa/searchlib/common/bitvector.h> +#include <vespa/vespalib/util/alloc.h> -namespace search -{ +namespace search { + +class BitVector; class ResultSet { @@ -17,32 +18,32 @@ private: unsigned int _elemsUsedInRankedHitsArray; unsigned int _rankedHitsArrayAllocElements; - BitVector::UP _bitOverflow; - vespalib::alloc::Alloc _rankedHitsArray; + std::unique_ptr<BitVector> _bitOverflow; + vespalib::alloc::Alloc _rankedHitsArray; public: typedef std::unique_ptr<ResultSet> UP; typedef std::shared_ptr<ResultSet> SP; - ResultSet(void); + ResultSet(); ResultSet(const ResultSet &); // Used only for testing ..... - virtual ~ResultSet(void); + virtual ~ResultSet(); void allocArray(unsigned int arrayAllocated); void setArrayUsed(unsigned int arrayUsed); - void setBitOverflow(BitVector::UP newBitOverflow); - const RankedHit * getArray(void) const { return static_cast<const RankedHit *>(_rankedHitsArray.get()); } - RankedHit * getArray(void) { return static_cast<RankedHit *>(_rankedHitsArray.get()); } - unsigned int getArrayUsed(void) const { return _elemsUsedInRankedHitsArray; } - unsigned int getArrayAllocated(void) const { return _rankedHitsArrayAllocElements; } + void setBitOverflow(std::unique_ptr<BitVector> newBitOverflow); + const RankedHit * getArray() const { return static_cast<const RankedHit *>(_rankedHitsArray.get()); } + RankedHit * getArray() { return static_cast<RankedHit *>(_rankedHitsArray.get()); } + unsigned int getArrayUsed() const { return _elemsUsedInRankedHitsArray; } + unsigned int getArrayAllocated() const { return _rankedHitsArrayAllocElements; } - const BitVector * getBitOverflow(void) const { return _bitOverflow.get(); } - BitVector * getBitOverflow(void) { return _bitOverflow.get(); } - unsigned int getNumHits(void) const; - void mergeWithBitOverflow(void); + const BitVector * getBitOverflow() const { return _bitOverflow.get(); } + BitVector * getBitOverflow() { return _bitOverflow.get(); } + unsigned int getNumHits() const; + void mergeWithBitOverflow(); /* isEmpty() is allowed to return false even if bitmap has no hits */ - bool isEmpty(void) const { return (_bitOverflow == NULL && _elemsUsedInRankedHitsArray == 0); } + bool isEmpty() const { return (_bitOverflow == NULL && _elemsUsedInRankedHitsArray == 0); } }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index 155929b661b..b7fdd0b85af 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -7,9 +7,8 @@ #include <vespa/searchlib/util/sort.h> #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/common/bitvector.h> -#include <vespa/vespalib/util/array.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/document/base/globalid.h> -#include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/log/log.h> LOG_SETUP(".search.attribute.sortresults"); @@ -229,7 +228,7 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) size_t variableWidth = 0; for (auto iter = _vectors.begin(); iter != _vectors.end(); ++iter) { if (iter->_type >= ASC_DOCID) { // doc id - fixedWidth += (_metaStore != nullptr) ? 8 : 4; + fixedWidth += sizeof(uint32_t) + sizeof(uint16_t); }else if (iter->_type >= ASC_RANK) { // rank value fixedWidth += sizeof(search::HitRank); } else { @@ -253,30 +252,20 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) uint32_t len = 0; for (auto iter = _vectors.begin(); iter != _vectors.end(); ++iter) { int written(0); - if (available < std::max(sizeof(hits->_docId), sizeof(hits->_rankValue))) { + if (available < std::max(sizeof(hits->_docId) + sizeof(_partitionId), sizeof(hits->_rankValue))) { mySortData = realloc(n, variableWidth, available, dataSize, mySortData); } do { switch (iter->_type) { case ASC_DOCID: - if (_metaStore) { - _metaStore->getGidEvenIfMoved(hits[i].getDocId(), gid); - serializeForSort<convertForSort<uint64_t, true> >(*reinterpret_cast<const uint64_t *>(gid.get()), mySortData); - written = sizeof(uint64_t); - } else { - serializeForSort<convertForSort<uint32_t, true> >(hits[i].getDocId(), mySortData); - written = sizeof(hits->_docId); - } + serializeForSort<convertForSort<uint32_t, true> >(hits[i].getDocId(), mySortData); + serializeForSort<convertForSort<uint16_t, true> >(_partitionId, mySortData + sizeof(hits->_docId)); + written = sizeof(hits->_docId) + sizeof(_partitionId); break; case DESC_DOCID: - if (_metaStore) { - _metaStore->getGidEvenIfMoved(hits[i].getDocId(), gid); - serializeForSort<convertForSort<uint64_t, false> >(*reinterpret_cast<const uint64_t *>(gid.get()), mySortData); - written = sizeof(uint64_t); - } else { - serializeForSort<convertForSort<uint32_t, false> >(hits[i].getDocId(), mySortData); - written = sizeof(hits->_docId); - } + serializeForSort<convertForSort<uint32_t, false> >(hits[i].getDocId(), mySortData); + serializeForSort<convertForSort<uint16_t, false> >(_partitionId, mySortData + sizeof(hits->_docId)); + written = sizeof(hits->_docId) + sizeof(_partitionId); break; case ASC_RANK: serializeForSort<convertForSort<search::HitRank, true> >(hits[i]._rankValue, mySortData); @@ -312,8 +301,8 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) } -FastS_SortSpec::FastS_SortSpec(const search::IDocumentMetaStore * metaStore, const Doom & doom, const ConverterFactory & ucaFactory, int method) : - _metaStore(metaStore), +FastS_SortSpec::FastS_SortSpec(uint32_t partitionId, const Doom & doom, const ConverterFactory & ucaFactory, int method) : + _partitionId(partitionId), _doom(doom), _ucaFactory(ucaFactory), _method(method), diff --git a/searchlib/src/vespa/searchlib/common/sortresults.h b/searchlib/src/vespa/searchlib/common/sortresults.h index e4d8a4d6510..bff8dbf199b 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.h +++ b/searchlib/src/vespa/searchlib/common/sortresults.h @@ -6,7 +6,6 @@ #include <vespa/searchlib/common/rankedhit.h> #include <vespa/searchlib/common/sortspec.h> -#include <vespa/searchcommon/attribute/iattributecontext.h> #include <algorithm> #include <vector> #include <vespa/vespalib/util/array.h> @@ -15,7 +14,12 @@ #define PREFETCH 64 #define INSERT_SORT_LEVEL 80 -namespace search { class IDocumentMetaStore; } +namespace search { + namespace attribute { + class IAttributeContext; + class IAttributeVector; + } +} /** * Sort the given array of results. * @@ -23,8 +27,7 @@ namespace search { class IDocumentMetaStore; } * @param n the number of hits * @param ntop the number of hits needed in correct order **/ -void FastS_SortResults(search::RankedHit a[], - unsigned int n, unsigned int ntop); +void FastS_SortResults(search::RankedHit a[], unsigned int n, unsigned int ntop); //----------------------------------------------------------------------------- @@ -46,8 +49,7 @@ struct FastS_IResultSorter { * @param n the number of hits * @param ntop the number of hits needed in correct order **/ - virtual void sortResults(search::RankedHit a[], uint32_t n, - uint32_t ntop) = 0; + virtual void sortResults(search::RankedHit a[], uint32_t n, uint32_t ntop) = 0; }; //----------------------------------------------------------------------------- @@ -59,10 +61,8 @@ private: public: static FastS_DefaultResultSorter *instance() { return &__instance; } - virtual bool completeSort() const { return false; } - virtual void sortResults(search::RankedHit a[], uint32_t n, - uint32_t ntop) - { + bool completeSort() const override { return false; } + void sortResults(search::RankedHit a[], uint32_t n, uint32_t ntop) override { return FastS_SortResults(a, n, ntop); } }; @@ -76,8 +76,8 @@ private: public: static FastS_DocIdResultSorter *Instance() { return &__instance; } - virtual bool completeSort() const { return true; } - virtual void sortResults(search::RankedHit[], uint32_t, uint32_t) { + bool completeSort() const override { return true; } + void sortResults(search::RankedHit[], uint32_t, uint32_t) override { // already sorted on docid } }; @@ -104,8 +104,7 @@ public: : _type(type), _vector(vector), _converter(converter) - { - } + { } uint32_t _type; const search::attribute::IAttributeVector *_vector; const search::common::BlobConverter *_converter; @@ -123,7 +122,7 @@ private: typedef vespalib::Array<uint8_t> BinarySortData; typedef vespalib::Array<SortData> SortDataArray; using ConverterFactory = search::common::ConverterFactory; - const search::IDocumentMetaStore * _metaStore; + uint16_t _partitionId; vespalib::Doom _doom; const ConverterFactory & _ucaFactory; int _method; @@ -139,8 +138,8 @@ private: public: FastS_SortSpec(const FastS_SortSpec &) = delete; FastS_SortSpec & operator = (const FastS_SortSpec &) = delete; - FastS_SortSpec(const search::IDocumentMetaStore * metaStore, const vespalib::Doom & doom, const ConverterFactory & ucaFactory, int method=2); - virtual ~FastS_SortSpec(); + FastS_SortSpec(uint32_t partitionId, const vespalib::Doom & doom, const ConverterFactory & ucaFactory, int method=2); + ~FastS_SortSpec(); std::pair<const char *, size_t> getSortRef(size_t i) const { return std::pair<const char *, size_t>((const char*)(&_binarySortData[0] + _sortDataArray[i]._idx), diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index eb6e4c9dad5..894dc45472f 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -32,12 +32,9 @@ using search::docsummary::DocumentSummary; using vespalib::getLastErrorString; -namespace search -{ - -namespace diskindex -{ +namespace search { +namespace diskindex { void FusionInputIndex::setSchema(const Schema::SP &schema) @@ -56,8 +53,7 @@ Fusion::Fusion(bool dynamicKPosIndexFormat, _outDir("merged"), _tuneFileIndexing(tuneFileIndexing), _fileHeaderContext(fileHeaderContext) -{ -} +{ } Fusion::~Fusion() @@ -405,7 +401,7 @@ Fusion::ReadMappingFiles(const SchemaUtil::IndexIterator *index) std::vector<uint32_t> oldIndexes; const Schema &oldSchema = oi.getSchema(); if (!SchemaUtil::getIndexIds(oldSchema, - Schema::STRING, + index::schema::STRING, oldIndexes)) return false; if (oldIndexes.empty()) { @@ -417,8 +413,7 @@ Fusion::ReadMappingFiles(const SchemaUtil::IndexIterator *index) } // Open word mapping file - vespalib::string old2newname = oi.getTmpPath() + - "/old2new.dat"; + vespalib::string old2newname = oi.getTmpPath() + "/old2new.dat"; wordNumMapping.readMappingFile(old2newname, _tuneFileIndexing._read); } diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index 0ec75cfbdf2..f1c34ceac43 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "indexbuilder.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/schemautil.h> @@ -9,17 +8,15 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/searchlib/diskindex/fieldwriter.h> +#include <vespa/log/log.h> LOG_SETUP(".diskindex.indexbuilder"); -namespace search -{ +namespace search { -namespace diskindex -{ +namespace diskindex { -namespace -{ +namespace { using index::DocIdAndFeatures; using index::PostingListCounts; @@ -545,7 +542,7 @@ IndexBuilder::IndexBuilder(const Schema &schema) const Schema::IndexField &iField = schema.getIndexField(i); FieldHandle fh(schema, i, this); // Only know how to handle string index for now. - if (iField.getDataType() == Schema::STRING) + if (iField.getDataType() == index::schema::STRING) fh.setValid(); _fields.push_back(fh); } diff --git a/searchlib/src/vespa/searchlib/fef/fieldinfo.h b/searchlib/src/vespa/searchlib/fef/fieldinfo.h index 8c0625f0c27..ca6796a49d0 100644 --- a/searchlib/src/vespa/searchlib/fef/fieldinfo.h +++ b/searchlib/src/vespa/searchlib/fef/fieldinfo.h @@ -5,7 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include "fieldtype.h" #include "collection_type.h" -#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchcommon/common/datatype.h> namespace search { namespace fef { @@ -19,7 +19,7 @@ const uint32_t IllegalFieldId = 0xffffffff; class FieldInfo { public: - using DataType = search::index::Schema::DataType; + using DataType = search::index::schema::DataType; typedef vespalib::string string; private: FieldType _type; diff --git a/searchlib/src/vespa/searchlib/fef/fieldpositionsiterator.h b/searchlib/src/vespa/searchlib/fef/fieldpositionsiterator.h index 933858b79c9..fcad7f34ac9 100644 --- a/searchlib/src/vespa/searchlib/fef/fieldpositionsiterator.h +++ b/searchlib/src/vespa/searchlib/fef/fieldpositionsiterator.h @@ -2,7 +2,6 @@ #pragma once -#include <vector> #include "termfieldmatchdataposition.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h index a3ce0ac4bb6..61286ba861f 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h @@ -2,11 +2,9 @@ #pragma once -#include "termfieldmatchdataposition.h" #include "fieldpositionsiterator.h" #include "fieldinfo.h" #include <vespa/searchlib/common/feature.h> -#include <vespa/vespalib/util/noncopyable.hpp> #include <string.h> #include <assert.h> diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataposition.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataposition.h index b7f82819bfb..601d37bfe52 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataposition.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataposition.h @@ -17,15 +17,13 @@ public: TermFieldMatchDataPositionKey() : _elementId(0u), _position(0u) - { - } + { } TermFieldMatchDataPositionKey(uint32_t elementId, uint32_t position) : _elementId(elementId), _position(position) - { - } + { } uint32_t getElementId() const { return _elementId; } uint32_t getPosition() const { return _position; } @@ -61,8 +59,7 @@ public: _elementLen(SEARCHLIB_FEF_UNKNOWN_FIELD_LENGTH), _matchLength(1), _matchExactness(1.0) - { - } + { } const TermFieldMatchDataPositionKey &key() const { return *this; @@ -89,8 +86,7 @@ public: _elementLen(elementLen), _matchLength(1), _matchExactness(1.0) - { - } + { } int32_t getElementWeight() const { return _elementWeight; } uint32_t getElementLen() const { return _elementLen; } diff --git a/searchlib/src/vespa/searchlib/index/docbuilder.cpp b/searchlib/src/vespa/searchlib/index/docbuilder.cpp index fc362b6a306..d88d56be958 100644 --- a/searchlib/src/vespa/searchlib/index/docbuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/docbuilder.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.docbuilder"); #include "docbuilder.h" #include "doctypebuilder.h" #include <vespa/document/datatype/annotationtype.h> @@ -26,8 +24,8 @@ namespace { void insertStr(const Schema::Field & sfield, document::FieldValue * fvalue, const vespalib::string & val) { - if (sfield.getDataType() == Schema::STRING || - sfield.getDataType() == Schema::RAW) + if (sfield.getDataType() == schema::STRING || + sfield.getDataType() == schema::RAW) { (dynamic_cast<LiteralFieldValueB *>(fvalue))->setValue(val); } else { @@ -38,13 +36,13 @@ insertStr(const Schema::Field & sfield, document::FieldValue * fvalue, const ves void insertInt(const Schema::Field & sfield, document::FieldValue * fvalue, int64_t val) { - if (sfield.getDataType() == Schema::INT8) { + if (sfield.getDataType() == schema::INT8) { (dynamic_cast<ByteFieldValue *>(fvalue))->setValue((uint8_t)val); - } else if (sfield.getDataType() == Schema::INT16) { + } else if (sfield.getDataType() == schema::INT16) { (dynamic_cast<ShortFieldValue *>(fvalue))->setValue((int16_t)val); - } else if (sfield.getDataType() == Schema::INT32) { + } else if (sfield.getDataType() == schema::INT32) { (dynamic_cast<IntFieldValue *>(fvalue))->setValue((int32_t)val); - } else if (sfield.getDataType() == Schema::INT64) { + } else if (sfield.getDataType() == schema::INT64) { (dynamic_cast<LongFieldValue *>(fvalue))->setValue(val); } else { throw DocBuilder::Error(vespalib::make_string("Field '%s' not compatible", sfield.getName().c_str())); @@ -54,9 +52,9 @@ insertInt(const Schema::Field & sfield, document::FieldValue * fvalue, int64_t v void insertFloat(const Schema::Field & sfield, document::FieldValue * fvalue, double val) { - if (sfield.getDataType() == Schema::FLOAT) { + if (sfield.getDataType() == schema::FLOAT) { (dynamic_cast<FloatFieldValue *>(fvalue))->setValue((float)val); - } else if (sfield.getDataType() == Schema::DOUBLE) { + } else if (sfield.getDataType() == schema::DOUBLE) { (dynamic_cast<DoubleFieldValue *>(fvalue))->setValue(val); } else { throw DocBuilder::Error(vespalib::make_string("Field '%s' not compatible", sfield.getName().c_str())); @@ -66,7 +64,7 @@ insertFloat(const Schema::Field & sfield, document::FieldValue * fvalue, double void insertPredicate(const Schema::Field &sfield, document::FieldValue *fvalue, std::unique_ptr<vespalib::Slime> val) { - if (sfield.getDataType() == Schema::BOOLEANTREE) { + if (sfield.getDataType() == schema::BOOLEANTREE) { *(dynamic_cast<PredicateFieldValue *>(fvalue)) = PredicateFieldValue(std::move(val)); } else { @@ -79,7 +77,7 @@ void insertPredicate(const Schema::Field &sfield, void insertTensor(const Schema::Field &schemaField, document::FieldValue *fvalue, std::unique_ptr<vespalib::tensor::Tensor> val) { - if (schemaField.getDataType() == Schema::TENSOR) { + if (schemaField.getDataType() == schema::TENSOR) { *(dynamic_cast<TensorFieldValue *>(fvalue)) = std::move(val); } else { throw DocBuilder::Error(vespalib::make_string( @@ -93,7 +91,7 @@ insertPosition(const Schema::Field & sfield, document::FieldValue * fvalue, int32_t xpos, int32_t ypos) { assert(*fvalue->getDataType() == *DataType::LONG); - assert(sfield.getDataType() == Schema::INT64); + assert(sfield.getDataType() == schema::INT64); (void) sfield; int64_t zpos = ZCurve::encode(xpos, ypos); document::LongFieldValue *zvalue = @@ -107,7 +105,7 @@ insertRaw(const Schema::Field & sfield, document::FieldValue *fvalue, const void *buf, size_t len) { assert(*fvalue->getDataType() == *DataType::RAW); - assert(sfield.getDataType() == Schema::RAW); + assert(sfield.getDataType() == schema::RAW); (void) sfield; document::RawFieldValue *rfvalue = dynamic_cast<RawFieldValue *>(fvalue); @@ -199,12 +197,11 @@ DocBuilder::CollectionFieldHandle::startElement(int32_t weight) void DocBuilder::CollectionFieldHandle::endElement() { - if (_sfield.getCollectionType() == Schema::ARRAY) { + if (_sfield.getCollectionType() == schema::ARRAY) { onEndElement(); ArrayFieldValue * value = dynamic_cast<ArrayFieldValue *>(_value.get()); value->add(*_element); - } else if (_sfield.getCollectionType() == - Schema::WEIGHTEDSET) { + } else if (_sfield.getCollectionType() == schema::WEIGHTEDSET) { onEndElement(); WeightedSetFieldValue * value = dynamic_cast<WeightedSetFieldValue *>(_value.get()); value->add(*_element, _elementWeight); @@ -232,7 +229,7 @@ DocBuilder::IndexFieldHandle::IndexFieldHandle(const FixedTypeRepo & repo, const { _str.reserve(1023); - if (_sfield.getCollectionType() == Schema::SINGLE) { + if (_sfield.getCollectionType() == schema::SINGLE) { if (*_value->getDataType() == document::UrlDataType::getInstance()) _uriField = true; } else { @@ -415,7 +412,7 @@ DocBuilder::IndexFieldHandle::onEndElement(void) if (_uriField) return; StringFieldValue * value; - if (_sfield.getCollectionType() != Schema::SINGLE) { + if (_sfield.getCollectionType() != schema::SINGLE) { value = dynamic_cast<StringFieldValue *>(_element.get()); } else { value = dynamic_cast<StringFieldValue *>(_value.get()); @@ -442,7 +439,7 @@ DocBuilder::IndexFieldHandle::onEndElement(void) void DocBuilder::IndexFieldHandle::onEndField(void) { - if (_sfield.getCollectionType() == Schema::SINGLE) + if (_sfield.getCollectionType() == schema::SINGLE) onEndElement(); } @@ -486,7 +483,7 @@ DocBuilder::IndexFieldHandle::endSubField(void) assert(!_subField.empty()); assert(_uriField); StructuredFieldValue *sValue; - if (_sfield.getCollectionType() != Schema::SINGLE) { + if (_sfield.getCollectionType() != schema::SINGLE) { sValue = dynamic_cast<StructFieldValue *>(_element.get()); } else { sValue = dynamic_cast<StructFieldValue *>(_value.get()); diff --git a/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp b/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp index fff8a735bf9..00ae2269609 100644 --- a/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp @@ -13,39 +13,34 @@ LOG_SETUP(".index.doctypebuilder"); using namespace document; using namespace search::index; -namespace search -{ - -namespace index -{ - -namespace -{ +namespace search { +namespace index { +namespace { const DataType *convert(Schema::DataType type) { switch (type) { - case Schema::UINT1: - case Schema::UINT2: - case Schema::UINT4: - case Schema::INT8: + case schema::UINT1: + case schema::UINT2: + case schema::UINT4: + case schema::INT8: return DataType::BYTE; - case Schema::INT16: + case schema::INT16: return DataType::SHORT; - case Schema::INT32: + case schema::INT32: return DataType::INT; - case Schema::INT64: + case schema::INT64: return DataType::LONG; - case Schema::FLOAT: + case schema::FLOAT: return DataType::FLOAT; - case Schema::DOUBLE: + case schema::DOUBLE: return DataType::DOUBLE; - case Schema::STRING: + case schema::STRING: return DataType::STRING; - case Schema::RAW: + case schema::RAW: return DataType::RAW; - case Schema::BOOLEANTREE: + case schema::BOOLEANTREE: return DataType::PREDICATE; - case Schema::TENSOR: + case schema::TENSOR: return DataType::TENSOR; default: break; @@ -100,7 +95,7 @@ DocTypeBuilder::UriField::valid(const Schema &schema, if (fieldId == Schema::UNKNOWN_FIELD_ID) return false; const Schema::IndexField &field = schema.getIndexField(fieldId); - if (field.getDataType() != Schema::STRING) + if (field.getDataType() != schema::STRING) return false; if (field.getCollectionType() != collectionType) return false; @@ -221,7 +216,7 @@ DocTypeBuilder::SchemaIndexFields::setup(const Schema &schema) continue; const Schema::IndexField &field = schema.getIndexField(fieldId); switch (field.getDataType()) { - case Schema::STRING: + case schema::STRING: _textFields.push_back(fieldId); break; default: @@ -242,9 +237,9 @@ using namespace document::config_builder; TypeOrId makeCollection(TypeOrId datatype, Schema::CollectionType collection_type) { switch (collection_type) { - case Schema::ARRAY: + case schema::ARRAY: return Array(datatype); - case Schema::WEIGHTEDSET: + case schema::WEIGHTEDSET: // TODO: consider using array of struct<primitive,int32> to keep order return Wset(datatype); default: @@ -282,7 +277,7 @@ document::DocumenttypesConfig DocTypeBuilder::makeConfig() const { _schema.getIndexField(_iFields._textFields[i]); // only handles string fields for now - assert(field.getDataType() == Schema::STRING); + assert(field.getDataType() == schema::STRING); header_struct.addField(field.getName(), type_cache.getType( DataType::T_STRING, field.getCollectionType())); header_struct.sstruct.field.back().id = field_id++; @@ -295,7 +290,7 @@ document::DocumenttypesConfig DocTypeBuilder::makeConfig() const { _schema.getIndexField(_iFields._uriFields[i]._all); // only handles string fields for now - assert(field.getDataType() == Schema::STRING); + assert(field.getDataType() == schema::STRING); header_struct.addField(field.getName(), type_cache.getType( uri_type, field.getCollectionType())); header_struct.sstruct.field.back().id = field_id++; diff --git a/searchlib/src/vespa/searchlib/index/schemautil.cpp b/searchlib/src/vespa/searchlib/index/schemautil.cpp index 6019c7ce4bd..414ebe227d6 100644 --- a/searchlib/src/vespa/searchlib/index/schemautil.cpp +++ b/searchlib/src/vespa/searchlib/index/schemautil.cpp @@ -7,18 +7,16 @@ #include <set> LOG_SETUP(".index.schemautil"); -namespace search -{ +namespace search { -namespace index -{ +namespace index { SchemaUtil::IndexSettings SchemaUtil::getIndexSettings(const Schema &schema, const uint32_t index) { IndexSettings ret; - Schema::DataType indexDataType(Schema::STRING); + Schema::DataType indexDataType(schema::STRING); bool error = false; bool somePrefixes = false; bool someNotPrefixes = false; @@ -42,7 +40,7 @@ SchemaUtil::getIndexSettings(const Schema &schema, someNotPositions = true; indexDataType = iField.getDataType(); switch (indexDataType) { - case Schema::STRING: + case schema::STRING: break; default: error = true; @@ -115,7 +113,7 @@ SchemaUtil::validateIndexField(const Schema::IndexField &field) field.getName().c_str()); ok = false; } - if (field.getDataType() != Schema::STRING) { + if (field.getDataType() != schema::STRING) { if (field.hasPrefix()) { LOG(error, "Field %s is non-string but has prefix", @@ -196,7 +194,7 @@ SchemaUtil::validateSchema(const Schema &schema) bool SchemaUtil::getIndexIds(const Schema &schema, - DataType dataType, + schema::DataType dataType, std::vector<uint32_t> &indexes) { typedef SchemaUtil::IndexIterator IndexIterator; diff --git a/searchlib/src/vespa/searchlib/index/schemautil.h b/searchlib/src/vespa/searchlib/index/schemautil.h index 1f7c351c43e..488fa7760c0 100644 --- a/searchlib/src/vespa/searchlib/index/schemautil.h +++ b/searchlib/src/vespa/searchlib/index/schemautil.h @@ -4,65 +4,39 @@ #include <vespa/searchcommon/common/schema.h> -namespace search -{ - -namespace index -{ +namespace search { +namespace index { class SchemaUtil { public: - typedef Schema::DataType DataType; class IndexSettings { - DataType _dataType; + schema::DataType _dataType; bool _error; // Schema is bad. bool _prefix; bool _phrases; bool _positions; public: - const DataType & - getDataType(void) const - { + const schema::DataType & getDataType(void) const { return _dataType; } - bool - hasError(void) const - { - return _error; - } - - bool - hasPrefix(void) const - { - return _prefix; - } - - bool - hasPhrases(void) const - { - return _phrases; - } - - bool - hasPositions(void) const - { - return _positions; - } + bool hasError(void) const { return _error; } + bool hasPrefix(void) const { return _prefix; } + bool hasPhrases(void) const { return _phrases; } + bool hasPositions(void) const { return _positions; } IndexSettings(void) - : _dataType(Schema::STRING), + : _dataType(schema::STRING), _error(false), _prefix(false), _phrases(false), _positions(false) - { - } + { } IndexSettings(const IndexSettings &rhs) : _dataType(rhs._dataType), @@ -70,10 +44,9 @@ public: _prefix(rhs._prefix), _phrases(rhs._phrases), _positions(rhs._positions) - { - } + { } - IndexSettings(DataType dataType, + IndexSettings(schema::DataType dataType, bool error, bool prefix, bool phrases, @@ -83,20 +56,15 @@ public: _prefix(prefix), _phrases(phrases), _positions(positions) - { - } + { } - IndexSettings & - operator=(const IndexSettings &rhs) - { + IndexSettings & operator=(const IndexSettings &rhs) { IndexSettings tmp(rhs); swap(tmp); return *this; } - void - swap(IndexSettings &rhs) - { + void swap(IndexSettings &rhs) { std::swap(_dataType, rhs._dataType); std::swap(_error, rhs._error); std::swap(_prefix, rhs._prefix); @@ -114,14 +82,12 @@ public: IndexIterator(const Schema &schema) : _schema(schema), _index(0u) - { - } + { } IndexIterator(const Schema &schema, uint32_t index) : _schema(schema), _index(index) - { - } + { } IndexIterator(const Schema &schema, const IndexIterator &rhs) : _schema(schema), @@ -131,42 +97,30 @@ public: _index = schema.getIndexFieldId(name); } - const Schema & - getSchema(void) const - { + const Schema getSchema(void) const { return _schema; } - uint32_t - getIndex(void) const - { + uint32_t getIndex(void) const { return _index; } - const vespalib::string & - getName(void) const - { + const vespalib::string &getName(void) const { return _schema.getIndexField(_index).getName(); } - IndexIterator & - operator++(void) - { + IndexIterator &operator++(void) { if (_index < _schema.getNumIndexFields()) { ++_index; } return *this; } - bool - isValid(void) const - { + bool isValid(void) const { return _index < _schema.getNumIndexFields(); } - IndexSettings - getIndexSettings(void) const - { + IndexSettings getIndexSettings(void) const { return SchemaUtil::getIndexSettings(_schema, _index); } @@ -178,8 +132,7 @@ public: * @param oldSchema old schema, present in an input index * @param phrases ask for phrase files */ - bool - hasOldFields(const Schema &oldSchema, bool phrases) const; + bool hasOldFields(const Schema &oldSchema, bool phrases) const; /** * Return if fields in old schema matches fields in new @@ -190,20 +143,16 @@ public: * @param oldSchema old schema, present in an input index * @param phrases ask for phrase files */ - bool - hasMatchingOldFields(const Schema &oldSchema, bool phrases) const; + bool hasMatchingOldFields(const Schema &oldSchema, bool phrases) const; }; - static IndexSettings - getIndexSettings(const Schema &schema, const uint32_t index); + static IndexSettings getIndexSettings(const Schema &schema, const uint32_t index); - static bool - validateIndexFieldType(DataType dataType) - { + static bool validateIndexFieldType(schema::DataType dataType) { switch (dataType) { - case Schema::STRING: - case Schema::INT32: + case schema::STRING: + case schema::INT32: return true; default: ; @@ -211,20 +160,10 @@ public: return false; } - static bool - validateIndexField(const Schema::IndexField &field); - - static bool - addIndexField(Schema &schema, - const Schema::IndexField &field); - - static bool - validateSchema(const Schema &schema); - - static bool - getIndexIds(const Schema &schema, - DataType dataType, - std::vector<uint32_t> &indexes); + static bool validateIndexField(const Schema::IndexField &field); + static bool addIndexField(Schema &schema, const Schema::IndexField &field); + static bool validateSchema(const Schema &schema); + static bool getIndexIds(const Schema &schema, schema::DataType dataType, std::vector<uint32_t> &indexes); }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp index a6899d87bba..29a7c1ac274 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".memoryindex.fieldinverter"); #include "fieldinverter.h" #include <vespa/document/datatype/urldatatype.h> #include <vespa/searchlib/util/url.h> @@ -20,11 +18,9 @@ LOG_SETUP(".memoryindex.fieldinverter"); #include <vespa/document/annotation/spantree.h> #include <vespa/document/annotation/spantreevisitor.h> -namespace search -{ +namespace search { -namespace memoryindex -{ +namespace memoryindex { using document::Field; using document::FieldValue; @@ -50,11 +46,9 @@ using index::Schema; using vespalib::make_string; using search::util::URL; -namespace documentinverterkludge -{ +namespace documentinverterkludge { -namespace linguistics -{ +namespace linguistics { const vespalib::string SPANTREE_NAME("linguistics"); @@ -433,14 +427,14 @@ FieldInverter::invertNormalDocTextField(const FieldValue &val) const vespalib::Identifiable::RuntimeClass & cInfo(val.getClass()); const Schema::IndexField &field = _schema.getIndexField(_fieldId); switch (field.getCollectionType()) { - case Schema::SINGLE: + case index::schema::SINGLE: if (cInfo.id() == StringFieldValue::classId) { processNormalDocTextField(static_cast<const StringFieldValue &>(val)); } else { throw std::runtime_error(make_string("Expected DataType::STRING, got '%s'", val.getDataType()->getName().c_str())); } break; - case Schema::WEIGHTEDSET: + case index::schema::WEIGHTEDSET: if (cInfo.id() == WeightedSetFieldValue::classId) { const WeightedSetFieldValue &wset = static_cast<const WeightedSetFieldValue &>(val); if (wset.getNestedType() == *DataType::STRING) { @@ -452,7 +446,7 @@ FieldInverter::invertNormalDocTextField(const FieldValue &val) throw std::runtime_error(make_string("Expected weighted set, got '%s'", cInfo.name())); } break; - case Schema::ARRAY: + case index::schema::ARRAY: if (cInfo.id() == ArrayFieldValue::classId) { const ArrayFieldValue &arr = static_cast<const ArrayFieldValue&>(val); if (arr.getNestedType() == *DataType::STRING) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp index 88602a365ae..3f15f3927b5 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp @@ -27,6 +27,13 @@ using index::Schema; namespace memoryindex { +vespalib::asciistream & +operator<<(vespalib::asciistream & os, const MemoryFieldIndex::WordKey & rhs) +{ + os << "wr(" << rhs._wordRef.ref() << ")"; + return os; +} + MemoryFieldIndex::MemoryFieldIndex(const Schema & schema, uint32_t fieldId) : _wordStore(), _numUniqueWords(0), @@ -37,8 +44,7 @@ MemoryFieldIndex::MemoryFieldIndex(const Schema & schema, uint32_t fieldId) _fieldId(fieldId), _remover(_wordStore), _inserter(std::make_unique<OrderedDocumentInserter>(*this)) -{ -} +{ } MemoryFieldIndex::~MemoryFieldIndex(void) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h index f6d6400555a..e00573374d0 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h @@ -35,22 +35,11 @@ public: struct WordKey { datastore::EntryRef _wordRef; - explicit WordKey(datastore::EntryRef wordRef) - : _wordRef(wordRef) - { - } - - WordKey(void) - : _wordRef() - { - } + explicit WordKey(datastore::EntryRef wordRef) : _wordRef(wordRef) { } + WordKey() : _wordRef() { } friend vespalib::asciistream & - operator<<(vespalib::asciistream & os, const WordKey & rhs) - { - os << "wr(" << rhs._wordRef.ref() << ")"; - return os; - } + operator<<(vespalib::asciistream & os, const WordKey & rhs); }; class KeyComp { @@ -71,8 +60,7 @@ public: KeyComp(const WordStore &wordStore, const vespalib::stringref word) : _wordStore(wordStore), _word(word) - { - } + { } bool operator()(const WordKey & lhs, const WordKey & rhs) const diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp index 80642ac6330..5629302804e 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".memoryindex.urlfieldinverter"); #include "urlfieldinverter.h" #include "fieldinverter.h" #include <vespa/document/datatype/urldatatype.h> @@ -11,15 +9,15 @@ LOG_SETUP(".memoryindex.urlfieldinverter"); #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/common/sort.h> +#include <vespa/log/log.h> +LOG_SETUP(".memoryindex.urlfieldinverter"); -namespace search -{ +namespace search { -namespace memoryindex -{ +namespace memoryindex { + +namespace { -namespace -{ static vespalib::string HOSTNAME_BEGIN("StArThOsT"); static vespalib::string HOSTNAME_END("EnDhOsT"); const vespalib::string SPANTREE_NAME("linguistics"); @@ -306,7 +304,7 @@ UrlFieldInverter::invertUrlField(const FieldValue &val) { const vespalib::Identifiable::RuntimeClass & cInfo(val.getClass()); switch (_collectionType) { - case Schema::SINGLE: + case index::schema::SINGLE: if (isUriType(*val.getDataType())) { startElement(1); processUrlField(val); @@ -315,7 +313,7 @@ UrlFieldInverter::invertUrlField(const FieldValue &val) throw std::runtime_error(make_string("Expected URI struct, got '%s'", val.getDataType()->getName().c_str())); } break; - case Schema::WEIGHTEDSET: + case index::schema::WEIGHTEDSET: if (cInfo.id() == WeightedSetFieldValue::classId) { const WeightedSetFieldValue &wset = static_cast<const WeightedSetFieldValue &>(val); if (isUriType(wset.getNestedType())) { @@ -327,7 +325,7 @@ UrlFieldInverter::invertUrlField(const FieldValue &val) throw std::runtime_error(make_string("Expected weighted set, got '%s'", cInfo.name())); } break; - case Schema::ARRAY: + case index::schema::ARRAY: if (cInfo.id() == ArrayFieldValue::classId) { const ArrayFieldValue &arr = static_cast<const ArrayFieldValue&>(val); if (isUriType(arr.getNestedType())) { diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp index 7b666a2a31a..8f501414e7c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp @@ -2,7 +2,6 @@ #include <vespa/fastos/fastos.h> #include "hitcollector.h" -#include "scores.h" #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/sort.h> #include <limits> @@ -85,6 +84,24 @@ HitCollector::RankedHitCollector::collect(uint32_t docId, feature_t score) } } +template <bool CollectRankedHit> +void +HitCollector::BitVectorCollector<CollectRankedHit>::collect(uint32_t docId, feature_t score) { + this->_hc._bitVector->setBit(docId); + if (CollectRankedHit) { + this->considerForHitVector(docId, score); + } +} + +void +HitCollector::CollectorBase::replaceHitInVector(uint32_t docId, feature_t score) { + // replace lowest scored hit in hit vector + std::pop_heap(_hc._hits.begin(), _hc._hits.end(), ScoreComparator()); + _hc._hits.back().first = docId; + _hc._hits.back().second = score; + std::push_heap(_hc._hits.begin(), _hc._hits.end(), ScoreComparator()); +} + void HitCollector::RankedHitCollector::collectAndChangeCollector(uint32_t docId, feature_t score) { diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index c521009ca79..8fbeff3f39a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -5,10 +5,9 @@ #include "scores.h" #include <vespa/searchlib/common/hitrank.h> #include <vespa/searchlib/common/resultset.h> -#include <vespa/searchlib/fef/matchdata.h> -#include <vespa/vespalib/util/sort.h> #include <algorithm> #include <vector> +#include <vespa/vespalib/util/sort.h> namespace search { @@ -37,13 +36,13 @@ private: const uint32_t _maxReRankHitsSize; const uint32_t _maxDocIdVectorSize; - std::vector<Hit> _hits; // used as a heap when _hits.size == _maxHitsSize - std::vector<uint32_t> _scoreOrder; // Holds an indirection to the N best hits - SortOrder _hitsSortOrder; - bool _unordered; - std::vector<uint32_t> _docIdVector; - BitVector::UP _bitVector; - std::vector<Hit> _reRankedHits; + std::vector<Hit> _hits; // used as a heap when _hits.size == _maxHitsSize + std::vector<uint32_t> _scoreOrder; // Holds an indirection to the N best hits + SortOrder _hitsSortOrder; + bool _unordered; + std::vector<uint32_t> _docIdVector; + std::unique_ptr<BitVector> _bitVector; + std::vector<Hit> _reRankedHits; std::pair<Scores, Scores> _ranges; feature_t _scale; @@ -108,43 +107,32 @@ private: } } protected: - void replaceHitInVector(uint32_t docId, feature_t score) { - // replace lowest scored hit in hit vector - std::pop_heap(_hc._hits.begin(), _hc._hits.end(), ScoreComparator()); - _hc._hits.back().first = docId; - _hc._hits.back().second = score; - std::push_heap(_hc._hits.begin(), _hc._hits.end(), ScoreComparator()); - } + void replaceHitInVector(uint32_t docId, feature_t score); HitCollector &_hc; }; class RankedHitCollector : public CollectorBase { public: RankedHitCollector(HitCollector &hc) : CollectorBase(hc) { } - virtual void collect(uint32_t docId, feature_t score); + void collect(uint32_t docId, feature_t score) override; void collectAndChangeCollector(uint32_t docId, feature_t score) __attribute__((noinline)); - virtual bool isRankedHitCollector() const { return true; } + bool isRankedHitCollector() const override { return true; } }; template <bool CollectRankedHit> class DocIdCollector : public CollectorBase { public: DocIdCollector(HitCollector &hc) : CollectorBase(hc) { } - virtual void collect(uint32_t docId, feature_t score); + void collect(uint32_t docId, feature_t score) override; void collectAndChangeCollector(uint32_t docId) __attribute__((noinline)); - virtual bool isDocIdCollector() const { return true; } + bool isDocIdCollector() const override { return true; } }; template <bool CollectRankedHit> class BitVectorCollector : public CollectorBase { public: BitVectorCollector(HitCollector &hc) : CollectorBase(hc) { } - virtual void collect(uint32_t docId, feature_t score) { - this->_hc._bitVector->setBit(docId); - if (CollectRankedHit) { - this->considerForHitVector(docId, score); - } - } + virtual void collect(uint32_t docId, feature_t score) override; }; HitRank getReScore(feature_t score) const { diff --git a/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp b/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp index 0fd34af84a4..7b7a2fd8e61 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp +++ b/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp @@ -56,8 +56,8 @@ struct Builder void TestDiskIndex::buildSchema(void) { - _schema.addIndexField(Schema::IndexField("f1", Schema::STRING)); - _schema.addIndexField(Schema::IndexField("f2", Schema::STRING)); + _schema.addIndexField(Schema::IndexField("f1", index::schema::STRING)); + _schema.addIndexField(Schema::IndexField("f2", index::schema::STRING)); _schema.addFieldSet(Schema::FieldSet("c2"). addField("f1"). addField("f2")); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp index 4ecf04bb59c..b6a8b541344 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp @@ -1,17 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fakewordset"); #include "fakewordset.h" #include "fakeword.h" #include <vespa/searchlib/index/schemautil.h> +#include <vespa/log/log.h> +LOG_SETUP(".fakewordset"); -namespace search -{ +namespace search { -namespace fakedata -{ +namespace fakedata { using index::PostingListParams; using index::SchemaUtil; @@ -65,16 +63,14 @@ FakeWordSet::setupParams(bool hasElements, _schema.clear(); assert(hasElements || !hasElementWeights); - Schema::CollectionType collectionType(Schema::SINGLE); + Schema::CollectionType collectionType(index::schema::SINGLE); if (hasElements) { if (hasElementWeights) - collectionType = Schema::WEIGHTEDSET; + collectionType = index::schema::WEIGHTEDSET; else - collectionType = Schema::ARRAY; + collectionType = index::schema::ARRAY; } - Schema::IndexField indexField("field0", - Schema::STRING, - collectionType); + Schema::IndexField indexField("field0", index::schema::STRING, collectionType); indexField.setAvgElemLen(512u); _schema.addIndexField(indexField); _fieldsParams.resize(_schema.getNumIndexFields()); |