diff options
259 files changed, 2939 insertions, 2610 deletions
diff --git a/document/src/vespa/document/base/documentcalculator.cpp b/document/src/vespa/document/base/documentcalculator.cpp index 89379dd4587..7a83d80764c 100644 --- a/document/src/vespa/document/base/documentcalculator.cpp +++ b/document/src/vespa/document/base/documentcalculator.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/base/documentcalculator.h> +#include "documentcalculator.h" #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/select/compare.h> #include <vespa/document/select/parser.h> @@ -18,14 +18,16 @@ DocumentCalculator::DocumentCalculator( _selectionNode = parser.parse(expression + " == 0"); } +DocumentCalculator::~DocumentCalculator() { } + double -DocumentCalculator::evaluate(const Document& doc, VariableMap& variables) +DocumentCalculator::evaluate(const Document& doc, VariableMap && variables) { select::Compare& compare(static_cast<select::Compare&>(*_selectionNode)); const select::ValueNode& left = compare.getLeft(); select::Context context(doc); - context._variables = variables; + context._variables = std::move(variables); std::unique_ptr<select::Value> value = left.getValue(context); select::NumberValue* num = dynamic_cast<select::NumberValue*>(value.get()); diff --git a/document/src/vespa/document/base/documentcalculator.h b/document/src/vespa/document/base/documentcalculator.h index 59b601cf21e..7e1feaf5688 100644 --- a/document/src/vespa/document/base/documentcalculator.h +++ b/document/src/vespa/document/base/documentcalculator.h @@ -9,11 +9,11 @@ class DocumentTypeRepo; class DocumentCalculator { public: - typedef vespalib::hash_map<vespalib::string, double> VariableMap; + using VariableMap = vespalib::hash_map<vespalib::string, double>; - DocumentCalculator(const DocumentTypeRepo& repo, - const vespalib::string& expression); - double evaluate(const Document& doc, VariableMap& variables); + DocumentCalculator(const DocumentTypeRepo& repo, const vespalib::string& expression); + ~DocumentCalculator(); + double evaluate(const Document& doc, VariableMap && variables); private: std::unique_ptr<select::Node> _selectionNode; diff --git a/document/src/vespa/document/base/globalid.cpp b/document/src/vespa/document/base/globalid.cpp index 514f2256c30..2a0fe9f0e5d 100644 --- a/document/src/vespa/document/base/globalid.cpp +++ b/document/src/vespa/document/base/globalid.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include <vespa/document/base/globalid.h> -#include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_set.hpp> namespace { @@ -196,3 +196,5 @@ std::ostream & operator << (std::ostream & os, const GlobalId & gid) } } // document + +VESPALIB_HASH_SET_INSTANTIATE_H(document::GlobalId, document::GlobalId::hash); diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index bbb5dede812..01a3d1bd428 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -1,15 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/document/bucket/bucketid.h> +#include "bucketid.h" #include <iomanip> -#include <sstream> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_set.hpp> using vespalib::nbostream; +template class vespalib::Array<document::BucketId>; + namespace document { @@ -146,3 +147,5 @@ operator>>(nbostream &is, BucketId &bucketId) } } // document + +VESPALIB_HASH_SET_INSTANTIATE_H(document::BucketId, document::BucketId::hash);
\ No newline at end of file diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp index 659309714ba..cadc2fca3c7 100644 --- a/document/src/vespa/document/datatype/structdatatype.cpp +++ b/document/src/vespa/document/datatype/structdatatype.cpp @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/document/datatype/structdatatype.h> +#include "structdatatype.h" #include <iomanip> -#include <vespa/log/log.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldvalue/structfieldvalue.h> #include <vespa/document/fieldvalue/document.h> - +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".document.datatype.struct"); namespace document { @@ -32,8 +31,7 @@ StructDataType::StructDataType(const vespalib::stringref &name) { } -StructDataType::StructDataType(const vespalib::stringref & name, - int32_t dataTypeId) +StructDataType::StructDataType(const vespalib::stringref & name, int32_t dataTypeId) : StructuredDataType(name, dataTypeId), _nameFieldMap(), _idFieldMap(), @@ -41,6 +39,13 @@ StructDataType::StructDataType(const vespalib::stringref & name, { } +StructDataType::~StructDataType() { } + +StructDataType* +StructDataType::clone() const { + return new StructDataType(*this); +} + void StructDataType::print(std::ostream& out, bool verbose, const std::string& indent) const diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h index d3e14db14f2..7d22ec91f2a 100644 --- a/document/src/vespa/document/datatype/structdatatype.h +++ b/document/src/vespa/document/datatype/structdatatype.h @@ -24,6 +24,7 @@ public: StructDataType(); StructDataType(const vespalib::stringref &name); StructDataType(const vespalib::stringref &name, int32_t id); + ~StructDataType(); /** * @throws vespalib::IllegalArgumentException if field conflicts with @@ -61,7 +62,7 @@ public: } virtual Field::Set getFieldSet() const; - virtual StructDataType* clone() const { return new StructDataType(*this); } + virtual StructDataType* clone() const; void setCompressionConfig(const CompressionConfig& cfg) { _compressionConfig = cfg; }; const CompressionConfig& getCompressionConfig() const { return _compressionConfig; } diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 95d2e871352..ef429d85143 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -5,6 +5,7 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/util/stringfmt.h> #include <algorithm> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".document.serializable-array"); diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 4d640dbcf3a..ae26db8ee99 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -11,7 +11,7 @@ #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/vespalib/objects/identifiable.h> -#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/closure.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp index 111a7ed40a1..b127e29c1ad 100644 --- a/document/src/vespa/document/select/operator.cpp +++ b/document/src/vespa/document/select/operator.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "operator.h" -#include <stdint.h> +#include "operator.h" #include <vespa/vespalib/util/regexp.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace document { namespace select { diff --git a/document/src/vespa/document/select/valuenode.cpp b/document/src/vespa/document/select/valuenode.cpp index 25842af21c7..f473b21147a 100644 --- a/document/src/vespa/document/select/valuenode.cpp +++ b/document/src/vespa/document/select/valuenode.cpp @@ -1,19 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "valuenode.h" #include "visitor.h" #include <iomanip> -#include <vespa/log/log.h> #include <vespa/document/base/exceptions.h> #include "parser.h" #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/vespalib/util/md5.h> #include <vespa/document/util/stringutil.h> -#include <sys/time.h> #include <vespa/vespalib/text/lowercase.h> #include <regex> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".document.select.valuenode"); namespace document { @@ -24,8 +23,7 @@ namespace { } namespace { - bool documentTypeEqualsName(const DocumentType& type, - const vespalib::stringref& name) + bool documentTypeEqualsName(const DocumentType& type, const vespalib::stringref& name) { if (type.getName() == name) return true; for (std::vector<const DocumentType *>::const_iterator it @@ -40,8 +38,7 @@ namespace { InvalidValueNode::InvalidValueNode(const vespalib::stringref & name) : _name(name) -{ -} +{ } void @@ -63,8 +60,7 @@ InvalidValueNode::print(std::ostream& out, bool verbose, NullValueNode::NullValueNode(const vespalib::stringref & name) : _name(name) -{ -} +{ } void @@ -263,8 +259,7 @@ FieldValueNode::getValue(const Context& context) const LOG(warning, "Caught exception while fetching field from document: %s", e.what()); return std::unique_ptr<Value>(new InvalidValue()); } catch (FieldNotFoundException& e) { - LOG(warning, "Tried to compare to field %s, not found in document type", - _fieldExpression.c_str()); + LOG(warning, "Tried to compare to field %s, not found in document type", _fieldExpression.c_str()); return std::unique_ptr<Value>(new InvalidValue()); } } diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp index a5ca977003c..cf2da086190 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.cpp +++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <boost/numeric/conversion/cast.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".document.update.fieldpathupdate"); @@ -62,6 +63,13 @@ AssignFieldPathUpdate::AssignFieldPathUpdate( } } +AssignFieldPathUpdate::~AssignFieldPathUpdate() { } + +FieldPathUpdate* +AssignFieldPathUpdate::clone() const { + return new AssignFieldPathUpdate(*this); +} + std::unique_ptr<FieldValue::IteratorHandler> AssignFieldPathUpdate::getIteratorHandler(Document& doc) const { @@ -115,7 +123,7 @@ AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) vars["value"] = fv.getAsDouble(); try { - double res = _calc.evaluate(_doc, vars); + double res = _calc.evaluate(_doc, std::move(vars)); if (_removeIfZero && static_cast<uint64_t>(res) == 0) { return REMOVED; } else { diff --git a/document/src/vespa/document/update/assignfieldpathupdate.h b/document/src/vespa/document/update/assignfieldpathupdate.h index ff54f76f4ed..3cfa234922b 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.h +++ b/document/src/vespa/document/update/assignfieldpathupdate.h @@ -30,6 +30,7 @@ public: stringref fieldPath, stringref whereClause, stringref expression); + ~AssignFieldPathUpdate(); void setRemoveIfZero(bool removeIfZero) { _removeIfZero = removeIfZero; @@ -43,7 +44,7 @@ public: bool hasValue() const { return _newValue.get() != nullptr; } const FieldValue & getValue() const { return *_newValue; } - FieldPathUpdate* clone() const { return new AssignFieldPathUpdate(*this); } + FieldPathUpdate* clone() const; bool operator==(const FieldPathUpdate& other) const; diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index 35ddfea4ed9..a4145a9b58a 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -49,6 +49,8 @@ FieldPathUpdate::FieldPathUpdate(const DocumentTypeRepo& repo, } } +FieldPathUpdate::~FieldPathUpdate() { } + bool FieldPathUpdate::operator==(const FieldPathUpdate& other) const { diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 9703edf5d6c..9424ed9c54f 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -39,7 +39,7 @@ public: stringref fieldPath, stringref whereClause = stringref()); - virtual ~FieldPathUpdate() {} + ~FieldPathUpdate(); enum FieldPathUpdateType { Add = IDENTIFIABLE_CLASSID(AddFieldPathUpdate), diff --git a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp index d992110513f..20b01ae59c9 100644 --- a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp +++ b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp @@ -3,6 +3,7 @@ #include "loadtypeset.h" #include <vespa/config-load-type.h> #include <vespa/config/config.h> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h index 3a626f966d8..98277cac427 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/util/bytebuffer.h> +#include "documentreply.h" #include <vespa/messagebus/message.h> #include <vespa/messagebus/reply.h> -#include "documentreply.h" #include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/documentapi/messagebus/priority.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h index addc3890d7e..c1e13ccbc16 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/messagebus/reply.h> -#include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/documentapi/messagebus/priority.h> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp index 4b073afbb6f..b177475f82a 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentstate.h" - #include <vespa/document/util/bytebuffer.h> #include <vespa/documentapi/common.h> #include <vespa/vespalib/objects/nbostream.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp index 5416b73d548..4525b37ef14 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/documentapi/messagebus/messages/documentsummarymessage.h> + +#include "documentsummarymessage.h" using vdslib::DocumentSummary; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp index 5d1d1ed00e5..ef87fa631fe 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/documentapi/messagebus/messages/emptybucketsmessage.h> + +#include "emptybucketsmessage.h" #include <vespa/documentapi/messagebus/documentprotocol.h> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp index ad7a44c1cc9..af68c6b4886 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "getbucketlistmessage.h" +#include "getbucketlistreply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/documentapi/messagebus/messages/getbucketlistmessage.h> -#include <vespa/documentapi/messagebus/messages/getbucketlistreply.h> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h index 8d5fa587ffd..b8cf4f72293 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "documentmessage.h" #include <vespa/document/bucket/bucketid.h> -#include <vespa/documentapi/messagebus/messages/documentmessage.h> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp index 4ac6a91fd47..b1dadc9bf0e 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp @@ -9,17 +9,13 @@ namespace documentapi { GetBucketListReply::BucketInfo::BucketInfo() : _bucket(), _bucketInformation() -{ - // empty -} +{ } GetBucketListReply::BucketInfo::BucketInfo(const document::BucketId &bucketId, const string &bucketInformation) : _bucket(bucketId), _bucketInformation(bucketInformation) -{ - // empty -} +{ } bool GetBucketListReply::BucketInfo::operator==(const GetBucketListReply::BucketInfo &rhs) const diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.cpp index 93c4f261fc5..fc465a61706 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.cpp @@ -1,24 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "getbucketstatemessage.h" +#include "getbucketstatereply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/documentapi/messagebus/messages/getbucketstatemessage.h> -#include <vespa/documentapi/messagebus/messages/getbucketstatereply.h> namespace documentapi { GetBucketStateMessage::GetBucketStateMessage() : DocumentMessage(), _bucket() -{ - // empty -} +{ } GetBucketStateMessage::GetBucketStateMessage(const document::BucketId &bucket) : DocumentMessage(), _bucket(bucket) -{ - // empty -} +{ } DocumentReply::UP GetBucketStateMessage::doCreateReply() const diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.cpp index f0ce18f66b7..c2b628c7d05 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.cpp @@ -1,16 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "getbucketstatereply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/documentapi/messagebus/messages/getbucketstatereply.h> namespace documentapi { GetBucketStateReply::GetBucketStateReply() : DocumentReply(DocumentProtocol::REPLY_GETBUCKETSTATE), _state() -{ - // empty -} +{ } GetBucketStateReply::GetBucketStateReply(std::vector<DocumentState> &state) : DocumentReply(DocumentProtocol::REPLY_GETBUCKETSTATE), @@ -19,4 +16,6 @@ GetBucketStateReply::GetBucketStateReply(std::vector<DocumentState> &state) : _state.swap(state); } +GetBucketStateReply::~GetBucketStateReply() { } + } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.h index 4de839d5e6c..ece8e4e69a1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/documentapi/messagebus/messages/documentreply.h> +#include "documentreply.h" #include "documentstate.h" namespace documentapi { @@ -23,6 +23,7 @@ public: * @param state The state to swap. */ GetBucketStateReply(std::vector<DocumentState> &state); + ~GetBucketStateReply(); /** * Sets the bucket state of this by swapping the content of the provided state object. diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp index 5a577ac450b..8d67c6fbe5c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/documentapi/messagebus/policies/contentpolicy.h> +#include "contentpolicy.h" namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp index 57103b265a3..1e3a46f6370 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/documentapi/messagebus/policies/externslobrokpolicy.h> + +#include "externslobrokpolicy.h" #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/messagebus/routing/routingcontext.h> -#include <vespa/slobrok/cfg.h> using slobrok::api::IMirrorAPI; using slobrok::api::MirrorAPI; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp index 399931dbbd5..f55967b1c09 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "messagetypepolicy.h" #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/vespalib/stllike/hash_map.hpp> using vespa::config::content::MessagetyperouteselectorpolicyConfig; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h index 536be0044fe..aee4e1bc071 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h @@ -30,8 +30,8 @@ private: typedef vespalib::PtrHolder<MessageTypeMap> MessageTypeHolder; typedef vespalib::PtrHolder<mbus::Route> RouteHolder; - MessageTypeHolder _map; - RouteHolder _defaultRoute; + MessageTypeHolder _map; + RouteHolder _defaultRoute; config::ConfigFetcher _fetcher; public: diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp index 39cdad6f46b..fba1d648d23 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -11,6 +11,8 @@ LOG_SETUP(".storagepolicy"); +using vespalib::make_string; + namespace documentapi { StoragePolicy::StoragePolicy(const string& param) @@ -207,9 +209,7 @@ StoragePolicy::doSelect(mbus::RoutingContext &context) } else { context.setError( mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE, - vespalib::make_string( - "Could not resolve a distributor to send to in cluster %s", - _clusterName.c_str())); + make_string("Could not resolve a distributor to send to in cluster %s", _clusterName.c_str())); } } @@ -219,8 +219,7 @@ StoragePolicy::getRecipient(mbus::RoutingContext& context, int distributor) slobrok::api::IMirrorAPI::SpecList entries = lookup(context, createPattern(_clusterName, distributor)); if (!entries.empty()) { - return mbus::Hop::parse( - entries[random() % entries.size()].second + "/default"); + return mbus::Hop::parse(entries[random() % entries.size()].second + "/default"); } return mbus::Hop(); @@ -248,21 +247,17 @@ StoragePolicy::updateStateFromReply(WrongDistributionReply& wdr) new storage::lib::ClusterState(wdr.getSystemState())); if (_state.get() == 0 || newState->getVersion() >= _state->getVersion()) { if (_state.get()) { - wdr.getTrace().trace(1, vespalib::make_string( - "System state changed from version %u to %u", - _state->getVersion(), - newState->getVersion())); + wdr.getTrace().trace(1, make_string("System state changed from version %u to %u", + _state->getVersion(), newState->getVersion())); } else { - wdr.getTrace().trace(1, vespalib::make_string( - "System state set to version %u", newState->getVersion())); + wdr.getTrace().trace(1, make_string("System state set to version %u", newState->getVersion())); } _state = std::move(newState); } else { - wdr.getTrace().trace(1, vespalib::make_string( - "System state cleared because system state returned had version %d, while old state had version %d. New states should not have a lower version than the old.", - newState->getVersion(), - _state->getVersion())); + wdr.getTrace().trace(1, make_string("System state cleared because system state returned had version %d, " + "while old state had version %d. New states should not have a lower version than the old.", + newState->getVersion(), _state->getVersion())); _state.reset(); } } diff --git a/fnet/src/vespa/fnet/channel.h b/fnet/src/vespa/fnet/channel.h index ee8ad42ff90..d2c798d0b5e 100644 --- a/fnet/src/vespa/fnet/channel.h +++ b/fnet/src/vespa/fnet/channel.h @@ -2,6 +2,14 @@ #pragma once +#include "context.h" +#include "ipackethandler.h" +#include <memory> + +#define FNET_NOID ((uint32_t)-1) + +class FNET_Connection; +class FNET_IPacketHandler; /** * A channel object represents an endpoint in a point-to-point packet * based virtual connection. Clients open channels by invoking the diff --git a/fnet/src/vespa/fnet/channellookup.cpp b/fnet/src/vespa/fnet/channellookup.cpp index df05b691bc1..1ba63a6019b 100644 --- a/fnet/src/vespa/fnet/channellookup.cpp +++ b/fnet/src/vespa/fnet/channellookup.cpp @@ -1,11 +1,23 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/fnet/fnet.h> +#include "channellookup.h" +#include "vespa/fnet/channel.h" +#include "controlpacket.h" +#include "ipackethandler.h" +#include <vespa/vespalib/stllike/hash_map.hpp> +namespace fnet { + +struct ChannelMap : public vespalib::hash_map<uint32_t, FNET_Channel *> { + using Parent = vespalib::hash_map<uint32_t, FNET_Channel *>; + ChannelMap(size_t sz) : Parent(sz) { } +}; + +} +using fnet::ChannelMap; FNET_ChannelLookup::FNET_ChannelLookup(uint32_t hashSize) - : _map(hashSize) + : _map(std::make_unique<ChannelMap>(hashSize)) { assert(hashSize > 0); } @@ -13,7 +25,7 @@ FNET_ChannelLookup::FNET_ChannelLookup(uint32_t hashSize) FNET_ChannelLookup::~FNET_ChannelLookup() { - assert(_map.empty()); + assert(_map->empty()); } @@ -21,15 +33,15 @@ void FNET_ChannelLookup::Register(FNET_Channel *channel) { assert(channel->GetHandler() != nullptr); - _map[channel->GetID()] = channel; + (*_map)[channel->GetID()] = channel; } FNET_Channel* FNET_ChannelLookup::Lookup(uint32_t id) { - auto found = _map.find(id); - return ((found != _map.end()) ? found->second : nullptr); + auto found = _map->find(id); + return ((found != _map->end()) ? found->second : nullptr); } @@ -38,7 +50,7 @@ FNET_ChannelLookup::Broadcast(FNET_ControlPacket *cpacket) { std::vector<uint32_t> toRemove; std::vector<FNET_Channel::UP> toFree; - for (const auto & pair : _map) { + for (const auto & pair : *_map) { FNET_Channel *ch = pair.second; FNET_IPacketHandler::HP_RetCode hp_rc = ch->Receive(cpacket); if (hp_rc > FNET_IPacketHandler::FNET_KEEP_CHANNEL) { @@ -49,7 +61,7 @@ FNET_ChannelLookup::Broadcast(FNET_ControlPacket *cpacket) } } for (uint32_t id : toRemove) { - _map.erase(id); + _map->erase(id); } return toFree; } @@ -57,10 +69,10 @@ FNET_ChannelLookup::Broadcast(FNET_ControlPacket *cpacket) bool FNET_ChannelLookup::Unregister(FNET_Channel *channel) { - auto found = _map.find(channel->GetID()); - if (found == _map.end()) { + auto found = _map->find(channel->GetID()); + if (found == _map->end()) { return false; } - _map.erase(found); + _map->erase(found); return true; } diff --git a/fnet/src/vespa/fnet/channellookup.h b/fnet/src/vespa/fnet/channellookup.h index df694c0fa61..7700a4f927a 100644 --- a/fnet/src/vespa/fnet/channellookup.h +++ b/fnet/src/vespa/fnet/channellookup.h @@ -2,7 +2,15 @@ #pragma once -#include <vespa/vespalib/stllike/hash_map.h> +#include <memory> +#include <vector> + +namespace fnet { + class ChannelMap; +} + +class FNET_Channel; +class FNET_ControlPacket; /** * This class handles registration/deregistration and lookup of @@ -14,8 +22,7 @@ class FNET_ChannelLookup { private: - typedef vespalib::hash_map<uint32_t, FNET_Channel *> ChannelMap; - ChannelMap _map; + std::unique_ptr<fnet::ChannelMap> _map; FNET_ChannelLookup(const FNET_ChannelLookup &); FNET_ChannelLookup &operator=(const FNET_ChannelLookup &); @@ -37,11 +44,6 @@ public: ~FNET_ChannelLookup(); /** - * @return number of registered channels. - **/ - uint32_t GetEntryCnt() { return _map.size(); } - - /** * Register a channel. If you register several channels with the * same ID, only the last registered channel will be availible by * calling the Lookup method. @@ -75,7 +77,7 @@ public: * @return vector of all channels to be freed. * @param cpacket the control packet you want to broadcast. **/ - std::vector<FNET_Channel::UP> Broadcast(FNET_ControlPacket *cpacket); + std::vector<std::unique_ptr<FNET_Channel>> Broadcast(FNET_ControlPacket *cpacket); /** * Unregister a channel. This method uses both the channel ID and diff --git a/fnet/src/vespa/fnet/context.h b/fnet/src/vespa/fnet/context.h index 27e9533436f..782009e5bec 100644 --- a/fnet/src/vespa/fnet/context.h +++ b/fnet/src/vespa/fnet/context.h @@ -2,6 +2,11 @@ #pragma once +class FNET_IOComponent; +class FNET_Connector; +class FNET_Connection; +class FNET_Channel; +class FNET_IExecutable; /** * This class indicates the context of a packet. It is external to the * packet class because a single packet may occur in many contexts at diff --git a/fnet/src/vespa/fnet/controlpacket.h b/fnet/src/vespa/fnet/controlpacket.h index 35b6b8470c6..2ecbef8fcbf 100644 --- a/fnet/src/vespa/fnet/controlpacket.h +++ b/fnet/src/vespa/fnet/controlpacket.h @@ -2,6 +2,8 @@ #pragma once +#include "packet.h" + /** * Packets of this type may be used to send simple control signals * between components in the application. Control packets only contain diff --git a/fnet/src/vespa/fnet/fnet.h b/fnet/src/vespa/fnet/fnet.h index 85f4f3e750e..6144817b885 100644 --- a/fnet/src/vespa/fnet/fnet.h +++ b/fnet/src/vespa/fnet/fnet.h @@ -11,8 +11,6 @@ // DEFINES -#define FNET_NOID ((uint32_t)-1) - // THREAD/MUTEX STUFF #ifdef FASTOS_NO_THREADS diff --git a/fnet/src/vespa/fnet/ipackethandler.h b/fnet/src/vespa/fnet/ipackethandler.h index b0af1ba0484..7042b092b51 100644 --- a/fnet/src/vespa/fnet/ipackethandler.h +++ b/fnet/src/vespa/fnet/ipackethandler.h @@ -2,6 +2,10 @@ #pragma once +#include "context.h" + +class FNET_Packet; + /** * Interface implemented by objects that can handle packets. **/ diff --git a/fnet/src/vespa/fnet/packet.h b/fnet/src/vespa/fnet/packet.h index a7033b8008f..0dea973ab9a 100644 --- a/fnet/src/vespa/fnet/packet.h +++ b/fnet/src/vespa/fnet/packet.h @@ -4,6 +4,8 @@ #include <vespa/vespalib/stllike/string.h> +class FNET_DataBuffer; + /** * This is a general superclass of all packets. Packets are used to * encapsulate data when communicating with other computers through diff --git a/juniper/src/vespa/juniper/stringmap.cpp b/juniper/src/vespa/juniper/stringmap.cpp index 5ee3f43abd1..1b4ce63f397 100644 --- a/juniper/src/vespa/juniper/stringmap.cpp +++ b/juniper/src/vespa/juniper/stringmap.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "stringmap.h" +#include <vespa/vespalib/stllike/hash_map.hpp> void Fast_StringMap::Insert(const char* key, const char* value) { diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp index 96295cd560b..4ba25a48950 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp @@ -3,9 +3,8 @@ #include "memfile_v1_verifier.h" #include "memfilemapper.h" #include "simplememfileiobuffer.h" -#include "buffer.h" -#include <vespa/memfilepersistence/memfile/memfile.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfilev1.verifier"); diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp index 505e9c32f3b..ecb66058362 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "simplememfileiobuffer.h" #include <vespa/document/repo/documenttyperepo.h> #include <vespa/memfilepersistence/common/environment.h> -#include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> #include <vespa/vespalib/util/crc.h> -#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/log/log.h> LOG_SETUP(".memfile.simpleiobuffer"); diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp index 6dfa983729b..62bbc7f3f4b 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp @@ -1,18 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/memfilepersistence/memfile/memfile.h> +#include "memfile.h" +#include "memfilecompactor.h" +#include "shared_data_location_tracker.h" #include <ext/algorithm> #include <vespa/memfilepersistence/common/exceptions.h> #include <vespa/memfilepersistence/mapper/memfilemapper.h> #include <vespa/memfilepersistence/mapper/simplememfileiobuffer.h> -#include <vespa/memfilepersistence/memfile/memfilecompactor.h> -#include <vespa/memfilepersistence/memfile/shared_data_location_tracker.h> #include <vespa/vespalib/util/crc.h> #include <vespa/memfilepersistence/common/environment.h> #include <iomanip> #include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfile.memfile"); diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecompactor.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecompactor.cpp index 17bf530d450..9af6fd13264 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecompactor.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecompactor.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "memfilecompactor.h" +#include "memfile.h" +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> -#include <vespa/memfilepersistence/memfile/memfile.h> -#include <vespa/memfilepersistence/memfile/memfilecompactor.h> -#include <algorithm> LOG_SETUP(".persistence.memfile.compactor"); @@ -19,8 +19,7 @@ struct DocumentVersionInfo { : _id(docId), _versions(1), _tombstoned(tombstoned) - { - } + { } bool newerVersionExists() const noexcept { return (_versions != 1); diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.cpp index 1780870c050..fba74aeb84d 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/memfilepersistence/memfile/slotiterator.h> +#include "slotiterator.h" #include <vespa/memfilepersistence/memfile/memfile.h> +#include <vespa/vespalib/stllike/hash_set.hpp> namespace storage { namespace memfile { diff --git a/messagebus/src/tests/slobrok/slobrok.cpp b/messagebus/src/tests/slobrok/slobrok.cpp index d51a5330cd5..82d16c9dcf2 100644 --- a/messagebus/src/tests/slobrok/slobrok.cpp +++ b/messagebus/src/tests/slobrok/slobrok.cpp @@ -1,7 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("slobrok_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/messagebus/testlib/slobrok.h> #include <string> diff --git a/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp b/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp index 13b0da585fa..3b0e0072d38 100644 --- a/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <algorithm> #include "rpcservicepool.h" #include "rpcnetwork.h" +#include <vespa/vespalib/stllike/lrucache_map.hpp> namespace mbus { diff --git a/messagebus/src/vespa/messagebus/testlib/oosserver.cpp b/messagebus/src/vespa/messagebus/testlib/oosserver.cpp index d78bacf6b06..ab95541f9c9 100644 --- a/messagebus/src/vespa/messagebus/testlib/oosserver.cpp +++ b/messagebus/src/vespa/messagebus/testlib/oosserver.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "oosserver.h" #include "slobrok.h" diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp index 6872e9a66cc..282797d461b 100644 --- a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp +++ b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "slobrok.h" -#include <vespa/slobrok/cfg.h> #include <vespa/slobrok/server/sbenv.h> -#include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/util/vstringfmt.h> + #include <vespa/log/log.h> LOG_SETUP(".slobrok"); diff --git a/metrics/src/vespa/metrics/loadmetric.hpp b/metrics/src/vespa/metrics/loadmetric.hpp index 7e0518d9403..e1faa756be1 100644 --- a/metrics/src/vespa/metrics/loadmetric.hpp +++ b/metrics/src/vespa/metrics/loadmetric.hpp @@ -3,6 +3,7 @@ #include "loadmetric.h" #include "memoryconsumption.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace metrics { diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 4c5fb22f717..13d7443d2bb 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -13,7 +13,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> - #include <vespa/log/log.h> LOG_SETUP(".metrics.manager"); @@ -22,6 +21,9 @@ namespace metrics { typedef MetricsmanagerConfig Config; +MetricManager::ConsumerSpec::ConsumerSpec() : includedMetrics() { } +MetricManager::ConsumerSpec::~ConsumerSpec() { } + void MetricManager::assertMetricLockLocked(const MetricLockGuard& g) const { if (!g.monitors(_waiter)) { @@ -355,8 +357,7 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) for (const auto & consumer : _config->consumer) { ConsumerMetricBuilder consumerMetricBuilder(consumer); _activeMetrics.getMetrics().visit(consumerMetricBuilder); - configMap[consumer.name] = ConsumerSpec::SP( - new ConsumerSpec(consumerMetricBuilder._matchedMetrics)); + configMap[consumer.name] = ConsumerSpec::SP(new ConsumerSpec(std::move(consumerMetricBuilder._matchedMetrics))); } LOG(debug, "Recreating snapshots to include altered metrics"); _activeMetrics.updateNames(_nameHash); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index ad2a807f8f4..138a4b0c06f 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -58,6 +58,8 @@ #include <vespa/vespalib/util/jsonwriter.h> #include <vespa/config/config.h> +template class vespalib::hash_set<metrics::Metric::String>; + namespace metrics { typedef vespalib::MonitorGuard MetricLockGuard; @@ -80,7 +82,10 @@ public: typedef std::shared_ptr<ConsumerSpec> SP; vespalib::hash_set<Metric::String> includedMetrics; - ConsumerSpec() : includedMetrics() {} + ConsumerSpec(ConsumerSpec &&) = default; + ConsumerSpec & operator= (ConsumerSpec &&) = default; + ConsumerSpec(); + ~ConsumerSpec(); bool contains(const Metric& m) const { return (includedMetrics.find(m.getPath()) != includedMetrics.end()); diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp index da589b7b6e0..397eaa85538 100644 --- a/metrics/src/vespa/metrics/metricset.cpp +++ b/metrics/src/vespa/metrics/metricset.cpp @@ -2,7 +2,7 @@ #include "metricset.h" #include "memoryconsumption.h" -#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <list> diff --git a/metrics/src/vespa/metrics/namehash.cpp b/metrics/src/vespa/metrics/namehash.cpp index 29a6f55257b..d957656a731 100644 --- a/metrics/src/vespa/metrics/namehash.cpp +++ b/metrics/src/vespa/metrics/namehash.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "namehash.h" -#include <vespa/metrics/memoryconsumption.h> +#include "memoryconsumption.h" #include <vespa/vespalib/stllike/hash_set.h> namespace metrics { diff --git a/metrics/src/vespa/metrics/namehash.h b/metrics/src/vespa/metrics/namehash.h index a6e73ac47d6..ee1bc3e52a8 100644 --- a/metrics/src/vespa/metrics/namehash.h +++ b/metrics/src/vespa/metrics/namehash.h @@ -20,7 +20,7 @@ */ #pragma once -#include <vespa/metrics/memoryconsumption.h> +#include "memoryconsumption.h" namespace metrics { diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index c088026bc78..c2404e88a7b 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -1,16 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <algorithm> -#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include "dummypersistence.h" #include <vespa/document/select/parser.h> #include <vespa/document/base/documentid.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/util/crc.h> #include <vespa/vespalib/util/vstringfmt.h> #include <vespa/document/fieldset/fieldsetrepo.h> -#include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> using std::binary_search; @@ -22,6 +21,16 @@ namespace storage { namespace spi { namespace dummy { +BucketContent::BucketContent() + : _entries(), + _gidMap(), + _info(), + _inUse(false), + _outdatedInfo(true), + _active(false) +{ } +BucketContent::~BucketContent() { } + uint32_t BucketContent::computeEntryChecksum(const BucketEntry& e) const { diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index 6a674182443..c1af86d5a77 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -53,14 +53,9 @@ struct BucketContent { mutable bool _outdatedInfo; bool _active; - BucketContent() - : _entries(), - _gidMap(), - _info(), - _inUse(false), - _outdatedInfo(true), - _active(false) - { } + BucketContent(); + ~BucketContent(); + uint32_t computeEntryChecksum(const BucketEntry&) const; BucketChecksum updateRollingChecksum(uint32_t entryChecksum); diff --git a/persistence/src/vespa/persistence/spi/context.cpp b/persistence/src/vespa/persistence/spi/context.cpp index dc84aaf0e34..aba054a9814 100644 --- a/persistence/src/vespa/persistence/spi/context.cpp +++ b/persistence/src/vespa/persistence/spi/context.cpp @@ -5,5 +5,14 @@ namespace storage { namespace spi { +Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel) + : _loadType(&loadType), + _priority(pri), + _trace(maxTraceLevel), + _readConsistency(ReadConsistency::STRONG) +{ } + +Context::~Context() { } + } // spi } // storage diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h index 6758355515f..ac4889da4bb 100644 --- a/persistence/src/vespa/persistence/spi/context.h +++ b/persistence/src/vespa/persistence/spi/context.h @@ -57,12 +57,10 @@ class Context { ReadConsistency _readConsistency; public: - Context(const LoadType& loadType, Priority pri, int maxTraceLevel) - : _loadType(&loadType), - _priority(pri), - _trace(maxTraceLevel), - _readConsistency(ReadConsistency::STRONG) - { } + Context(Context &&) = default; + Context & operator = (Context &&) = default; + Context(const LoadType& loadType, Priority pri, int maxTraceLevel); + ~Context(); const LoadType& getLoadType() const { return *_loadType; } Priority getPriority() const { return _priority; } diff --git a/persistence/src/vespa/persistence/spi/result.cpp b/persistence/src/vespa/persistence/spi/result.cpp index 4d86fc01212..511a73134f0 100644 --- a/persistence/src/vespa/persistence/spi/result.cpp +++ b/persistence/src/vespa/persistence/spi/result.cpp @@ -4,7 +4,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/stllike/asciistream.h> - namespace storage { namespace spi { diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp index 88403f49d61..af1014fb167 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "schema.h" #include <fstream> #include <vespa/config/common/configparser.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".index.schema"); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index ecc68558f50..3c6cb26e5e9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -1,22 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.attribute.attributemanager"); +#include "attributemanager.h" #include "attribute_factory.h" -#include "attribute_initializer.h" #include "attributedisklayout.h" -#include "attributemanager.h" -#include "flushableattribute.h" #include "sequential_attributes_initializer.h" +#include "i_attribute_functor.h" #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/data/fileheader.h> -#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchlib/attribute/interlock.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> -#include <memory> -#include "i_attribute_functor.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.attribute.attributemanager"); using search::AttributeContext; using search::AttributeEnumGuard; @@ -245,6 +241,8 @@ AttributeManager::AttributeManager(const AttributeManager &currMgr, transferExtraAttributes(currMgr); } +AttributeManager::~AttributeManager() { } + AttributeVector::SP AttributeManager::addAttribute(const vespalib::string &name, const Config &cfg, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index cbefeff8e7e..878e6b3558f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -116,6 +116,7 @@ public: AttributeManager(const AttributeManager &currMgr, const Spec &newSpec, IAttributeInitializerRegistry &initializerRegistry); + ~AttributeManager(); search::AttributeVector::SP addAttribute(const vespalib::string &name, const Config &cfg, diff --git a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp index 71991317d2d..bcc24101019 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> + +#include "attrupdate.h" #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/predicatefieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> @@ -13,13 +13,13 @@ #include <vespa/document/update/arithmeticvalueupdate.h> #include <vespa/document/update/clearvalueupdate.h> #include <vespa/document/base/forcelink.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/common/base.h> -#include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> -#include "attrupdate.h" #include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/changevector.hpp> + +#include <vespa/log/log.h> LOG_SETUP(".attrupdate"); using namespace document; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 11ff8474e88..e097966b029 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -2,7 +2,7 @@ #include "documentmetastore.h" #include "search_context.h" - +#include "documentmetastoresaver.h" #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/btree/btree.hpp> #include <vespa/searchlib/btree/btreenode.hpp> @@ -17,7 +17,8 @@ #include <vespa/searchcore/proton/bucketdb/joinbucketssession.h> #include <vespa/searchcore/proton/bucketdb/splitbucketsession.h> #include <vespa/searchlib/util/bufferwriter.h> -#include "documentmetastoresaver.h" +#include <vespa/searchlib/common/rcuvector.hpp> + using document::GlobalId; using document::BucketId; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt index 6ef5d80106c..dddcfbd9327 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt @@ -12,5 +12,6 @@ vespa_add_library(searchcore_flushengine STATIC prepare_restart_flush_strategy.cpp threadedflushtarget.cpp tls_stats_factory.cpp + tls_stats_map.cpp DEPENDS ) diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h index bfa31086c42..07e726d2032 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h @@ -19,13 +19,17 @@ public: : _numBytes(0), _firstSerial(0), _lastSerial(0) - { - } + { } TlsStats(uint64_t numBytes, uint64_t firstSerial, uint64_t lastSerial) : _numBytes(numBytes), _firstSerial(firstSerial), _lastSerial(lastSerial) - { + { } + + bool operator == (const TlsStats & b) const { + return (_numBytes == b._numBytes) && + (_firstSerial == b._firstSerial) && + (_lastSerial == b._lastSerial); } uint64_t getNumBytes() const { return _numBytes; } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.cpp new file mode 100644 index 00000000000..b7b12822b82 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.cpp @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tls_stats_map.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + +namespace proton { +namespace flushengine { + +TlsStatsMap::TlsStatsMap(Map &&map) + : _map(std::move(map)) +{ } + +TlsStatsMap::~TlsStatsMap() { } + +const TlsStats & +TlsStatsMap::getTlsStats(const vespalib::string &domain) const { + auto itr = _map.find(domain); + if (itr != _map.end()) { + return itr->second; + } + abort(); +} + +} +} + +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, proton::flushengine::TlsStats); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.h b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.h index b03da7407a4..7e46ac13ecf 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.h @@ -19,18 +19,10 @@ private: Map _map; public: - TlsStatsMap(Map &&map) - : _map(std::move(map)) - { - } + TlsStatsMap(Map &&map); + ~TlsStatsMap(); - const TlsStats &getTlsStats(const vespalib::string &domain) const { - auto itr = _map.find(domain); - if (itr != _map.end()) { - return itr->second; - } - abort(); - } + const TlsStats &getTlsStats(const vespalib::string &domain) const; }; } // namespace proton::flushengine diff --git a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h index 2958b4a182a..f08e7a5f97b 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h +++ b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h @@ -1,6 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <memory> +#include <vector> + namespace proton { namespace initializer { @@ -23,19 +26,12 @@ private: List _dependencies; public: InitializerTask(); - virtual ~InitializerTask(); - State getState() const { return _state; } - const List &getDependencies() const { return _dependencies; } - void setRunning() { _state = State::RUNNING; } - void setDone() { _state = State::DONE; } - void addDependency(SP dependency); - virtual void run() = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp index c653dd30b9e..16fcf94b1d3 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp +++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "task_runner.h" #include <vespa/searchlib/common/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutor.h> @@ -24,8 +23,7 @@ TaskRunner::~TaskRunner() } void -TaskRunner::getReadyTasks(const InitializerTask::SP task, TaskList &readyTasks, - TaskSet &checked) +TaskRunner::getReadyTasks(const InitializerTask::SP task, TaskList &readyTasks, TaskSet &checked) { if (task->getState() != State::BLOCKED) { return; // task running or done, all dependencies done diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h index 6fc52a6c289..2f68f0a24f7 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h +++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h @@ -18,7 +18,7 @@ class TaskRunner { uint32_t _runningTasks; // used by context executor using State = InitializerTask::State; using TaskList = InitializerTask::List; - using TaskSet = vespalib::hash_set<void *>; + using TaskSet = vespalib::hash_set<const void *>; class Context { InitializerTask::SP _rootTask; @@ -44,8 +44,7 @@ class TaskRunner { const InitializerTask::SP &rootTask() { return _rootTask; } void schedulePoll(); }; - void getReadyTasks(const InitializerTask::SP task, TaskList &readyTasks, - TaskSet &checked); + void getReadyTasks(const InitializerTask::SP task, TaskList &readyTasks, TaskSet &checked); void setTaskRunning(InitializerTask &task); diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp index e1ad0bd0400..e82d4101178 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.matching.queryenvironment"); - #include "queryenvironment.h" using search::attribute::IAttributeContext; diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp index 15d68ecfb08..a676b89c65d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.cpp @@ -1,46 +1,201 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "sessionmanager.h" +#include <vespa/vespalib/stllike/lrucache_map.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> + #include <vespa/log/log.h> LOG_SETUP(".sessionmanager"); -#include "sessionmanager.h" using search::grouping::GroupingSession; namespace proton { namespace matching { -void SessionManager::SessionCacheBase::entryDropped(const SessionId &id) { - LOG(debug, "Session cache is full, dropping entry to fit " - "session '%s'", id.c_str()); - _stats.numDropped++; -} + namespace { + using Stats = SessionManager::Stats; + struct SessionCacheBase { + protected: + Stats _stats; + vespalib::Lock _lock; + + void entryDropped(const SessionId &id); + ~SessionCacheBase() {} + }; + + template <typename T> + struct SessionCache : SessionCacheBase { + typedef typename T::LP EntryLP; + typedef typename T::UP EntryUP; + vespalib::lrucache_map<vespalib::LruParam<SessionId, EntryLP> > _cache; + + SessionCache(uint32_t max_size) : _cache(max_size) {} + + void insert(EntryUP session) { + vespalib::LockGuard guard(_lock); + const SessionId &id(session->getSessionId()); + if (_cache.size() >= _cache.capacity()) { + entryDropped(id); + } + _cache.insert(id, EntryLP(session.release())); + _stats.numInsert++; + } + EntryUP pick(const SessionId & id) { + vespalib::LockGuard guard(_lock); + EntryUP ret; + if (_cache.hasKey(id)) { + _stats.numPick++; + EntryLP session(_cache.get(id)); + _cache.erase(id); + ret.reset(session.release()); + } + return ret; + } + void pruneTimedOutSessions(fastos::TimeStamp currentTime) { + std::vector<EntryLP> toDestruct = stealTimedOutSessions(currentTime); + toDestruct.clear(); + } + std::vector<EntryLP> stealTimedOutSessions(fastos::TimeStamp currentTime) { + std::vector<EntryLP> toDestruct; + vespalib::LockGuard guard(_lock); + toDestruct.reserve(_cache.size()); + for (auto it(_cache.begin()), mt(_cache.end()); it != mt;) { + EntryLP session = *it; + if (session->getTimeOfDoom() < currentTime) { + toDestruct.push_back(session); + it = _cache.erase(it); + _stats.numTimedout++; + } else { + it++; + } + } + return toDestruct; + } + Stats getStats() { + vespalib::LockGuard guard(_lock); + Stats stats = _stats; + stats.numCached = _cache.size(); + _stats = Stats(); + return stats; + } + bool empty() const { + vespalib::LockGuard guard(_lock); + return _cache.empty(); + } + }; + + template <typename T> + struct SessionMap : SessionCacheBase { + typedef typename T::SP EntrySP; + vespalib::hash_map<SessionId, EntrySP> _map; + + void insert(EntrySP session) { + vespalib::LockGuard guard(_lock); + const SessionId &id(session->getSessionId()); + _map.insert(std::make_pair(id, session)); + _stats.numInsert++; + } + EntrySP pick(const SessionId & id) { + vespalib::LockGuard guard(_lock); + auto it = _map.find(id); + if (it != _map.end()) { + _stats.numPick++; + return it->second; + } + return EntrySP(); + } + void pruneTimedOutSessions(fastos::TimeStamp currentTime) { + std::vector<EntrySP> toDestruct = stealTimedOutSessions(currentTime); + toDestruct.clear(); + } + std::vector<EntrySP> stealTimedOutSessions(fastos::TimeStamp currentTime) { + std::vector<EntrySP> toDestruct; + std::vector<SessionId> keys; + vespalib::LockGuard guard(_lock); + keys.reserve(_map.size()); + toDestruct.reserve(_map.size()); + for (auto & it : _map) { + EntrySP &session = it.second; + if (session->getTimeOfDoom() < currentTime) { + keys.push_back(it.first); + toDestruct.push_back(EntrySP()); + toDestruct.back().swap(session); + } + } + for (auto key : keys) { + _map.erase(key); + _stats.numTimedout++; + } + return toDestruct; + } + Stats getStats() { + vespalib::LockGuard guard(_lock); + Stats stats = _stats; + stats.numCached = _map.size(); + _stats = Stats(); + return stats; + } + size_t size() const { + vespalib::LockGuard guard(_lock); + return _map.size(); + } + bool empty() const { + vespalib::LockGuard guard(_lock); + return _map.empty(); + } + template <typename F> + void each(F f) const { + vespalib::LockGuard guard(_lock); + for (const auto &entry: _map) { + f(*entry.second); + } + } + }; + + void SessionCacheBase::entryDropped(const SessionId &id) { + LOG(debug, "Session cache is full, dropping entry to fit session '%s'", id.c_str()); + _stats.numDropped++; + } + + } + + struct GroupingSessionCache : public SessionCache<search::grouping::GroupingSession> { + using Parent = SessionCache<search::grouping::GroupingSession>; + using Parent::Parent; + }; + + struct SearchSessionCache : public SessionMap<SearchSession> { + + }; + SessionManager::SessionManager(uint32_t maxSize) - : _grouping_cache(maxSize), - _search_map() { + : _grouping_cache(std::make_unique<GroupingSessionCache>(maxSize)), + _search_map(std::make_unique<SearchSessionCache>()) { } +SessionManager::~SessionManager() { } + void SessionManager::insert(search::grouping::GroupingSession::UP session) { - _grouping_cache.insert(std::move(session)); + _grouping_cache->insert(std::move(session)); } void SessionManager::insert(SearchSession::SP session) { - _search_map.insert(std::move(session)); + _search_map->insert(std::move(session)); } GroupingSession::UP SessionManager::pickGrouping(const SessionId &id) { - return _grouping_cache.pick(id); + return _grouping_cache->pick(id); } SearchSession::SP SessionManager::pickSearch(const SessionId &id) { - return _search_map.pick(id); + return _search_map->pick(id); } std::vector<SessionManager::SearchSessionInfo> SessionManager::getSortedSearchSessionInfo() const { std::vector<SearchSessionInfo> sessions; - _search_map.each([&sessions](const SearchSession &session) + _search_map->each([&sessions](const SearchSession &session) { sessions.emplace_back(session.getSessionId(), session.getCreateTime(), @@ -56,14 +211,24 @@ SessionManager::getSortedSearchSessionInfo() const } void SessionManager::pruneTimedOutSessions(fastos::TimeStamp currentTime) { - _grouping_cache.pruneTimedOutSessions(currentTime); - _search_map.pruneTimedOutSessions(currentTime); + _grouping_cache->pruneTimedOutSessions(currentTime); + _search_map->pruneTimedOutSessions(currentTime); } void SessionManager::close() { pruneTimedOutSessions(fastos::TimeStamp::FUTURE); - assert(_grouping_cache.empty()); - assert(_search_map.empty()); + assert(_grouping_cache->empty()); + assert(_search_map->empty()); +} + +SessionManager::Stats SessionManager::getGroupingStats() { + return _grouping_cache->getStats(); +} +SessionManager::Stats SessionManager::getSearchStats() { + return _search_map->getStats(); +} +size_t SessionManager::getNumSearchSessions() const { + return _search_map->size(); } } // namespace proton::matching diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h index 56b48047611..ddb3fd253d3 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h +++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h @@ -12,6 +12,9 @@ namespace matching { typedef vespalib::string SessionId; +class GroupingSessionCache; +class SearchSessionCache; + class SessionManager : public ISessionCachePruner { public: struct Stats { @@ -40,161 +43,24 @@ public: }; private: - struct SessionCacheBase { - protected: - Stats _stats; - vespalib::Lock _lock; - - void entryDropped(const SessionId &id); - ~SessionCacheBase() {} - }; - - template <typename T> - struct SessionCache : SessionCacheBase { - typedef typename T::LP EntryLP; - typedef typename T::UP EntryUP; - vespalib::lrucache_map<vespalib::LruParam<SessionId, EntryLP> > _cache; - - SessionCache(uint32_t max_size) : _cache(max_size) {} - - void insert(EntryUP session) { - vespalib::LockGuard guard(_lock); - const SessionId &id(session->getSessionId()); - if (_cache.size() >= _cache.capacity()) { - entryDropped(id); - } - _cache.insert(id, EntryLP(session.release())); - _stats.numInsert++; - } - EntryUP pick(const SessionId & id) { - vespalib::LockGuard guard(_lock); - EntryUP ret; - if (_cache.hasKey(id)) { - _stats.numPick++; - EntryLP session(_cache.get(id)); - _cache.erase(id); - ret.reset(session.release()); - } - return ret; - } - void pruneTimedOutSessions(fastos::TimeStamp currentTime) { - std::vector<EntryLP> toDestruct = stealTimedOutSessions(currentTime); - toDestruct.clear(); - } - std::vector<EntryLP> stealTimedOutSessions(fastos::TimeStamp currentTime) { - std::vector<EntryLP> toDestruct; - vespalib::LockGuard guard(_lock); - toDestruct.reserve(_cache.size()); - for (auto it(_cache.begin()), mt(_cache.end()); it != mt;) { - EntryLP session = *it; - if (session->getTimeOfDoom() < currentTime) { - toDestruct.push_back(session); - it = _cache.erase(it); - _stats.numTimedout++; - } else { - it++; - } - } - return toDestruct; - } - Stats getStats() { - vespalib::LockGuard guard(_lock); - Stats stats = _stats; - stats.numCached = _cache.size(); - _stats = Stats(); - return stats; - } - bool empty() const { - vespalib::LockGuard guard(_lock); - return _cache.empty(); - } - }; - - template <typename T> - struct SessionMap : SessionCacheBase { - typedef typename T::SP EntrySP; - vespalib::hash_map<SessionId, EntrySP> _map; - - void insert(EntrySP session) { - vespalib::LockGuard guard(_lock); - const SessionId &id(session->getSessionId()); - _map.insert(std::make_pair(id, session)); - _stats.numInsert++; - } - EntrySP pick(const SessionId & id) { - vespalib::LockGuard guard(_lock); - auto it = _map.find(id); - if (it != _map.end()) { - _stats.numPick++; - return it->second; - } - return EntrySP(); - } - void pruneTimedOutSessions(fastos::TimeStamp currentTime) { - std::vector<EntrySP> toDestruct = stealTimedOutSessions(currentTime); - toDestruct.clear(); - } - std::vector<EntrySP> stealTimedOutSessions(fastos::TimeStamp currentTime) { - std::vector<EntrySP> toDestruct; - std::vector<SessionId> keys; - vespalib::LockGuard guard(_lock); - keys.reserve(_map.size()); - toDestruct.reserve(_map.size()); - for (auto & it : _map) { - EntrySP &session = it.second; - if (session->getTimeOfDoom() < currentTime) { - keys.push_back(it.first); - toDestruct.push_back(EntrySP()); - toDestruct.back().swap(session); - } - } - for (auto key : keys) { - _map.erase(key); - _stats.numTimedout++; - } - return toDestruct; - } - Stats getStats() { - vespalib::LockGuard guard(_lock); - Stats stats = _stats; - stats.numCached = _map.size(); - _stats = Stats(); - return stats; - } - size_t size() const { - vespalib::LockGuard guard(_lock); - return _map.size(); - } - bool empty() const { - vespalib::LockGuard guard(_lock); - return _map.empty(); - } - template <typename F> - void each(F f) const { - vespalib::LockGuard guard(_lock); - for (const auto &entry: _map) { - f(*entry.second); - } - } - }; - - SessionCache<search::grouping::GroupingSession> _grouping_cache; - SessionMap<SearchSession> _search_map; + std::unique_ptr<GroupingSessionCache> _grouping_cache; + std::unique_ptr<SearchSessionCache> _search_map; public: typedef std::unique_ptr<SessionManager> UP; typedef std::shared_ptr<SessionManager> SP; SessionManager(uint32_t maxSizeGrouping); + ~SessionManager(); void insert(search::grouping::GroupingSession::UP session); search::grouping::GroupingSession::UP pickGrouping(const SessionId &id); - Stats getGroupingStats() { return _grouping_cache.getStats(); } + Stats getGroupingStats(); void insert(SearchSession::SP session); SearchSession::SP pickSearch(const SessionId &id); - Stats getSearchStats() { return _search_map.getStats(); } - size_t getNumSearchSessions() const { return _search_map.size(); } + Stats getSearchStats(); + size_t getNumSearchSessions() const; std::vector<SearchSessionInfo> getSortedSearchSessionInfo() const; void pruneTimedOutSessions(fastos::TimeStamp currentTime); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 1f4a32bc4b5..40106e5b332 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -1,16 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "persistenceengine.h" +#include "ipersistenceengineowner.h" +#include "transport_latch.h" +#include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/documentapi/messagebus/messages/feedreply.h> #include <vespa/documentapi/messagebus/messages/removedocumentreply.h> #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> -#include <vespa/searchcore/proton/common/feedtoken.h> -#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> -#include "ipersistenceengineowner.h" -#include "transport_latch.h" -#include <vespa/vespalib/util/exception.h> -#include <vespa/vespalib/util/sequence.h> #include <vespa/vespalib/stllike/hash_set.h> + #include <vespa/log/log.h> LOG_SETUP(".proton.persistenceengine.persistenceengine"); @@ -55,8 +53,7 @@ public: GenericResultHandler(uint32_t waitCnt) : ResultHandlerBase(waitCnt), _result() - { - } + { } virtual void handle(const Result &result) { if (result.hasError()) { vespalib::LockGuard guard(_lock); @@ -80,8 +77,7 @@ private: public: BucketIdListResultHandler() : _bucketSet() - { - } + { } virtual void handle(const BucketIdListResult &result) { const BucketIdListResult::List &buckets = result.getList(); for (size_t i = 0; i < buckets.size(); ++i) { @@ -106,8 +102,7 @@ public: SynchronizedBucketIdListResultHandler(uint32_t waitCnt) : ResultHandlerBase(waitCnt), BucketIdListResultHandler() - { - } + { } virtual void handle(const BucketIdListResult &result) { { vespalib::LockGuard guard(_lock); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp index 2aadee9b320..5c77d573788 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp @@ -1,30 +1,21 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".documentretrieverbase"); - #include "documentretrieverbase.h" -#include <vespa/document/base/documentid.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/searchlib/common/idocumentmetastore.h> -#include <vespa/searchcore/proton/common/cachedselect.h> -#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h> +#include <vespa/vespalib/stllike/lrucache_map.hpp> using document::DocumentId; using document::GlobalId; using search::index::Schema; -namespace -{ +namespace { const DocumentId docId("doc:test:1"); const Schema emptySchema; } -namespace proton -{ +namespace proton { DocumentRetrieverBase::DocumentRetrieverBase( const DocTypeName &docTypeName, @@ -46,6 +37,8 @@ DocumentRetrieverBase::DocumentRetrieverBase( _emptyDoc->setRepo(_repo); } +DocumentRetrieverBase::~DocumentRetrieverBase() { } + const document::DocumentTypeRepo & DocumentRetrieverBase::getDocumentTypeRepo() const { return _repo; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h index 4894baf46e0..22d31c26895 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h @@ -9,8 +9,7 @@ #include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/attribute/iattributemanager.h> -namespace proton -{ +namespace proton { class DocumentRetrieverBase : public IDocumentRetriever { @@ -18,7 +17,7 @@ class DocumentRetrieverBase : public IDocumentRetriever const document::DocumentTypeRepo &_repo; const IDocumentMetaStoreContext &_meta_store; - typedef vespalib::lrucache_map<vespalib::LruParam<vespalib::string, CachedSelect::SP>> SelectCache; + using SelectCache = vespalib::lrucache_map<vespalib::LruParam<vespalib::string, CachedSelect::SP>>; mutable SelectCache _selectCache; vespalib::Lock _lock; @@ -33,6 +32,7 @@ public: const document::DocumentTypeRepo &repo, const IDocumentMetaStoreContext &meta_store, bool hasFields); + ~DocumentRetrieverBase(); const document::DocumentTypeRepo &getDocumentTypeRepo() const override; void getBucketMetaData(const storage::spi::Bucket &bucket, diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp index 9d2322e1286..c577b31ff0f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.matchers"); #include "matchers.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace proton { @@ -14,8 +12,9 @@ Matchers::Matchers(const vespalib::Clock &clock, _fallback(new matching::Matcher(search::index::Schema(), search::fef::Properties(), clock, queryLimiter, constantValueRepo, -1)), _default() -{ -} +{ } + +Matchers::~Matchers() { } void Matchers::add(const vespalib::string &name, matching::Matcher::SP matcher) diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.h b/searchcore/src/vespa/searchcore/proton/server/matchers.h index cb01ab7dbb9..b83603f04c1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.h +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.h @@ -19,6 +19,7 @@ public: Matchers(const vespalib::Clock &clock, matching::QueryLimiter &queryLimiter, const matching::IConstantValueRepo &constantValueRepo); + ~Matchers(); void add(const vespalib::string &name, matching::Matcher::SP matcher); matching::MatchingStats getStats() const; matching::MatchingStats getStats(const vespalib::string &name) const; diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 7a3d5bd5579..2ada2a5d038 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -9,19 +9,16 @@ #include "indexreadutilities.h" #include "indexwriteutilities.h" #include <vespa/searchlib/common/serialnumfileheadercontext.h> -#include <vespa/searchlib/attribute/fixedsourceselector.h> -#include <vespa/searchlib/common/fileheadercontext.h> -#include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/util/autoclosurecaller.h> -#include <vespa/vespalib/util/closure.h> #include <vespa/vespalib/util/closuretask.h> #include <sstream> -#include <vector> #include <vespa/searchcorespi/flush/closureflushtask.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/array.hpp> #include <vespa/log/log.h> + LOG_SETUP(".searchcorespi.index.indexmaintainer"); using document::Document; diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp index ad96184cb57..90daea72703 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp @@ -1,9 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchcorespi/index/warmupindexcollection.h> -#include <vespa/searchcorespi/index/idiskindex.h> +#include "warmupindexcollection.h" +#include "idiskindex.h" #include <vespa/vespalib/util/closuretask.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/query/tree/termnodes.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/aggregation/group.cpp b/searchlib/src/vespa/searchlib/aggregation/group.cpp index f11b536705c..936236e61c3 100644 --- a/searchlib/src/vespa/searchlib/aggregation/group.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/group.cpp @@ -1,21 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/searchlib/aggregation/group.h> -#include <vespa/searchlib/aggregation/maxaggregationresult.h> -#include <vespa/searchlib/aggregation/groupinglevel.h> -#include <vespa/searchlib/aggregation/grouping.h> -#include <vespa/searchlib/expression/aggregationrefnode.h> -#include <vespa/vespalib/objects/visit.h> -#include <vespa/vespalib/objects/objectpredicate.h> -#include <vespa/vespalib/objects/objectoperation.h> +#include "group.h" +#include "maxaggregationresult.h" +#include "groupinglevel.h" +#include "grouping.h" #include <vespa/vespalib/objects/objectdumper.h> -#include <vespa/vespalib/util/optimized.h> #include <vespa/vespalib/util/vstringfmt.h> -#include <cmath> -#include <cstdlib> +#include <vespa/vespalib/stllike/hash_set.hpp> -LOG_SETUP(".searchlib.aggregation.group"); namespace search { namespace aggregation { @@ -447,7 +438,6 @@ Serializer & Group::onSerialize(Serializer & os) const assert(_children[i]->cmpId(*_children[i-1]) > 0); } } - LOG(debug, "%s", _id->asString().c_str()); os << _id << _rank; os << uint32_t(getOrderBySize()); for (size_t i(0), m(getOrderBySize()); i < m; i++) { @@ -519,7 +509,6 @@ Deserializer & Group::onDeserialize(Deserializer & is) _children[i] = group; } is >> _tag; - LOG(debug, "%s", _id->asString().c_str()); if (getChildrenSize() > 1) { for (size_t i(1), m(getChildrenSize()); i < m; i++) { assert(_children[i]->cmpId(*_children[i-1]) > 0); diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index 501c78f75b6..2434b51cc28 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "attribute_weighted_set_blueprint.h" -#include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/attributeguard.h> +#include "attributevector.h" +#include "attributeguard.h" #include <vespa/searchlib/queryeval/weighted_set_term_search.h> -#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/attributecontext.cpp b/searchlib/src/vespa/searchlib/attribute/attributecontext.cpp index 3c1ada3108d..018505958e5 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributecontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributecontext.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".attributecontext"); #include "attributecontext.h" +#include <vespa/vespalib/stllike/hash_map.hpp> using namespace search; using namespace search::attribute; @@ -37,8 +35,9 @@ AttributeContext::AttributeContext(const IAttributeManager & manager) : _attributes(), _enumAttributes(), _cacheLock() -{ -} +{ } + +AttributeContext::~AttributeContext() { } const IAttributeVector * AttributeContext::getAttribute(const string & name) const diff --git a/searchlib/src/vespa/searchlib/attribute/attributecontext.h b/searchlib/src/vespa/searchlib/attribute/attributecontext.h index 5d70c06d16a..91589a14eb4 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributecontext.h +++ b/searchlib/src/vespa/searchlib/attribute/attributecontext.h @@ -28,6 +28,7 @@ private: public: AttributeContext(const search::IAttributeManager & manager); + ~AttributeContext(); // Implements IAttributeContext const attribute::IAttributeVector * getAttribute(const string & name) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp index 43bf6946feb..01733a3fb11 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp @@ -1,17 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attributemanager.h" -#include <vespa/searchlib/attribute/attributecontext.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/attribute/attributemanager.h> -#include <vespa/searchlib/attribute/attrvector.h> -#include <vespa/searchlib/attribute/attributefile.h> -#include <stdexcept> -#include <vespa/searchlib/attribute/attrvector.hpp> -#include <vespa/searchlib/attribute/multistringattribute.h> -#include <vespa/vespalib/util/exceptions.h> +#include "attributecontext.h" +#include "attributefactory.h" +#include "attrvector.h" +#include "attributefile.h" +#include "multistringattribute.h" #include "interlock.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + #include <vespa/log/log.h> LOG_SETUP(".searchlib.attributemanager"); diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.cpp b/searchlib/src/vespa/searchlib/attribute/changevector.cpp index 028ae98d985..912d6884279 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/changevector.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/attribute/changevector.h> -#include <vespa/searchlib/attribute/stringbase.h> +#include "changevector.hpp" +#include "stringbase.h" + #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.changevector"); @@ -18,4 +19,8 @@ StringChangeData::StringChangeData(const vespalib::string & s) } } -} +template class ChangeVectorT<ChangeTemplate<StringChangeData>>; +template class ChangeVectorT<ChangeTemplate<NumericChangeData<int64_t>>>; +template class ChangeVectorT<ChangeTemplate<NumericChangeData<double>>>; + +}
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index 90a8ff7e2d4..f042064f8a4 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -2,7 +2,6 @@ #pragma once -#include <vector> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/searchcommon/common/undefinedvalues.h> @@ -142,7 +141,8 @@ private: typedef vespalib::hash_map<uint32_t, uint32_t> Map; typedef vespalib::Array<T> Vector; public: - ChangeVectorT() : _tail(0) { } + ChangeVectorT(); + ~ChangeVectorT(); class const_iterator { public: const_iterator(const Vector & vector, uint32_t next) : _v(&vector), _next(next) { } @@ -165,7 +165,7 @@ public: const T & back() const { return _v.back(); } T & back() { return _v.back(); } size_t size() const { return _v.size(); } - void clear() { _v.clear(); _docs.clear(); } + void clear(); const_iterator begin() const { return const_iterator(_v, 0); } const_iterator end() const { return const_iterator(_v, size()); } private: @@ -175,56 +175,4 @@ private: uint32_t _tail; }; -template <typename T> -void -ChangeVectorT<T>::push_back(const T & c) -{ - size_t index(size()); - _v.push_back(c); - linkIn(c._doc, index, index); -} - -template <typename T> -template <typename Accessor> -void -ChangeVectorT<T>::push_back(uint32_t doc, Accessor & ac) -{ - if (ac.size() <= 0) { return; } - - size_t index(size()); - _v.reserve(vespalib::roundUp2inN(index + ac.size())); - for (size_t i(0), m(ac.size()); i < m; i++, ac.next()) { - _v.push_back(T(ChangeBase::APPEND, doc, typename T::DataType(ac.value()), ac.weight())); - _v.back().setNext(index + i + 1); - } - linkIn(doc, index, size() - 1); -} - -template <typename T> -void -ChangeVectorT<T>::linkIn(uint32_t doc, size_t first, size_t last) -{ - if (first != 0 && (_v[_tail]._doc == doc)) { - _v[_tail].setNext(first); - _tail = last; - } else { - Map::iterator found(_docs.find(doc)); - if (found == _docs.end()) { - _docs[doc] = last; - if (_tail != first) { - _v[_tail].setNext(first); - } - _tail = last; - } else { - uint32_t prev(found->second); - for (; _v[_v[prev].getNext()]._doc == doc; prev = _v[prev].getNext()); - _v[last].setNext(_v[prev].getNext()); - _v[prev].setNext(first); - found->second = last; - } - } - _v[_tail].setNext(size()); -} - } // namespace search - diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.hpp b/searchlib/src/vespa/searchlib/attribute/changevector.hpp new file mode 100644 index 00000000000..c23a53c7412 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/changevector.hpp @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "changevector.h" +#include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> + +namespace search { + +template <typename T> +ChangeVectorT<T>::ChangeVectorT() : _tail(0) { } + +template <typename T> +ChangeVectorT<T>::~ChangeVectorT() { } + +template <typename T> +void +ChangeVectorT<T>::clear() { + _v.clear(); + _docs.clear(); +} + +template <typename T> +void +ChangeVectorT<T>::push_back(const T & c) +{ + size_t index(size()); + _v.push_back(c); + linkIn(c._doc, index, index); +} + +template <typename T> +template <typename Accessor> +void +ChangeVectorT<T>::push_back(uint32_t doc, Accessor & ac) +{ + if (ac.size() <= 0) { return; } + + size_t index(size()); + _v.reserve(vespalib::roundUp2inN(index + ac.size())); + for (size_t i(0), m(ac.size()); i < m; i++, ac.next()) { + _v.push_back(T(ChangeBase::APPEND, doc, typename T::DataType(ac.value()), ac.weight())); + _v.back().setNext(index + i + 1); + } + linkIn(doc, index, size() - 1); +} + +template <typename T> +void +ChangeVectorT<T>::linkIn(uint32_t doc, size_t first, size_t last) +{ + if (first != 0 && (_v[_tail]._doc == doc)) { + _v[_tail].setNext(first); + _tail = last; + } else { + Map::iterator found(_docs.find(doc)); + if (found == _docs.end()) { + _docs[doc] = last; + if (_tail != first) { + _v[_tail].setNext(first); + } + _tail = last; + } else { + uint32_t prev(found->second); + for (; _v[_v[prev].getNext()]._doc == doc; prev = _v[prev].getNext()); + _v[last].setNext(_v[prev].getNext()); + _v[prev].setNext(first); + found->second = last; + } + } + _v[_tail].setNext(size()); +} + +} // namespace search + diff --git a/searchlib/src/vespa/searchlib/attribute/diversity.h b/searchlib/src/vespa/searchlib/attribute/diversity.h index 68440d05cea..a67e2cf8f8b 100644 --- a/searchlib/src/vespa/searchlib/attribute/diversity.h +++ b/searchlib/src/vespa/searchlib/attribute/diversity.h @@ -2,9 +2,9 @@ #pragma once -#include <vespa/vespalib/stllike/hash_map.h> #include "singleenumattribute.h" #include "singlenumericattribute.h" +#include <vespa/vespalib/stllike/hash_map.hpp> /** * This file contains low-level code used to implement diversified @@ -113,7 +113,9 @@ public: size_t cutoff_max_groups, bool cutoff_strict, Result &result, size_t max_total) : _total_count(0), _max_total(max_total), _diversity(diversity), _max_per_group(max_per_group), - _cutoff_max_groups(cutoff_max_groups), _cutoff_strict(cutoff_strict), _seen(std::min(cutoff_max_groups, 10000ul)*3), _result(result) {} + _cutoff_max_groups(cutoff_max_groups), _cutoff_strict(cutoff_strict), + _seen(std::min(cutoff_max_groups, 10000ul)*3), _result(result) + { } template <typename Item> void push_back(Item item) { if (_total_count < _max_total) { diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 12afc55467f..a31693823b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -159,7 +159,8 @@ public: Uniques _uniques; uint64_t _bufferSize; public: - Builder() : _uniques(), _bufferSize(Index::align(1)) {} + Builder(); + ~Builder(); Index insert(Type value, uint32_t pidx = 0) { uint32_t entrySize = getEntrySize(value); _uniques.push_back(UniqueEntry(value, entrySize, pidx)); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 16945f17815..61ec2fba5d0 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -2,8 +2,8 @@ #pragma once -#include <vespa/searchlib/attribute/enumstore.h> -#include <vespa/searchlib/attribute/enumcomparator.h> +#include "enumstore.h" +#include "enumcomparator.h" #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> @@ -13,6 +13,7 @@ #include <vespa/searchlib/btree/btreebuilder.hpp> #include <vespa/searchlib/btree/btree.hpp> #include <vespa/searchlib/util/bufferwriter.h> +#include <vespa/vespalib/util/array.hpp> namespace search { @@ -83,6 +84,15 @@ EnumStoreT<EntryType>::printBuffer(vespalib::asciistream & os, uint32_t bufferId } template <typename EntryType> +EnumStoreT<EntryType>::Builder::Builder() + : _uniques(), + _bufferSize(Index::align(1)) +{ } + +template <typename EntryType> +EnumStoreT<EntryType>::Builder::~Builder() { } + +template <typename EntryType> void EnumStoreT<EntryType>::printValue(vespalib::asciistream & os, Index idx) const { diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp index 130260ea611..80872556b0a 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp @@ -1,18 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "enumstorebase.h" +#include "enumstore.h" #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <stdexcept> -#include "enumstore.h" #include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/util/bufferwriter.h> +#include <vespa/searchlib/common/rcuvector.hpp> -namespace search -{ +namespace search { using btree::BTreeNode; @@ -650,4 +647,10 @@ template class EnumStoreDict<EnumTree>; template class EnumStoreDict<EnumPostingTree>; +namespace attribute { + + template class RcuVectorBase<EnumStoreIndex>; + +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h index aeb53033035..d083efe5637 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h @@ -30,8 +30,7 @@ class EnumStoreBase; class EnumStoreComparator; class EnumStoreComparatorWrapper; -typedef datastore::DataStoreT<datastore::AlignedEntryRefT<31, 4> > -EnumStoreDataStoreType; +typedef datastore::DataStoreT<datastore::AlignedEntryRefT<31, 4> > EnumStoreDataStoreType; typedef EnumStoreDataStoreType::RefType EnumStoreIndex; typedef vespalib::Array<EnumStoreIndex> EnumStoreIndexVector; typedef vespalib::Array<uint32_t> EnumStoreEnumVector; diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp index dbd9817d9e4..16357ee1134 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "load_utils.hpp" #include "multivalue.h" #include "enumstorebase.h" #include "loadedenumvalue.h" #include "multi_value_mapping.h" +#include <vespa/vespalib/util/array.hpp> using search::multivalue::Value; using search::multivalue::WeightedValue; diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.h b/searchlib/src/vespa/searchlib/attribute/load_utils.h index 370bb39d76c..efae44e61e6 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.h +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.h @@ -3,6 +3,7 @@ #pragma once #include "attributevector.h" +#include <vespa/vespalib/util/arrayref.h> namespace search { namespace attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp index d6f1459f504..c4499fbe05f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp @@ -6,6 +6,7 @@ #include "multivalue.h" #include "enumstorebase.h" #include "attributevector.h" +#include <vespa/vespalib/util/array.hpp> using search::multivalue::Value; using search::multivalue::WeightedValue; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index 8c374d37de8..72f71672175 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -25,8 +25,10 @@ private: ArrayStore _store; public: + MultiValueMapping(const MultiValueMapping &) = delete; + MultiValueMapping & operator = (const MultiValueMapping &) = delete; MultiValueMapping(const datastore::ArrayStoreConfig &storeCfg, - const GrowStrategy &gs = GrowStrategy()); + const GrowStrategy &gs = GrowStrategy()); virtual ~MultiValueMapping(); ConstArrayRef get(uint32_t docId) const { return _store.get(_indices[docId]); } ConstArrayRef getDataForIdx(EntryRef idx) const { return _store.get(idx); } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index 084c74a288a..8bdf36383eb 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -3,6 +3,7 @@ #pragma once #include <vespa/searchlib/datastore/array_store.hpp> +#include <vespa/searchlib/common/rcuvector.hpp> namespace search { namespace attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index b3af07acc0c..6c848510dc6 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -2,9 +2,11 @@ #pragma once -#include <vespa/searchlib/attribute/multinumericenumattribute.h> +#include "multinumericenumattribute.h" +#include "loadednumericvalue.h" +#include <vespa/searchlib/util/fileutil.hpp> #include <vespa/fastlib/io/bufferedfile.h> -#include <vespa/searchlib/attribute/loadednumericvalue.h> + namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp index 9156c25c383..27de73908ac 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp @@ -1,17 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "postingchange.h" #include "multivalue.h" #include "multi_value_mapping.h" #include "postinglistattribute.h" -#include <vespa/searchlib/common/bitvector.h> -#include <map> +#include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { -namespace -{ +namespace { void removeDupAdditions(PostingChange<AttributePosting>::A &additions) @@ -126,6 +124,10 @@ PostingChange<AttributeWeightPosting>::removeDups(void) removeDupRemovals(_removals); } +template <typename P> +PostingChange<P>::PostingChange() { } +template <typename P> +PostingChange<P>::~PostingChange() { } template <typename P> void @@ -167,16 +169,8 @@ public: const WeightedIndex * entriesOld, size_t szOld, AlwaysWeightedIndexVector & added, AlwaysWeightedIndexVector & changed, AlwaysWeightedIndexVector & removed); - ActualChangeComputer(const EnumStoreComparator &compare, - const EnumIndexMapper &mapper) - : _oldEntries(), - _newEntries(), - _cachedMapping(), - _compare(compare), - _mapper(mapper), - _hasFold(mapper.hasFold()) - { - } + ActualChangeComputer(const EnumStoreComparator &compare, const EnumIndexMapper &mapper); + ~ActualChangeComputer(); private: WeightedIndexVector _oldEntries; @@ -267,6 +261,20 @@ public: }; template <typename WeightedIndex> +ActualChangeComputer<WeightedIndex>::ActualChangeComputer(const EnumStoreComparator &compare, + const EnumIndexMapper &mapper) + : _oldEntries(), + _newEntries(), + _cachedMapping(), + _compare(compare), + _mapper(mapper), + _hasFold(mapper.hasFold()) +{ } + +template <typename WeightedIndex> +ActualChangeComputer<WeightedIndex>::~ActualChangeComputer() { } + +template <typename WeightedIndex> void ActualChangeComputer<WeightedIndex>::compute(const WeightedIndex * entriesNew, size_t szNew, const WeightedIndex * entriesOld, size_t szOld, diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.h b/searchlib/src/vespa/searchlib/attribute/postingchange.h index d728fa89fb0..357704d7851 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.h +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.h @@ -23,6 +23,8 @@ public: A _additions; R _removals; + PostingChange(); + ~PostingChange(); inline void add(uint32_t docId, int32_t weight); PostingChange & remove(uint32_t docId) { diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index 847071cf4f4..b816f19ee79 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "postinglistattribute.h" #include "loadednumericvalue.h" -#include "loadedstringvalue.h" #include "enumcomparator.h" +#include <vespa/vespalib/util/array.hpp> -namespace search -{ +namespace search { using attribute::LoadedNumericValue; @@ -21,8 +19,7 @@ PostingListAttributeBase(AttributeVector &attr, _attr(attr), _dict(enumStore.getPostingDictionary()), _esb(enumStore) -{ -} +{ } template <typename P> diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp index d8185a0b614..8cc351e0330 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "singleenumattributesaver.h" #include <vespa/searchlib/util/bufferwriter.h> +#include <vespa/vespalib/util/array.hpp> + using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/attribute/stringattribute.cpp b/searchlib/src/vespa/searchlib/attribute/stringattribute.cpp index 95f38484fae..835bcde310d 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringattribute.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "stringattribute.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.stringattribute"); +#include "stringattribute.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index d4ad5d84004..01128f464f9 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "stringbase.h" -#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/searchlib/util/fileutil.hpp> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.stringbase"); @@ -78,7 +78,6 @@ public: } }; - class SortDataCharEof { public: diff --git a/searchlib/src/vespa/searchlib/btree/btreenodeallocator.cpp b/searchlib/src/vespa/searchlib/btree/btreenodeallocator.cpp index cac07f8c89f..fcf5a41b12a 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenodeallocator.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreenodeallocator.cpp @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreenodeallocator.h" -#include "btreerootbase.h" #include "btreenodeallocator.hpp" +#include <vespa/vespalib/util/array.hpp> -namespace search -{ -namespace btree -{ +template class vespalib::Array<search::datastore::EntryRef>; + +namespace search { +namespace btree { template class BTreeNodeAllocator<uint32_t, uint32_t, NoAggregated, diff --git a/searchlib/src/vespa/searchlib/btree/btreenodeallocator.hpp b/searchlib/src/vespa/searchlib/btree/btreenodeallocator.hpp index 78ab68e454b..0df47e1d186 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenodeallocator.hpp +++ b/searchlib/src/vespa/searchlib/btree/btreenodeallocator.hpp @@ -5,6 +5,7 @@ #include "btreenodeallocator.h" #include "btreerootbase.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/array.hpp> #include "btreenodestore.hpp" namespace search { diff --git a/searchlib/src/vespa/searchlib/btree/minmaxaggregated.h b/searchlib/src/vespa/searchlib/btree/minmaxaggregated.h index 1b876918d1a..90040b129e7 100644 --- a/searchlib/src/vespa/searchlib/btree/minmaxaggregated.h +++ b/searchlib/src/vespa/searchlib/btree/minmaxaggregated.h @@ -3,11 +3,10 @@ #pragma once #include <limits> +#include <stdint.h> -namespace search -{ -namespace btree -{ +namespace search { +namespace btree { class MinMaxAggregated { @@ -18,26 +17,15 @@ public: MinMaxAggregated(void) : _min(std::numeric_limits<int32_t>::max()), _max(std::numeric_limits<int32_t>::min()) - { - } + { } MinMaxAggregated(int32_t min, int32_t max) : _min(min), _max(max) - { - } + { } - int32_t - getMin(void) const - { - return _min; - } - - int32_t - getMax(void) const - { - return _max; - } + int32_t getMin(void) const { return _min; } + int32_t getMax(void) const { return _max; } void add(int32_t val) diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index 4b03fde1428..6669ef2c06b 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -19,6 +19,7 @@ vespa_add_library(searchlib_common OBJECT mapnames.cpp packets.cpp partialbitvector.cpp + rcuvector.cpp resultset.cpp sequencedtaskexecutor.cpp serialnumfileheadercontext.cpp diff --git a/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp b/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp index cf2011b1f2a..4fdf938d649 100644 --- a/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp @@ -1,7 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/common/bitvectorcache.h> -#include <vespa/log/log.h> +#include "bitvectorcache.h" +#include <vespa/vespalib/stllike/hash_set.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".searchlib.common.bitvectorcache"); namespace search { diff --git a/searchlib/src/vespa/searchlib/common/bitvectorcache.h b/searchlib/src/vespa/searchlib/common/bitvectorcache.h index 9a93adfe55a..5622910a69a 100644 --- a/searchlib/src/vespa/searchlib/common/bitvectorcache.h +++ b/searchlib/src/vespa/searchlib/common/bitvectorcache.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchlib/common/condensedbitvectors.h> +#include "condensedbitvectors.h" #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/fastos/dynamiclibrary.h> diff --git a/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp b/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp index bcf2bc34738..ea10403c3ea 100644 --- a/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp +++ b/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp @@ -2,6 +2,7 @@ #include <vespa/searchlib/common/condensedbitvectors.h> #include <vespa/searchlib/common/rcuvector.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::IllegalArgumentException; using vespalib::make_string; diff --git a/searchlib/src/vespa/searchlib/common/condensedbitvectors.h b/searchlib/src/vespa/searchlib/common/condensedbitvectors.h index b8d97cbcb07..fe8a718d46a 100644 --- a/searchlib/src/vespa/searchlib/common/condensedbitvectors.h +++ b/searchlib/src/vespa/searchlib/common/condensedbitvectors.h @@ -3,7 +3,7 @@ #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/generationholder.h> -#include <vector> +#include <vespa/vespalib/util/arrayref.h> #include <set> namespace search { diff --git a/searchlib/src/vespa/searchlib/common/rcuvector.cpp b/searchlib/src/vespa/searchlib/common/rcuvector.cpp new file mode 100644 index 00000000000..abc59b47a48 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/rcuvector.cpp @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "rcuvector.hpp" + +namespace search { +namespace attribute { + +template class RcuVectorBase<uint8_t>; +template class RcuVectorBase<uint16_t>; +template class RcuVectorBase<uint32_t>; +template class RcuVectorBase<int8_t>; +template class RcuVectorBase<int16_t>; +template class RcuVectorBase<int32_t>; +template class RcuVectorBase<int64_t>; +template class RcuVectorBase<float>; +template class RcuVectorBase<double>; + + +} +} diff --git a/searchlib/src/vespa/searchlib/common/rcuvector.h b/searchlib/src/vespa/searchlib/common/rcuvector.h index b7b840a6938..f9d8cf209c6 100644 --- a/searchlib/src/vespa/searchlib/common/rcuvector.h +++ b/searchlib/src/vespa/searchlib/common/rcuvector.h @@ -16,16 +16,9 @@ class RcuVectorHeld : public vespalib::GenerationHeldBase std::unique_ptr<T> _data; public: - RcuVectorHeld(size_t size, std::unique_ptr<T> data) - : vespalib::GenerationHeldBase(size), - _data(std::move(data)) - { - } + RcuVectorHeld(size_t size, std::unique_ptr<T> data); - virtual - ~RcuVectorHeld(void) - { - } + ~RcuVectorHeld(); }; @@ -77,14 +70,11 @@ public: * New capacity is calculated based on old capacity and grow parameters: * nc = oc + (oc * growPercent / 100) + growDelta. **/ - RcuVectorBase(size_t initialCapacity, size_t growPercent, size_t growDelta, - GenerationHolder &genHolder); + RcuVectorBase(size_t initialCapacity, size_t growPercent, size_t growDelta, GenerationHolder &genHolder); + + RcuVectorBase(GrowStrategy growStrategy, GenerationHolder &genHolder); - RcuVectorBase(GrowStrategy growStrategy, GenerationHolder &genHolder) - : RcuVectorBase(growStrategy.getDocsInitialCapacity(), - growStrategy.getDocsGrowPercent(), - growStrategy.getDocsGrowDelta(), - genHolder) {} + ~RcuVectorBase(); /** * Return whether all capacity has been used. If true the next @@ -101,16 +91,9 @@ public: // vector interface // no swap method, use reset() to forget old capacity and holds // NOTE: Unsafe resize/reserve may invalidate data references held by readers! - void unsafe_resize(size_t n) { _data.resize(n); } - void unsafe_reserve(size_t n) { _data.reserve(n); } - void ensure_size(size_t n, T fill = T()) { - if (n > capacity()) { - expand(calcSize(n)); - } - while (size() < n) { - _data.push_back(fill); - } - } + void unsafe_resize(size_t n); + void unsafe_reserve(size_t n); + void ensure_size(size_t n, T fill = T()); void push_back(const T & v) { if (_data.size() < _data.capacity()) { _data.push_back(v); @@ -119,9 +102,7 @@ public: } } - bool - empty(void) const - { + bool empty() const { return _data.empty(); } @@ -131,99 +112,11 @@ public: T & operator[](size_t i) { return _data[i]; } const T & operator[](size_t i) const { return _data[i]; } - void - reset(void) - { - // Assumes no readers at this moment - Array().swap(_data); - _data.reserve(16); - } - - void - shrink(size_t newSize) __attribute__((noinline)); + void reset(); + void shrink(size_t newSize) __attribute__((noinline)); }; template <typename T> -void -RcuVectorBase<T>::expand(size_t newCapacity) { - std::unique_ptr<Array> tmpData(new Array()); - tmpData->reserve(newCapacity); - tmpData->resize(_data.size()); - memcpy(tmpData->begin(), _data.begin(), _data.size() * sizeof(T)); - tmpData->swap(_data); // atomic switch of underlying data - size_t holdSize = tmpData->size() * sizeof(T); - vespalib::GenerationHeldBase::UP hold(new RcuVectorHeld<Array>(holdSize, std::move(tmpData))); - _genHolder.hold(std::move(hold)); -} - -template <typename T> -void -RcuVectorBase<T>::expandAndInsert(const T & v) -{ - expand(calcSize()); - assert(_data.size() < _data.capacity()); - _data.push_back(v); -} - - -template <typename T> -void -RcuVectorBase<T>::shrink(size_t newSize) -{ - // TODO: Extend Array class to support more optimial shrink when - // backing store is memory mapped. - assert(newSize <= _data.size()); - std::unique_ptr<Array> tmpData(new Array()); - tmpData->reserve(newSize); - tmpData->resize(newSize); - for (uint32_t i = 0; i < newSize; ++i) { - (*tmpData)[i] = _data[i]; - } - // Users of RCU vector must ensure that no readers use old size - // after swap. Attribute vectors uses _committedDocIdLimit for this. - tmpData->swap(_data); // atomic switch of underlying data - // Use capacity() instead of size() ? - size_t holdSize = tmpData->size() * sizeof(T); - vespalib::GenerationHeldBase::UP hold(new RcuVectorHeld<Array>(holdSize, std::move(tmpData))); - _genHolder.hold(std::move(hold)); -} - - -template <typename T> -RcuVectorBase<T>::RcuVectorBase(GenerationHolder &genHolder) - : _data(), - _growPercent(100), - _growDelta(0), - _genHolder(genHolder) -{ - _data.reserve(16); -} - -template <typename T> -RcuVectorBase<T>::RcuVectorBase(size_t initialCapacity, - size_t growPercent, - size_t growDelta, - GenerationHolder &genHolder) - : _data(), - _growPercent(growPercent), - _growDelta(growDelta), - _genHolder(genHolder) -{ - _data.reserve(initialCapacity); -} - -template <typename T> -MemoryUsage -RcuVectorBase<T>::getMemoryUsage() const -{ - MemoryUsage retval; - retval.incAllocatedBytes(_data.capacity() * sizeof(T)); - retval.incUsedBytes(_data.size() * sizeof(T)); - return retval; -} - - -template <typename T> class RcuVector : public RcuVectorBase<T> { private: @@ -233,20 +126,10 @@ private: generation_t _generation; GenerationHolder _genHolderStore; - void - expandAndInsert(const T & v) - { - RcuVectorBase<T>::expandAndInsert(v); - _genHolderStore.transferHoldLists(_generation); - } + void expandAndInsert(const T & v); public: - RcuVector() - : RcuVectorBase<T>(_genHolderStore), - _generation(0), - _genHolderStore() - { - } + RcuVector(); /** * Construct a new vector with the given initial capacity and grow @@ -255,44 +138,19 @@ public: * New capacity is calculated based on old capacity and grow parameters: * nc = oc + (oc * growPercent / 100) + growDelta. **/ - RcuVector(size_t initialCapacity, size_t growPercent, size_t growDelta) - : RcuVectorBase<T>(initialCapacity, growPercent, growDelta, - _genHolderStore), - _generation(0), - _genHolderStore() - { - } + RcuVector(size_t initialCapacity, size_t growPercent, size_t growDelta); - RcuVector(GrowStrategy growStrategy) - : RcuVectorBase<T>(growStrategy, _genHolderStore), _generation(0), _genHolderStore() - { - } + RcuVector(GrowStrategy growStrategy); - ~RcuVector() - { - _genHolderStore.clearHoldLists(); - } - - generation_t - getGeneration() const - { - return _generation; - } + ~RcuVector(); - void - setGeneration(generation_t generation) - { - _generation = generation; - } + generation_t getGeneration() const { return _generation; } + void setGeneration(generation_t generation) { _generation = generation; } /** * Remove all old data vectors where generation < firstUsed. **/ - void - removeOldGenerations(generation_t firstUsed) - { - _genHolderStore.trimHoldLists(firstUsed); - } + void removeOldGenerations(generation_t firstUsed); void push_back(const T & v) @@ -304,16 +162,8 @@ public: } } - MemoryUsage - getMemoryUsage() const - { - MemoryUsage retval(RcuVectorBase<T>::getMemoryUsage()); - retval.mergeGenerationHeldBytes(_genHolderStore.getHeldBytes()); - return retval; - } + MemoryUsage getMemoryUsage() const; }; - } } - diff --git a/searchlib/src/vespa/searchlib/common/rcuvector.hpp b/searchlib/src/vespa/searchlib/common/rcuvector.hpp new file mode 100644 index 00000000000..920c0292435 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/rcuvector.hpp @@ -0,0 +1,191 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "rcuvector.h" +#include <vespa/vespalib/util/array.hpp> + +namespace search { +namespace attribute { + +template <typename T> +RcuVectorHeld<T>::RcuVectorHeld(size_t size, std::unique_ptr<T> data) + : vespalib::GenerationHeldBase(size), + _data(std::move(data)) +{ } + +template <typename T> +RcuVectorHeld<T>::~RcuVectorHeld() { } + +template <typename T> +void +RcuVectorBase<T>::unsafe_resize(size_t n) { + _data.resize(n); +} + +template <typename T> +void +RcuVectorBase<T>::unsafe_reserve(size_t n) { + _data.reserve(n); +} + +template <typename T> +void +RcuVectorBase<T>::ensure_size(size_t n, T fill) { + if (n > capacity()) { + expand(calcSize(n)); + } + while (size() < n) { + _data.push_back(fill); + } +} + +template <typename T> +void +RcuVectorBase<T>::reset() { + // Assumes no readers at this moment + Array().swap(_data); + _data.reserve(16); +} + +template <typename T> +RcuVectorBase<T>::RcuVectorBase(GrowStrategy growStrategy, GenerationHolder &genHolder) + : RcuVectorBase(growStrategy.getDocsInitialCapacity(), growStrategy.getDocsGrowPercent(), + growStrategy.getDocsGrowDelta(), genHolder) +{ } + +template <typename T> +RcuVectorBase<T>::~RcuVectorBase() { } + +template <typename T> +void +RcuVectorBase<T>::expand(size_t newCapacity) { + std::unique_ptr<Array> tmpData(new Array()); + tmpData->reserve(newCapacity); + tmpData->resize(_data.size()); + memcpy(tmpData->begin(), _data.begin(), _data.size() * sizeof(T)); + tmpData->swap(_data); // atomic switch of underlying data + size_t holdSize = tmpData->size() * sizeof(T); + vespalib::GenerationHeldBase::UP hold(new RcuVectorHeld<Array>(holdSize, std::move(tmpData))); + _genHolder.hold(std::move(hold)); +} + +template <typename T> +void +RcuVectorBase<T>::expandAndInsert(const T & v) +{ + expand(calcSize()); + assert(_data.size() < _data.capacity()); + _data.push_back(v); +} + + +template <typename T> +void +RcuVectorBase<T>::shrink(size_t newSize) +{ + // TODO: Extend Array class to support more optimial shrink when + // backing store is memory mapped. + assert(newSize <= _data.size()); + std::unique_ptr<Array> tmpData(new Array()); + tmpData->reserve(newSize); + tmpData->resize(newSize); + for (uint32_t i = 0; i < newSize; ++i) { + (*tmpData)[i] = _data[i]; + } + // Users of RCU vector must ensure that no readers use old size + // after swap. Attribute vectors uses _committedDocIdLimit for this. + tmpData->swap(_data); // atomic switch of underlying data + // Use capacity() instead of size() ? + size_t holdSize = tmpData->size() * sizeof(T); + vespalib::GenerationHeldBase::UP hold(new RcuVectorHeld<Array>(holdSize, std::move(tmpData))); + _genHolder.hold(std::move(hold)); +} + + +template <typename T> +RcuVectorBase<T>::RcuVectorBase(GenerationHolder &genHolder) + : _data(), + _growPercent(100), + _growDelta(0), + _genHolder(genHolder) +{ + _data.reserve(16); +} + +template <typename T> +RcuVectorBase<T>::RcuVectorBase(size_t initialCapacity, + size_t growPercent, + size_t growDelta, + GenerationHolder &genHolder) + : _data(), + _growPercent(growPercent), + _growDelta(growDelta), + _genHolder(genHolder) +{ + _data.reserve(initialCapacity); +} + +template <typename T> +MemoryUsage +RcuVectorBase<T>::getMemoryUsage() const +{ + MemoryUsage retval; + retval.incAllocatedBytes(_data.capacity() * sizeof(T)); + retval.incUsedBytes(_data.size() * sizeof(T)); + return retval; +} + +template <typename T> +void +RcuVector<T>::expandAndInsert(const T & v) +{ + RcuVectorBase<T>::expandAndInsert(v); + _genHolderStore.transferHoldLists(_generation); +} + +template <typename T> +RcuVector<T>::RcuVector() + : RcuVectorBase<T>(_genHolderStore), + _generation(0), + _genHolderStore() +{ } + +template <typename T> +RcuVector<T>::RcuVector(size_t initialCapacity, size_t growPercent, size_t growDelta) + : RcuVectorBase<T>(initialCapacity, growPercent, growDelta, _genHolderStore), + _generation(0), + _genHolderStore() +{ } + +template <typename T> +RcuVector<T>::RcuVector(GrowStrategy growStrategy) + : RcuVectorBase<T>(growStrategy, _genHolderStore), + _generation(0), + _genHolderStore() +{ } + +template <typename T> +RcuVector<T>::~RcuVector() +{ + _genHolderStore.clearHoldLists(); +} + +template <typename T> +void +RcuVector<T>::removeOldGenerations(generation_t firstUsed) +{ + _genHolderStore.trimHoldLists(firstUsed); +} + +template <typename T> +MemoryUsage +RcuVector<T>::getMemoryUsage() const +{ + MemoryUsage retval(RcuVectorBase<T>::getMemoryUsage()); + retval.mergeGenerationHeldBytes(_genHolderStore.getHeldBytes()); + return retval; +} + +} +} diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp index 16ab7ab5ee5..f31803bb627 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp @@ -1,19 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".common.sequencedtaskexecutor"); - #include "sequencedtaskexecutor.h" -#include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::BlockingThreadStackExecutor; -namespace search -{ +namespace search { -namespace -{ +namespace { constexpr uint32_t stackSize = 128 * 1024; diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index b7fdd0b85af..8878e8bdd19 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -2,13 +2,14 @@ // Copyright (C) 2001-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include "sortresults.h" #include <vespa/searchlib/util/sort.h> #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/document/base/globalid.h> +#include <vespa/vespalib/util/array.hpp> + #include <vespa/log/log.h> LOG_SETUP(".search.attribute.sortresults"); diff --git a/searchlib/src/vespa/searchlib/datastore/allocator.h b/searchlib/src/vespa/searchlib/datastore/allocator.h index 047380012aa..404e1c54d44 100644 --- a/searchlib/src/vespa/searchlib/datastore/allocator.h +++ b/searchlib/src/vespa/searchlib/datastore/allocator.h @@ -5,6 +5,7 @@ #include "datastorebase.h" #include "entryref.h" #include "handle.h" +#include <vespa/vespalib/util/arrayref.h> namespace search { namespace datastore { diff --git a/searchlib/src/vespa/searchlib/datastore/buffer_type.cpp b/searchlib/src/vespa/searchlib/datastore/buffer_type.cpp index bff68306398..8d01bc09710 100644 --- a/searchlib/src/vespa/searchlib/datastore/buffer_type.cpp +++ b/searchlib/src/vespa/searchlib/datastore/buffer_type.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "buffer_type.h" +#include <algorithm> namespace search { namespace datastore { @@ -18,26 +18,24 @@ BufferTypeBase::BufferTypeBase(uint32_t clusterSize, _holdBuffers(0), _activeUsedElems(0), _holdUsedElems(0), - _lastUsedElems(NULL) -{ -} + _lastUsedElems(nullptr) +{ } BufferTypeBase::BufferTypeBase(uint32_t clusterSize, uint32_t minClusters, uint32_t maxClusters) : BufferTypeBase(clusterSize, minClusters, maxClusters, 0u) -{ -} +{ } -BufferTypeBase::~BufferTypeBase(void) +BufferTypeBase::~BufferTypeBase() { assert(_activeBuffers == 0); assert(_holdBuffers == 0); assert(_activeUsedElems == 0); assert(_holdUsedElems == 0); - assert(_lastUsedElems == NULL); + assert(_lastUsedElems == nullptr); } size_t diff --git a/searchlib/src/vespa/searchlib/datastore/buffer_type.h b/searchlib/src/vespa/searchlib/datastore/buffer_type.h index 30282a19b97..d937a60209b 100644 --- a/searchlib/src/vespa/searchlib/datastore/buffer_type.h +++ b/searchlib/src/vespa/searchlib/datastore/buffer_type.h @@ -3,6 +3,8 @@ #pragma once #include <assert.h> +#include <cstdint> +#include <sys/types.h> namespace search { namespace datastore { @@ -85,14 +87,9 @@ public: BufferType(const BufferType &rhs) = delete; BufferType & operator=(const BufferType &rhs) = delete; - BufferType(uint32_t clusterSize, uint32_t minClusters, uint32_t maxClusters) - : BufferTypeBase(clusterSize, minClusters, maxClusters), - _emptyEntry() - { } - BufferType(uint32_t clusterSize, uint32_t minClusters, uint32_t maxClusters, uint32_t numClustersForNewBuffer) - : BufferTypeBase(clusterSize, minClusters, maxClusters, numClustersForNewBuffer), - _emptyEntry() - { } + BufferType(uint32_t clusterSize, uint32_t minClusters, uint32_t maxClusters); + BufferType(uint32_t clusterSize, uint32_t minClusters, uint32_t maxClusters, uint32_t numClustersForNewBuffer); + ~BufferType(); void destroyElements(void *buffer, size_t numElements) override; void fallbackCopy(void *newBuffer, const void *oldBuffer, size_t numElements) override; void initializeReservedElements(void *buffer, size_t reservedElements) override; @@ -100,6 +97,20 @@ public: size_t elementSize() const override { return sizeof(EntryType); } }; +template <typename EntryType> +BufferType<EntryType>::BufferType(uint32_t clusterSize, uint32_t minClusters, uint32_t maxClusters) + : BufferTypeBase(clusterSize, minClusters, maxClusters), + _emptyEntry() +{ } + +template <typename EntryType> +BufferType<EntryType>::BufferType(uint32_t clusterSize, uint32_t minClusters, uint32_t maxClusters, uint32_t numClustersForNewBuffer) + : BufferTypeBase(clusterSize, minClusters, maxClusters, numClustersForNewBuffer), + _emptyEntry() +{ } + +template <typename EntryType> +BufferType<EntryType>::~BufferType() { } template <typename EntryType> void diff --git a/searchlib/src/vespa/searchlib/datastore/datastore.cpp b/searchlib/src/vespa/searchlib/datastore/datastore.cpp index e39d29ef9a9..13b3bf3a6e2 100644 --- a/searchlib/src/vespa/searchlib/datastore/datastore.cpp +++ b/searchlib/src/vespa/searchlib/datastore/datastore.cpp @@ -2,6 +2,8 @@ #include "datastore.h" #include "datastore.hpp" +#include <vespa/vespalib/util/array.hpp> +#include <vespa/searchlib/common/rcuvector.hpp> namespace search { namespace datastore { @@ -10,3 +12,8 @@ template class DataStoreT<EntryRefT<22> >; } // namespace datastore } // namespace search + +template void vespalib::Array<search::datastore::DataStoreBase::ElemHold1ListElem>::increase(size_t); +template class search::attribute::RcuVector<search::datastore::EntryRef>; +template class search::attribute::RcuVectorBase<search::datastore::EntryRef>; +//template void search::attribute::RcuVectorBase<search::datastore::EntryRef>::expandAndInsert(const search::datastore::EntryRef &);
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/datastore/datastore.h b/searchlib/src/vespa/searchlib/datastore/datastore.h index 82ceb855347..3124fa33826 100644 --- a/searchlib/src/vespa/searchlib/datastore/datastore.h +++ b/searchlib/src/vespa/searchlib/datastore/datastore.h @@ -25,16 +25,13 @@ template <typename RefT = EntryRefT<22> > class DataStoreT : public DataStoreBase { private: - DataStoreT(const DataStoreT &rhs); - - DataStoreT & - operator=(const DataStoreT &rhs); public: typedef RefT RefType; - DataStoreT(void); - - ~DataStoreT(void); + DataStoreT(const DataStoreT &rhs) = delete; + DataStoreT &operator=(const DataStoreT &rhs) = delete; + DataStoreT(); + ~DataStoreT(); /** * Increase number of dead elements in buffer. diff --git a/searchlib/src/vespa/searchlib/datastore/datastore.hpp b/searchlib/src/vespa/searchlib/datastore/datastore.hpp index cfaaaf34951..56db025c28c 100644 --- a/searchlib/src/vespa/searchlib/datastore/datastore.hpp +++ b/searchlib/src/vespa/searchlib/datastore/datastore.hpp @@ -6,6 +6,7 @@ #include "allocator.hpp" #include "free_list_allocator.hpp" #include "raw_allocator.hpp" +#include <vespa/vespalib/util/array.hpp> namespace search { namespace datastore { diff --git a/searchlib/src/vespa/searchlib/datastore/datastorebase.cpp b/searchlib/src/vespa/searchlib/datastore/datastorebase.cpp index e210d9a0705..4e3f1c6be84 100644 --- a/searchlib/src/vespa/searchlib/datastore/datastorebase.cpp +++ b/searchlib/src/vespa/searchlib/datastore/datastorebase.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "datastore.h" +#include <vespa/vespalib/util/array.hpp> using vespalib::GenerationHeldBase; diff --git a/searchlib/src/vespa/searchlib/datastore/datastorebase.h b/searchlib/src/vespa/searchlib/datastore/datastorebase.h index 3807c5b2785..db688a5f4b4 100644 --- a/searchlib/src/vespa/searchlib/datastore/datastorebase.h +++ b/searchlib/src/vespa/searchlib/datastore/datastorebase.h @@ -17,15 +17,8 @@ class DataStoreBase private: DataStoreBase(const DataStoreBase &rhs); - DataStoreBase & - operator=(const DataStoreBase &rhs); -protected: - typedef vespalib::GenerationHandler::generation_t generation_t; - typedef vespalib::GenerationHandler::sgeneration_t sgeneration_t; - - std::vector<void *> _buffers; // For fast mapping with known types - std::vector<uint32_t> _activeBufferIds; // typeId -> active buffer - + DataStoreBase &operator=(const DataStoreBase &rhs); +public: // Hold list before freeze, before knowing how long elements must be held class ElemHold1ListElem { @@ -34,12 +27,18 @@ protected: uint64_t _len; // Aligned length ElemHold1ListElem(EntryRef ref, uint64_t len) - : _ref(ref), - _len(len) - { - } + : _ref(ref), + _len(len) + { } }; +protected: + typedef vespalib::GenerationHandler::generation_t generation_t; + typedef vespalib::GenerationHandler::sgeneration_t sgeneration_t; + + std::vector<void *> _buffers; // For fast mapping with known types + std::vector<uint32_t> _activeBufferIds; // typeId -> active buffer + // Hold list at freeze, when knowing how long elements must be held class ElemHold2ListElem : public ElemHold1ListElem { diff --git a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp index c0f89e54b23..61897d2785b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp @@ -1,19 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "diskindex.h" #include "disktermblueprint.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h> -#include <vespa/searchlib/queryeval/split_float.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> -#include <vespa/searchlib/queryeval/termasstring.h> #include <vespa/searchlib/util/dirtraverse.h> -#include <vespa/searchlib/query/tree/simplequery.h> -#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/hash_set.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/cache.hpp> #include "pagedict4randread.h" #include "fileheader.h" -#include "bitvectorkeyscope.h" + #include <vespa/log/log.h> LOG_SETUP(".diskindex.diskindex"); @@ -21,11 +20,9 @@ using namespace search::index; using namespace search::query; using namespace search::queryeval; -namespace search -{ +namespace search { -namespace diskindex -{ +namespace diskindex { void swap(DiskIndex::LookupResult & a, DiskIndex::LookupResult & b) { @@ -333,8 +330,7 @@ DiskIndex::calculateSize() } -namespace -{ +namespace { DiskIndex::LookupResult _G_nothing; @@ -345,8 +341,7 @@ public: _diskIndex(diskIndex), _fieldIds(fieldIds), _cache() - { - } + { } const DiskIndex::LookupResult & lookup(const vespalib::string & word, uint32_t fieldId) { Cache::const_iterator it = _cache.find(word); diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index f1c34ceac43..bf688b4766c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "indexbuilder.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/common/documentsummary.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/searchlib/diskindex/fieldwriter.h> +#include <vespa/vespalib/util/array.hpp> #include <vespa/log/log.h> LOG_SETUP(".diskindex.indexbuilder"); diff --git a/searchlib/src/vespa/searchlib/docstore/cachestats.h b/searchlib/src/vespa/searchlib/docstore/cachestats.h index 216b62f199a..5f4e690ce8c 100644 --- a/searchlib/src/vespa/searchlib/docstore/cachestats.h +++ b/searchlib/src/vespa/searchlib/docstore/cachestats.h @@ -2,6 +2,9 @@ #pragma once +#include <cstdint> +#include <sys/types.h> + namespace search { struct CacheStats { @@ -15,16 +18,14 @@ struct CacheStats { misses(0), elements(0), memory_used(0) - { - } + { } CacheStats(size_t hit, size_t miss, size_t elem, size_t mem) : hits(hit), misses(miss), elements(elem), memory_used(mem) - { - } + { } CacheStats & operator+=(const CacheStats &rhs) diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index 931e3a11353..05a9127153b 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/docstore/chunk.h> -#include <vespa/searchlib/docstore/chunkformats.h> -#include <vespa/vespalib/stllike/hash_map.h> + +#include "chunk.h" +#include "chunkformats.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp index 44032a1a73d..b3630a6f916 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp +++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp @@ -2,8 +2,9 @@ #include "compacter.h" #include "logdatastore.h" -#include <vespa/log/log.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".searchlib.docstore.compacter"); namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index 038e16bf872..b9f005ddfcc 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "cachestats.h" #include "documentstore.h" #include "visitcache.h" #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/atomic.h> -#include <vespa/vespalib/stllike/cache.h> +#include <vespa/vespalib/stllike/cache.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index 13ef079c586..0ebb81ee18f 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/docstore/filechunk.h> +#include "filechunk.h" +#include "data_store_file_chunk_stats.h" #include <vespa/searchlib/util/filekit.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <stdexcept> -#include <map> #include <vespa/vespalib/data/fileheader.h> -#include "data_store_file_chunk_stats.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".search.filechunk"); @@ -19,8 +18,7 @@ using vespalib::getLastErrorString; using vespalib::getErrorString; -namespace search -{ +namespace search { namespace { diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 091aa8949b8..44a8d6b640c 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -1,21 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "storebybucket.h" +#include "compacter.h" #include "logdatastore.h" #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/benchmark_timer.h> -#include <stdexcept> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.docstore.logdatastore"); #include <vespa/vespalib/data/fileheader.h> -#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/searchlib/common/rcuvector.hpp> #include <thread> -#include "compacter.h" -namespace search -{ +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.docstore.logdatastore"); + +namespace search { using vespalib::LockGuard; using vespalib::getLastErrorString; diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp index 0b171ef5f4f..86bd5aa60cf 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp @@ -2,6 +2,7 @@ #include "storebybucket.h" #include <vespa/vespalib/util/closuretask.h> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { namespace docstore { @@ -23,6 +24,8 @@ StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & e createChunk().swap(_current); } +StoreByBucket::~StoreByBucket() { } + void StoreByBucket::add(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) { diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h index b6a25ef1dff..4837dc0e025 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h @@ -27,6 +27,8 @@ class StoreByBucket public: StoreByBucket(vespalib::MemoryDataStore & backingMemory, const document::CompressionConfig & compression); StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & executor, const CompressionConfig & compression); + StoreByBucket(StoreByBucket &&) = default; + ~StoreByBucket(); class IWrite { public: using BucketId=document::BucketId; diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 2f957410bb9..065c8141b9e 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "visitcache.h" +#include <vespa/vespalib/stllike/cache.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { namespace docstore { @@ -202,6 +204,8 @@ VisitCache::Cache::Cache(BackingStore & b, size_t maxBytes) : Parent(b, maxBytes) { } +VisitCache::Cache::~Cache() { } + void VisitCache::Cache::removeKey(uint32_t subKey) { // Need to take hashLock diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index ccec52cd598..22b20f75355 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -135,6 +135,7 @@ private: class Cache : public vespalib::cache<CacheParams> { public: Cache(BackingStore & b, size_t maxBytes); + ~Cache(); CompressedBlobSet readSet(const KeySet & keys); void removeKey(uint32_t key); private: diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index c462b23341f..548a81339bb 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -1,16 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/docstore/writeablefilechunk.h> -#include <stdexcept> +#include "writeablefilechunk.h" +#include "data_store_file_chunk_stats.h" #include <vespa/vespalib/util/closuretask.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/log/log.h> -#include <map> -#include <limits> +#include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> -#include "data_store_file_chunk_stats.h" +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".search.writeablefilechunk"); diff --git a/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp b/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp index 8f90ebdfc75..c24b4e25821 100644 --- a/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp +++ b/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp @@ -1,15 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".engine.propertiesmap"); #include "propertiesmap.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { namespace engine { search::fef::Properties PropertiesMap::_emptyProperties; +PropertiesMap::PropertiesMap() + : _propertiesMap() +{ } + +PropertiesMap::~PropertiesMap() { } + search::fef::Properties & PropertiesMap::lookupCreate(const vespalib::stringref &name) { @@ -28,3 +32,5 @@ PropertiesMap::lookup(const vespalib::stringref &name) const } // namespace engine } // namespace search + +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, search::fef::Properties);
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/engine/propertiesmap.h b/searchlib/src/vespa/searchlib/engine/propertiesmap.h index f6f34512483..1c57d8bc9e0 100644 --- a/searchlib/src/vespa/searchlib/engine/propertiesmap.h +++ b/searchlib/src/vespa/searchlib/engine/propertiesmap.h @@ -2,7 +2,6 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/common/mapnames.h> @@ -35,6 +34,9 @@ private: public: typedef PropsMap::const_iterator ITR; + PropertiesMap(); + ~PropertiesMap(); + /** * Obtain a named collection of properties. This method will * create the properties if they did not exist yet. diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h index 23cc8327b44..5550f30bf2d 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h @@ -2,13 +2,12 @@ #pragma once -#include <vespa/vespalib/stllike/hash_map.h> +#include "utils.h" #include <vespa/searchlib/fef/blueprint.h> #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/attribute/multivalue.h> #include <vespa/vespalib/hwaccelrated/iaccelrated.h> -#include "utils.h" - +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp index e1c558c7603..1c491cf1871 100644 --- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "flow_completeness_feature.h" +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".features.flowcompleteness"); -#include "flow_completeness_feature.h" -#include <vespa/vespalib/stllike/hash_map.h> namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.cpp b/searchlib/src/vespa/searchlib/fef/objectstore.cpp index 9e7aa5d9b81..bc67cb0bfaf 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.cpp +++ b/searchlib/src/vespa/searchlib/fef/objectstore.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/fef/objectstore.h> +#include "objectstore.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/properties.cpp b/searchlib/src/vespa/searchlib/fef/properties.cpp index 7351bc4e5bf..cf0cbe2d1d5 100644 --- a/searchlib/src/vespa/searchlib/fef/properties.cpp +++ b/searchlib/src/vespa/searchlib/fef/properties.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.properties"); #include "properties.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { namespace fef { @@ -82,7 +80,7 @@ Properties::Properties() Properties::~Properties() { - LOG_ASSERT(_numValues >= _data.size()); + assert(_numValues >= _data.size()); } Properties & @@ -267,3 +265,5 @@ void Properties::swap(Properties & rhs) } // namespace fef } // namespace search + +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, search::fef::Property::Values);
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/grouping/collect.cpp b/searchlib/src/vespa/searchlib/grouping/collect.cpp index bbcfe0790bf..a4bb2641704 100644 --- a/searchlib/src/vespa/searchlib/grouping/collect.cpp +++ b/searchlib/src/vespa/searchlib/grouping/collect.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/grouping/collect.h> -#include <cstdlib> + +#include "collect.h" +#include <vespa/vespalib/util/array.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/grouping/groupengine.cpp b/searchlib/src/vespa/searchlib/grouping/groupengine.cpp index 48ecf6931ee..4a33cf32cdf 100644 --- a/searchlib/src/vespa/searchlib/grouping/groupengine.cpp +++ b/searchlib/src/vespa/searchlib/grouping/groupengine.cpp @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/grouping/groupengine.h> + +#include "groupengine.h" #include <vespa/searchlib/expression/nullresultnode.h> #include <vespa/searchlib/common/sort.h> +#include <vespa/vespalib/stllike/hash_set.hpp> namespace search { @@ -89,6 +90,11 @@ void GroupEngine::merge(const GroupEngine & b) } } +std::unique_ptr<GroupEngine::Children> +GroupEngine::createChildren() { + return std::unique_ptr<Children>(new Children(0, GroupHash(*this), GroupEqual(*this))); +} + #if 0 int GroupEngine::cmpRank(GroupRef a, GroupRef b) const { diff --git a/searchlib/src/vespa/searchlib/grouping/groupengine.h b/searchlib/src/vespa/searchlib/grouping/groupengine.h index 4ac29d77b3d..2f9a83d45e1 100644 --- a/searchlib/src/vespa/searchlib/grouping/groupengine.h +++ b/searchlib/src/vespa/searchlib/grouping/groupengine.h @@ -74,7 +74,7 @@ public: virtual void merge(Children & children, const GroupEngine & b); virtual void merge(const GroupEngine & b); - std::unique_ptr<Children> createChildren() { return std::unique_ptr<Children>(new Children(0, GroupHash(*this), GroupEqual(*this))); } + std::unique_ptr<Children> createChildren(); virtual aggregation::Group::UP getGroup(GroupRef ref) const; aggregation::Group::UP getRootGroup() const { return getGroup(GroupRef(0)); } diff --git a/searchlib/src/vespa/searchlib/index/dictionaryfile.h b/searchlib/src/vespa/searchlib/index/dictionaryfile.h index 8a3f101ba99..db9524d129c 100644 --- a/searchlib/src/vespa/searchlib/index/dictionaryfile.h +++ b/searchlib/src/vespa/searchlib/index/dictionaryfile.h @@ -8,20 +8,14 @@ #include <map> #include <vector> #include <string> -#include <limits> -namespace search -{ - -namespace common -{ +class FastOS_FileInterface; -class FileHeaderContext; +namespace search { -} +namespace common { class FileHeaderContext; } -namespace index -{ +namespace index { /** * Interface for dictionary file containing words and counts for words. @@ -29,37 +23,24 @@ namespace index class DictionaryFileSeqRead : public PostingListCountFileSeqRead { public: - DictionaryFileSeqRead(void) - { - } + DictionaryFileSeqRead() { } - virtual - ~DictionaryFileSeqRead(void); + virtual ~DictionaryFileSeqRead(); /** * Read word and counts. Only nonzero counts are returned. If at * end of dictionary then noWordNumHigh() is returned as word number. */ - virtual void - readWord(vespalib::string &word, - uint64_t &wordNum, - PostingListCounts &counts) = 0; + virtual void readWord(vespalib::string &word, uint64_t &wordNum, PostingListCounts &counts) = 0; /** * Open dictionary file for sequential read. */ - virtual bool - open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) = 0; + virtual bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) = 0; - static uint64_t - noWordNum(void) - { - return 0u; - } + static uint64_t noWordNum() { return 0u; } - static uint64_t - noWordNumHigh(void) - { + static uint64_t noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } }; @@ -71,19 +52,14 @@ class DictionaryFileSeqWrite : public PostingListCountFileSeqWrite { protected: public: - DictionaryFileSeqWrite(void) - { - } + DictionaryFileSeqWrite() { } - virtual - ~DictionaryFileSeqWrite(void); + virtual~DictionaryFileSeqWrite(); /** * Write word and counts. Only nonzero counts should be supplied. */ - virtual void - writeWord(const vespalib::stringref &word, - const PostingListCounts &counts) = 0; + virtual void writeWord(const vespalib::stringref &word, const PostingListCounts &counts) = 0; }; @@ -96,40 +72,28 @@ protected: // Can be examined after open bool _memoryMapped; public: - DictionaryFileRandRead(void); + DictionaryFileRandRead(); - virtual - ~DictionaryFileRandRead(void); + virtual ~DictionaryFileRandRead(); - virtual bool - lookup(const vespalib::stringref &word, - uint64_t &wordNum, - PostingListOffsetAndCounts &offsetAndCounts) = 0; + virtual bool lookup(const vespalib::stringref &word, uint64_t &wordNum, + PostingListOffsetAndCounts &offsetAndCounts) = 0; /** * Open dictionary file for random read. */ - virtual bool - open(const vespalib::string &name, - const TuneFileRandRead &tuneFileRead) = 0; + virtual bool open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) = 0; /** * Close dictionary file. */ - virtual bool - close(void) = 0; + virtual bool close() = 0; - bool - getMemoryMapped(void) const - { - return _memoryMapped; - } + bool getMemoryMapped() const { return _memoryMapped; } - virtual uint64_t - getNumWordIds(void) const = 0; + virtual uint64_t getNumWordIds(void) const = 0; protected: - void - afterOpen(FastOS_FileInterface &file); + void afterOpen(FastOS_FileInterface &file); }; } // namespace index diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.h b/searchlib/src/vespa/searchlib/index/postinglistfile.h index 1518948cc6f..95bb3c39a70 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.h @@ -8,30 +8,18 @@ #include <map> #include <string> -namespace vespalib -{ - -class nbostream; - -} - -namespace search -{ +class FastOS_FileInterface; -namespace common -{ - -class FileHeaderContext; +namespace vespalib { class nbostream; } -} +namespace search { -namespace index -{ +namespace common { class FileHeaderContext; } +namespace index { class DocIdAndFeatures; - /** * Interface for posting list files containing document ids and features * for words. @@ -42,16 +30,14 @@ protected: PostingListCounts _counts; unsigned int _residueDocs; // Docids left to read for word public: - PostingListFileSeqRead(void); + PostingListFileSeqRead(); - virtual - ~PostingListFileSeqRead(void); + virtual ~PostingListFileSeqRead(); /** * Read document id and features. */ - virtual void - readDocIdAndFeatures(DocIdAndFeatures &features) = 0; + virtual void readDocIdAndFeatures(DocIdAndFeatures &features) = 0; /** * Checkpoint write. Used at semi-regular intervals during indexing @@ -59,53 +45,44 @@ public: * flush from memory to disk, and possibly also sync to permanent * storage media. */ - virtual void - checkPointWrite(vespalib::nbostream &out) = 0; + virtual void checkPointWrite(vespalib::nbostream &out) = 0; /** * Checkpoint read. Used when resuming indexing after an interrupt. */ - virtual void - checkPointRead(vespalib::nbostream &in) = 0; + virtual void checkPointRead(vespalib::nbostream &in) = 0; /** * Read counts for a word. */ - virtual void - readCounts(const PostingListCounts &counts) = 0; + virtual void readCounts(const PostingListCounts &counts) = 0; /** * Open posting list file for sequential read. */ - virtual bool - open(const vespalib::string &name, - const TuneFileSeqRead &tuneFileRead) = 0; + virtual bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) = 0; /** * Close posting list file. */ - virtual bool - close(void) = 0; + virtual bool close() = 0; /* * Get current parameters. */ - virtual void - getParams(PostingListParams ¶ms); + virtual void getParams(PostingListParams ¶ms); /* * Set (word, docid) feature parameters. * * Typically can only enable or disable cooked features. */ - virtual void - setFeatureParams(const PostingListParams ¶ms); + virtual void setFeatureParams(const PostingListParams ¶ms); /* * Get current (word, docid) feature parameters. */ - virtual void - getFeatureParams(PostingListParams ¶ms); + virtual void getFeatureParams(PostingListParams ¶ms); // Methods used when generating posting list for common word pairs. @@ -115,8 +92,7 @@ public: * * @return current posting offset, measured in bits. */ - virtual uint64_t - getCurrentPostingOffset(void) const = 0; + virtual uint64_t getCurrentPostingOffset() const = 0; /** * Set current posting offset, measured in bits. First posting @@ -127,25 +103,14 @@ public: * @param readAheadOffset end of posting list for either this or a * later word pair, depending on disk seek cost. */ - virtual void - setPostingOffset(uint64_t offset, - uint64_t endOffset, - uint64_t readAheadOffset) = 0; + virtual void setPostingOffset(uint64_t offset, uint64_t endOffset, uint64_t readAheadOffset) = 0; /** * Get counts read by last readCounts(). */ - const PostingListCounts & - getCounts(void) const - { - return _counts; - } - - PostingListCounts & - getCounts(void) - { - return _counts; - } + const PostingListCounts &getCounts() const { return _counts; } + + PostingListCounts &getCounts() { return _counts; } }; /** @@ -157,23 +122,19 @@ class PostingListFileSeqWrite protected: PostingListCounts _counts; public: - PostingListFileSeqWrite(void); - - virtual - ~PostingListFileSeqWrite(void); + PostingListFileSeqWrite(); + virtual ~PostingListFileSeqWrite(); /** * Write document id and features. */ - virtual void - writeDocIdAndFeatures(const DocIdAndFeatures &features) = 0; + virtual void writeDocIdAndFeatures(const DocIdAndFeatures &features) = 0; /** * Flush word (during write) after it is complete to buffers, i.e. * prepare for next word, but not for application crash. */ - virtual void - flushWord(void) = 0; + virtual void flushWord() = 0; /** * Checkpoint write. Used at semi-regular intervals during indexing @@ -181,14 +142,12 @@ public: * flush from memory to disk, and possibly also sync to permanent * storage media. */ - virtual void - checkPointWrite(vespalib::nbostream &out) = 0; + virtual void checkPointWrite(vespalib::nbostream &out) = 0; /** * Checkpoint read. Used when resuming indexing after an interrupt. */ - virtual void - checkPointRead(vespalib::nbostream &in) = 0; + virtual void checkPointRead(vespalib::nbostream &in) = 0; /** * Open posting list file for sequential write. @@ -201,38 +160,29 @@ public: /** * Close posting list file. */ - virtual bool - close(void) = 0; + virtual bool close() = 0; /* * Set parameters. */ - virtual void - setParams(const PostingListParams ¶ms); + virtual void setParams(const PostingListParams ¶ms); /* * Get current parameters. */ - virtual void - getParams(PostingListParams ¶ms); + virtual void getParams(PostingListParams ¶ms); /* * Set (word, docid) feature parameters. */ - virtual void - setFeatureParams(const PostingListParams ¶ms); + virtual void setFeatureParams(const PostingListParams ¶ms); /* * Get current (word, docid) feature parameters. */ - virtual void - getFeatureParams(PostingListParams ¶ms); + virtual void getFeatureParams(PostingListParams ¶ms); - PostingListCounts & - getCounts(void) - { - return _counts; - } + PostingListCounts &getCounts(void) { return _counts; } }; @@ -248,10 +198,8 @@ protected: public: typedef std::shared_ptr<PostingListFileRandRead> SP; - PostingListFileRandRead(void); - - virtual - ~PostingListFileRandRead(void); + PostingListFileRandRead(); + virtual ~PostingListFileRandRead(); /** * Create iterator for single word. Semantic lifetime of counts and @@ -280,25 +228,17 @@ public: /** * Open posting list file for random read. */ - virtual bool - open(const vespalib::string &name, - const TuneFileRandRead &tuneFileRead) = 0; + virtual bool open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) = 0; /** * Close posting list file. */ - virtual bool - close(void) = 0; + virtual bool close() = 0; - bool - getMemoryMapped(void) const - { - return _memoryMapped; - } + bool getMemoryMapped() const { return _memoryMapped; } protected: - void - afterOpen(FastOS_FileInterface &file); + void afterOpen(FastOS_FileInterface &file); }; @@ -312,11 +252,8 @@ protected: bool _ownLower; public: - PostingListFileRandReadPassThrough(PostingListFileRandRead *lower, - bool ownLower); - - virtual - ~PostingListFileRandReadPassThrough(void); + PostingListFileRandReadPassThrough(PostingListFileRandRead *lower, bool ownLower); + virtual~PostingListFileRandReadPassThrough(); virtual search::queryeval::SearchIterator * createIterator(const PostingListCounts &counts, @@ -330,15 +267,12 @@ public: uint32_t numSegments, PostingListHandle &handle); - virtual bool - open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead); + virtual bool open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead); - virtual bool - close(void); + virtual bool close(); }; } // namespace index -} // namespace search - +} // namespace search
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp index 061291709e4..f92fc22d30d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp @@ -1,9 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".memoryindex.compact_document_words_store"); + #include "compact_document_words_store.h" #include <vespa/searchlib/datastore/datastore.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> +LOG_SETUP(".memoryindex.compact_document_words_store"); namespace search { namespace memoryindex { @@ -35,6 +36,13 @@ serialize(const Builder &builder, uint32_t *begin) } +CompactDocumentWordsStore::Builder::Builder(uint32_t docId_) + : _docId(docId_), + _words() +{ } + +CompactDocumentWordsStore::Builder::~Builder() { } + CompactDocumentWordsStore::Builder & CompactDocumentWordsStore::Builder::insert(datastore::EntryRef wordRef) { @@ -122,8 +130,9 @@ CompactDocumentWordsStore::Store::get(datastore::EntryRef ref) const CompactDocumentWordsStore::CompactDocumentWordsStore() : _docs(), _wordsStore() -{ -} +{ } + +CompactDocumentWordsStore::~CompactDocumentWordsStore() { } void CompactDocumentWordsStore::insert(const Builder &builder) diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h b/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h index 566dd918edb..f3881c5fb23 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h @@ -18,7 +18,6 @@ namespace memoryindex { class CompactDocumentWordsStore { public: - /** * Builder used to collect all wordRefs for a field. */ @@ -33,7 +32,8 @@ public: WordRefVector _words; public: - Builder(uint32_t docId_) : _docId(docId_), _words() {} + Builder(uint32_t docId_); + ~Builder(); Builder &insert(datastore::EntryRef wordRef); uint32_t docId() const { return _docId; } const WordRefVector &words() const { return _words; } @@ -91,6 +91,7 @@ private: public: CompactDocumentWordsStore(); + ~CompactDocumentWordsStore(); void insert(const Builder &builder); void remove(uint32_t docId); Iterator get(uint32_t docId) const; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp index 3f15f3927b5..dcd4da4a8cf 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> +#include "memoryfieldindex.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> -#include "memoryfieldindex.h" #include <vespa/searchlib/bitcompression/posocccompression.h> @@ -16,8 +14,7 @@ #include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/searchlib/btree/btree.hpp> #include "ordereddocumentinserter.h" - -LOG_SETUP(".memoryindex.memoryfieldindex"); +#include <vespa/vespalib/util/array.hpp> namespace search { @@ -122,14 +119,7 @@ MemoryFieldIndex::compactFeatures(void) // Filter on which buffers to move features from when // performing incremental compaction. - datastore::EntryRef newFeatures = - _featureStore.moveFeatures(packedIndex, oldFeatures); - -#if 0 - LOG(info, - "Moved features from 0x%x to 0x%x\n", - oldFeatures.ref(), newFeatures.ref()); -#endif + datastore::EntryRef newFeatures = _featureStore.moveFeatures(packedIndex, oldFeatures); // Features must be written before reference is updated. std::atomic_thread_fence(std::memory_order_release); @@ -148,14 +138,7 @@ MemoryFieldIndex::compactFeatures(void) // Filter on which buffers to move features from when // performing incremental compaction. - datastore::EntryRef newFeatures = - _featureStore.moveFeatures(packedIndex, oldFeatures); - -#if 0 - LOG(info, - "Moved features from 0x%x to 0x%x\n", - oldFeatures.ref(), newFeatures.ref()); -#endif + datastore::EntryRef newFeatures = _featureStore.moveFeatures(packedIndex, oldFeatures); // Features must be written before reference is updated. std::atomic_thread_fence(std::memory_order_release); diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp index 90a0957ccab..1a845686474 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp @@ -1,33 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.memoryindex"); - #include "memoryindex.h" -#include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/searchlib/index/docidandfeatures.h> -#include <vespa/searchlib/index/indexbuilder.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/searchlib/memoryindex/featurestore.h> #include <vespa/searchlib/memoryindex/postingiterator.h> #include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h> -#include <vespa/searchlib/queryeval/split_float.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/emptysearch.h> -#include <vespa/searchlib/queryeval/searchiterator.h> -#include <vespa/searchlib/queryeval/searchable.h> -#include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> -#include <vespa/searchlib/queryeval/termasstring.h> -#include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> - -#include <algorithm> - -#include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> -#include <vespa/searchlib/btree/btreeroot.hpp> +#include <vespa/vespalib/stllike/hash_set.hpp> +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.memoryindex.memoryindex"); using document::ArrayFieldValue; using document::WeightedSetFieldValue; diff --git a/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp b/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp index de40e2975b2..0ec12812e46 100644 --- a/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp +++ b/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp @@ -1,18 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - #include "document_features_store.h" #include "predicate_index.h" #include "predicate_range_expander.h" #include "predicate_tree_annotator.h" #include <vespa/searchlib/btree/btreenode.h> #include <vespa/vespalib/data/databuffer.h> - +#include <vespa/vespalib/stllike/hash_map.hpp> #include <unordered_map> #include <vector> -#include <vespa/log/log.h> -LOG_SETUP(".document_features_store"); using search::btree::BTreeNoLeafData; using search::datastore::EntryRef; diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_index.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_index.cpp index 5da30aa8dd5..778c1d8b33c 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_index.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_index.cpp @@ -1,15 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - #include "predicate_index.h" - -#include "predicate_tree_annotator.h" -#include <vespa/searchlib/datastore/entryref.h> #include "predicate_hash.h" -#include <algorithm> -#include <vespa/log/log.h> -LOG_SETUP(".predicate_index"); +#include <vespa/vespalib/stllike/hash_set.hpp> + using search::datastore::EntryRef; using vespalib::DataBuffer; diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h index 2918c96a0c5..f422edc32fc 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h @@ -42,12 +42,9 @@ public: * Handler must implement handleRange(string) and handleEdge(string, uint64_t). */ template <typename Handler> -void PredicateRangeTermExpander::expand( - const vespalib::string &key, int64_t signed_value, Handler &handler) { +void PredicateRangeTermExpander::expand(const vespalib::string &key, int64_t signed_value, Handler &handler) { if (signed_value < _lower_bound || signed_value > _upper_bound) { - LOG(warning, "Search outside bounds should have been rejected by " - "ValidatePredicateSearcher."); - return; + throw vespalib::IllegalArgumentException("Search outside bounds should have been rejected by ValidatePredicateSearcher."); } char buffer[21 * 2 + 3 + key.size()]; // 2 numbers + punctuation + key int size; diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.h b/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.h index d2146aa66a5..f397ba0c30e 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.h @@ -6,6 +6,7 @@ #include <vespa/vespalib/data/slime/memory.h> #include <vespa/vespalib/stllike/string.h> #include "predicate_interval.h" +#include <climits> namespace vespalib { namespace slime { class Inspector; } diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.cpp b/searchlib/src/vespa/searchlib/predicate/simple_index.cpp index fa6fbcd1d90..78b746376cd 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.cpp +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "simple_index.hpp" +#include "simple_index.hpp" +#include <vespa/vespalib/util/array.hpp> #include <vespa/log/log.h> LOG_SETUP(".searchlib.simple_index"); diff --git a/searchlib/src/vespa/searchlib/queryeval/posting_info.h b/searchlib/src/vespa/searchlib/queryeval/posting_info.h index 087e0fb62d7..71f5214f382 100644 --- a/searchlib/src/vespa/searchlib/queryeval/posting_info.h +++ b/searchlib/src/vespa/searchlib/queryeval/posting_info.h @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once - namespace search { namespace queryeval { diff --git a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp index add0a616631..2cf0046cbb5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp @@ -1,20 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - #include "predicate_blueprint.h" #include <vespa/searchlib/predicate/predicate_bounds_posting_list.h> #include <vespa/searchlib/predicate/predicate_interval_posting_list.h> #include <vespa/searchlib/predicate/predicate_zero_constraint_posting_list.h> #include <vespa/searchlib/predicate/predicate_zstar_compressed_posting_list.h> -#include <vespa/searchlib/attribute/predicate_attribute.h> -#include <vespa/searchlib/query/tree/predicate_query_term.h> -#include <vespa/searchlib/query/tree/termnodes.h> +#include <vespa/searchlib/predicate/predicate_range_term_expander.h> #include <vespa/searchlib/predicate/predicate_hash.h> +#include <vespa/searchlib/query/tree/termnodes.h> +#include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/log/log.h> LOG_SETUP(".predicate_blueprint"); -#include <vespa/searchlib/predicate/predicate_range_term_expander.h> using search::query::PredicateQuery; using search::query::PredicateQueryTerm; diff --git a/searchlib/src/vespa/searchlib/queryeval/predicate_search.cpp b/searchlib/src/vespa/searchlib/queryeval/predicate_search.cpp index 65f2169325f..5bbea4ddf58 100644 --- a/searchlib/src/vespa/searchlib/queryeval/predicate_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/predicate_search.cpp @@ -1,13 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP(".predicate_search"); -#include <vespa/fastos/fastos.h> - #include "predicate_search.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -#include <algorithm> using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataArray; diff --git a/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp b/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp index 58d61d83d08..5791577650d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "ranksearch.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp index 34388a11af2..8b49f9ea7f2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sourceblendersearch.h" #include <vespa/vespalib/objects/visit.hpp> +#include <vespa/vespalib/util/array.hpp> namespace search { namespace queryeval { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 20c3169f9c7..788de7fe487 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor_attribute.h" #include "dense_tensor_attribute_saver.h" #include "tensor_attribute.hpp" diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index b88a59948f1..d023559bfa2 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -1,16 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor_store.h" #include <vespa/vespalib/tensor/tensor.h> #include <vespa/vespalib/tensor/dense/dense_tensor_view.h> #include <vespa/vespalib/tensor/dense/mutable_dense_tensor_view.h> #include <vespa/vespalib/tensor/dense/dense_tensor.h> #include <vespa/vespalib/tensor/serialization/typed_binary_format.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/macro.h> -#include <vespa/document/util/serializable.h> #include <vespa/searchlib/datastore/datastore.hpp> using search::datastore::Handle; diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp index bb0f942e9a1..7d9fea13f3e 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "generic_tensor_attribute.h" -#include <vespa/vespalib/tensor/tensor.h> #include "generic_tensor_attribute_saver.h" #include "tensor_attribute.hpp" +#include <vespa/vespalib/tensor/tensor.h> +#include <vespa/searchlib/common/rcuvector.cpp> using vespalib::eval::ValueType; using vespalib::tensor::Tensor; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 436c60ffdab..1a0a0026e3c 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor_attribute.h" #include <vespa/vespalib/tensor/default_tensor.h> -#include <vespa/vespalib/tensor/tensor.h> +#include <vespa/searchlib/common/rcuvector.hpp> using vespalib::eval::ValueType; using vespalib::tensor::Tensor; diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index f69654cf3d4..76d1e942f5c 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/util/fileutil.h> -#include <vespa/vespalib/util/stringfmt.h> + +#include "fileutil.hpp" +#include "filesizecalculator.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/guard.h> -#include <vespa/vespalib/data/fileheader.h> -#include "filesizecalculator.h" -#include <stdexcept> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.fileutil"); @@ -16,9 +13,7 @@ using vespalib::GenericHeader; using vespalib::FileDescriptor; using vespalib::getLastErrorString; -namespace search -{ - +namespace search { FileUtil::LoadedMmap::LoadedMmap(const vespalib::string &fileName) : LoadedBuffer(NULL, 0), @@ -124,33 +119,55 @@ SequentialFileArray::SequentialFileArray(const vespalib::string & fname) : _backingFile(), _name(fname) { - _backingFile.EnableDirectIO(); + _backingFile->EnableDirectIO(); +} + +SequentialFileArray::~SequentialFileArray() +{ + close(); } void SequentialFileArray::rewind() { - assert(_backingFile.SetPosition(0)); + assert(_backingFile->SetPosition(0)); } void SequentialFileArray::close() { - _backingFile.Close(); + _backingFile->Close(); } void SequentialFileArray::erase() { close(); - FastOS_File::Delete(_backingFile.GetFileName()); + FastOS_File::Delete(_backingFile->GetFileName()); } void SequentialFileArray::openReadOnly() { - _backingFile.ReadOpen(_name.c_str()); + _backingFile->ReadOpen(_name.c_str()); } void SequentialFileArray::openWriteOnly() { - _backingFile.OpenWriteOnlyTruncate(_name.c_str()); + _backingFile->OpenWriteOnlyTruncate(_name.c_str()); } +ssize_t +FileReaderBase::read(void *buf, size_t sz) { + ssize_t numRead = _file.Read(buf, sz); + if (numRead != ssize_t(sz)) { + handleError(numRead, sz); + } + return numRead; +} + +ssize_t +FileWriterBase::write(const void *buf, size_t sz) { + ssize_t numWritten = _file.Write2(buf, sz); + if (numWritten != ssize_t(sz)) { + handleError(numWritten, sz); + } + return numWritten; +} } diff --git a/searchlib/src/vespa/searchlib/util/fileutil.h b/searchlib/src/vespa/searchlib/util/fileutil.h index b61c4e9d0ac..b83037be96e 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.h +++ b/searchlib/src/vespa/searchlib/util/fileutil.h @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastlib/io/bufferedfile.h> #include <vector> #include <memory> #include <vespa/vespalib/data/fileheader.h> @@ -9,6 +8,7 @@ #include <vespa/vespalib/stllike/string.h> using vespalib::GenericHeader; +class Fast_BufferedFile; namespace search { @@ -77,13 +77,7 @@ class FileReaderBase { public: FileReaderBase(FastOS_FileInterface & file) : _file(file) { } - ssize_t read(void *buf, size_t sz) { - ssize_t numRead = _file.Read(buf, sz); - if (numRead != ssize_t(sz)) { - handleError(numRead, sz); - } - return numRead; - } + ssize_t read(void *buf, size_t sz); private: void handleError(ssize_t numRead, size_t wanted); FastOS_FileInterface & _file; @@ -93,13 +87,7 @@ class FileWriterBase { public: FileWriterBase(FastOS_FileInterface & file) : _file(file) { } - ssize_t write(const void *buf, size_t sz) { - ssize_t numWritten = _file.Write2(buf, sz); - if (numWritten != ssize_t(sz)) { - handleError(numWritten, sz); - } - return numWritten; - } + ssize_t write(const void *buf, size_t sz); protected: void handleError(ssize_t numWritten, size_t wanted); private: @@ -122,7 +110,7 @@ class SequentialFileArray { public: SequentialFileArray(const vespalib::string & fname); - virtual ~SequentialFileArray() { close(); } + virtual ~SequentialFileArray(); const vespalib::string & getName() const { return _name; } void rewind(); void close(); @@ -130,7 +118,7 @@ public: protected: void openReadOnly(); void openWriteOnly(); - mutable Fast_BufferedFile _backingFile; + std::unique_ptr<Fast_BufferedFile> _backingFile; vespalib::string _name; }; @@ -139,9 +127,10 @@ class SequentialFileArrayRead : public SequentialFileArray { public: SequentialFileArrayRead(const vespalib::string & fname); + ~SequentialFileArrayRead(); T getNext() const { return _fileReader.readHostOrder(); } - bool hasNext() const { return _backingFile.BytesLeft() >= sizeof(T); } - size_t size() const { return _backingFile.GetSize()/sizeof(T); } + bool hasNext() const; + size_t size() const; private: mutable FileReader<T> _fileReader; }; @@ -159,23 +148,6 @@ private: FileWriterBase _fileWriter; }; -template <typename T> -SequentialFileArrayRead<T>::SequentialFileArrayRead(const vespalib::string & fname) : - SequentialFileArray(fname), - _fileReader(_backingFile) -{ - openReadOnly(); -} - -template <typename T> -SequentialFileArrayWrite<T>::SequentialFileArrayWrite(const vespalib::string & fname) : - SequentialFileArray(fname), - _count(0), - _fileWriter(_backingFile) -{ - openWriteOnly(); -} - template <typename T, typename S> class MergeSorter { @@ -195,92 +167,6 @@ private: vespalib::string _name; }; -template <typename T, typename S> -MergeSorter<T, S>::MergeSorter(const vespalib::string & name, size_t chunkSize) : - _chunk(), - _chunkCount(0), - _name(name + ".sorted") -{ - _chunk.reserve(chunkSize); -} - -template <typename T, typename S> -void MergeSorter<T, S>::push_back(const T & v) -{ - if (_chunk.size() < _chunk.capacity()) { - _chunk.push_back(v); - if (_chunk.size() == _chunk.capacity()) { - sortChunk(); - } - } -} - -template <typename T, typename S> -vespalib::string MergeSorter<T, S>::genName(size_t n) -{ - char tmp[32]; - sprintf(tmp, ".%zd", n); - vespalib::string fname(_name); - fname += tmp; - return fname; -} - -template <typename T, typename S> -void MergeSorter<T, S>::merge() -{ - S sorter; - std::vector< SequentialFileArrayRead<T> *> fileParts; - size_t count(0); - for(size_t i(0); i < _chunkCount; i++) { - std::unique_ptr< SequentialFileArrayRead<T> > part(new SequentialFileArrayRead<T>(genName(i))); - size_t sz = part->size(); - if (sz > 0) { - fileParts.push_back(part.release()); - } else { - part->erase(); - } - count += sz; - } - - std::vector<T> cachedValue; - for(size_t i(0), m(fileParts.size()); i < m; i++) { - cachedValue.push_back(fileParts[i]->getNext()); - } - SequentialFileArrayWrite<T> merged(_name); - for(size_t j(0); j < count; j++) { - size_t firstIndex(0); - for(size_t i(1), m(cachedValue.size()); i < m; i++) { - if (sorter.cmp(cachedValue[i], cachedValue[firstIndex])) { - firstIndex = i; - } - } - merged.push_back(cachedValue[firstIndex]); - if ( ! fileParts[firstIndex]->hasNext() ) { - fileParts[firstIndex]->erase(); - delete fileParts[firstIndex]; - fileParts.erase(fileParts.begin()+firstIndex); - cachedValue.erase(cachedValue.begin()+firstIndex); - } else { - cachedValue[firstIndex] = fileParts[firstIndex]->getNext(); - } - } -} - -template <typename T, typename S> -void MergeSorter<T, S>::sortChunk() -{ - S sorter; - sorter.sort(&_chunk[0], _chunk.size()); - FastOS_File chunkFile(genName(_chunkCount).c_str()); - chunkFile.EnableDirectIO(); - if (chunkFile.OpenWriteOnlyTruncate()) { - chunkFile.CheckedWrite(&_chunk[0], _chunk.size()*sizeof(_chunk[0])); - } - chunkFile.Close(); - _chunkCount++; - _chunk.clear(); -} - template <typename T> class SequentialReadModifyWriteInterface { @@ -301,14 +187,15 @@ class SequentialReadModifyWriteVector : public SequentialReadModifyWriteInterfac private: typedef vespalib::Array<T> Vector; public: - SequentialReadModifyWriteVector() : Vector(), _rp(0), _wp(0) { } - SequentialReadModifyWriteVector(size_t sz) : Vector(sz), _rp(0), _wp(0) { } - virtual const T & read() { return (*this)[_rp]; } - virtual void write(const T & v) { (*this)[_wp++] = v; } - virtual bool next() { _rp++; return _rp < Vector::size(); } - virtual bool empty() const { return Vector::empty(); } - virtual size_t size() const { return Vector::size(); } - virtual void rewind() { _rp = 0; _wp = 0; } + SequentialReadModifyWriteVector(); + SequentialReadModifyWriteVector(size_t sz); + ~SequentialReadModifyWriteVector(); + const T & read() override { return (*this)[_rp]; } + void write(const T & v) override { (*this)[_wp++] = v; } + bool next() override { _rp++; return _rp < Vector::size(); } + bool empty() const override { return Vector::empty(); } + size_t size() const override { return Vector::size(); } + void rewind() override { _rp = 0; _wp = 0; } private: size_t _rp; size_t _wp; @@ -318,12 +205,8 @@ template <typename T, typename R, typename W> class SequentialReaderWriter : public SequentialReadModifyWriteInterface<T> { public: - SequentialReaderWriter(R & reader, W & writer) : - _reader(reader), - _writer(writer) - { - next(); - } + SequentialReaderWriter(R & reader, W & writer); + ~SequentialReaderWriter(); virtual const T & read() { return _lastRead; } virtual void write(const T & v) { _writer.push_back(v); } virtual bool next() { diff --git a/searchlib/src/vespa/searchlib/util/fileutil.hpp b/searchlib/src/vespa/searchlib/util/fileutil.hpp new file mode 100644 index 00000000000..eb91c9e9491 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/fileutil.hpp @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "fileutil.h" +#include <vespa/fastlib/io/bufferedfile.h> + +namespace search { + +template <typename T> +SequentialFileArrayRead<T>::SequentialFileArrayRead(const vespalib::string & fname) : + SequentialFileArray(fname), + _fileReader(std::make_unique<Fast_BufferedFile>(_backingFile)) +{ + openReadOnly(); +} + +template <typename T> +bool +SequentialFileArrayRead<T>::hasNext() const { + return _backingFile->BytesLeft() >= sizeof(T); + +} + +template <typename T> +size_t SequentialFileArrayRead<T>::size() const +{ + return _backingFile->GetSize()/sizeof(T); +} + +template <typename T> +SequentialFileArrayWrite<T>::SequentialFileArrayWrite(const vespalib::string & fname) : + SequentialFileArray(fname), + _count(0), + _fileWriter(_backingFile) +{ + openWriteOnly(); +} + +template <typename T, typename S> +MergeSorter<T, S>::MergeSorter(const vespalib::string & name, size_t chunkSize) : + _chunk(), + _chunkCount(0), + _name(name + ".sorted") +{ + _chunk.reserve(chunkSize); +} + +template <typename T, typename S> +void MergeSorter<T, S>::push_back(const T & v) +{ + if (_chunk.size() < _chunk.capacity()) { + _chunk.push_back(v); + if (_chunk.size() == _chunk.capacity()) { + sortChunk(); + } + } +} + +template <typename T, typename S> +vespalib::string MergeSorter<T, S>::genName(size_t n) +{ + char tmp[32]; + sprintf(tmp, ".%zd", n); + vespalib::string fname(_name); + fname += tmp; + return fname; +} + +template <typename T, typename S> +void MergeSorter<T, S>::merge() +{ + S sorter; + std::vector< SequentialFileArrayRead<T> *> fileParts; + size_t count(0); + for(size_t i(0); i < _chunkCount; i++) { + std::unique_ptr< SequentialFileArrayRead<T> > part(new SequentialFileArrayRead<T>(genName(i))); + size_t sz = part->size(); + if (sz > 0) { + fileParts.push_back(part.release()); + } else { + part->erase(); + } + count += sz; + } + + std::vector<T> cachedValue; + for(size_t i(0), m(fileParts.size()); i < m; i++) { + cachedValue.push_back(fileParts[i]->getNext()); + } + SequentialFileArrayWrite<T> merged(_name); + for(size_t j(0); j < count; j++) { + size_t firstIndex(0); + for(size_t i(1), m(cachedValue.size()); i < m; i++) { + if (sorter.cmp(cachedValue[i], cachedValue[firstIndex])) { + firstIndex = i; + } + } + merged.push_back(cachedValue[firstIndex]); + if ( ! fileParts[firstIndex]->hasNext() ) { + fileParts[firstIndex]->erase(); + delete fileParts[firstIndex]; + fileParts.erase(fileParts.begin()+firstIndex); + cachedValue.erase(cachedValue.begin()+firstIndex); + } else { + cachedValue[firstIndex] = fileParts[firstIndex]->getNext(); + } + } +} + +template <typename T, typename S> +void MergeSorter<T, S>::sortChunk() +{ + S sorter; + sorter.sort(&_chunk[0], _chunk.size()); + FastOS_File chunkFile(genName(_chunkCount).c_str()); + chunkFile.EnableDirectIO(); + if (chunkFile.OpenWriteOnlyTruncate()) { + chunkFile.CheckedWrite(&_chunk[0], _chunk.size()*sizeof(_chunk[0])); + } + chunkFile.Close(); + _chunkCount++; + _chunk.clear(); +} + +template <typename T> +SequentialReadModifyWriteVector<T>::SequentialReadModifyWriteVector() + : Vector(), + _rp(0), + _wp(0) +{ } + +template <typename T> +SequentialReadModifyWriteVector<T>::SequentialReadModifyWriteVector(size_t sz) + : Vector(sz), + _rp(0), + _wp(0) +{ } + +template <typename T> +SequentialReadModifyWriteVector<T>::~SequentialReadModifyWriteVector() { } + +template <typename T, typename R, typename W> +SequentialReaderWriter<T, R, W>::SequentialReaderWriter(R & reader, W & writer) + : _reader(reader), + _writer(writer) +{ + next(); +} + +template <typename T, typename R, typename W> +SequentialReaderWriter<T, R, W>::~SequentialReaderWriter() { } + +} diff --git a/searchlib/src/vespa/searchlib/util/stringenum.cpp b/searchlib/src/vespa/searchlib/util/stringenum.cpp index c89312e4276..cd2684c359c 100644 --- a/searchlib/src/vespa/searchlib/util/stringenum.cpp +++ b/searchlib/src/vespa/searchlib/util/stringenum.cpp @@ -2,12 +2,11 @@ // Copyright (C) 2001-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS +#include "stringenum.h" +#include <vespa/fastlib/io/bufferedfile.h> -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP(".seachlib.util.stringenum"); -#include <vespa/searchlib/util/stringenum.h> -#include <vespa/fastlib/io/bufferedfile.h> namespace search { namespace util { @@ -34,9 +33,7 @@ StripString(char *str) return first; } -StringEnum::~StringEnum() -{ -} +StringEnum::~StringEnum() { } void StringEnum::CreateReverseMapping() const diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp index 1316a370a7c..782d06468f9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/searchsummary/docsummary/getdocsumargs.h> -#include <vespa/searchsummary/docsummary/resultconfig.h> +#include "getdocsumargs.h" +#include "resultconfig.h" namespace search { namespace docsummary { @@ -20,13 +17,10 @@ GetDocsumArgs::GetDocsumArgs() _flags(0u), _propertiesMap(), _isLocationSet(false) -{ -} +{ } -GetDocsumArgs::~GetDocsumArgs() -{ -} +GetDocsumArgs::~GetDocsumArgs() { } void GetDocsumArgs::setTimeout(const fastos::TimeStamp & timeout) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index 19d8baccf45..e6420043929 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -2,15 +2,13 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/fnet/frt/frt.h> -#include <vespa/searchsummary/docsummary/resultclass.h> -#include <vespa/searchsummary/docsummary/resultconfig.h> -#include <zlib.h> +#include "resultclass.h" +#include "resultconfig.h" + +#include <vespa/vespalib/stllike/hash_map.hpp> -LOG_SETUP(".searchlib.docsummary.resultclass"); +#include <zlib.h> namespace search { namespace docsummary { @@ -23,13 +21,10 @@ ResultClass::ResultClass(const char *name, uint32_t id, util::StringEnum & field _fieldEnum(fieldEnum), _enumMap(), _dynInfo(NULL) -{ -} +{ } -ResultClass::~ResultClass() -{ -} +ResultClass::~ResultClass() { } bool diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index de635d14854..a08b0d11d5e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/searchsummary/docsummary/resultconfig.h> -#include <vespa/searchsummary/docsummary/urlresult.h> +#include "resultconfig.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.resultconfig"); namespace search { diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.h b/staging_vespalib/src/vespa/vespalib/data/fileheader.h index ca9f33714db..4ad6172bd2b 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.h +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.h @@ -3,9 +3,10 @@ #include <map> #include "databuffer.h" -#include <vespa/fastos/file.h> #include <vespa/vespalib/util/exception.h> +class FastOS_FileInterface; + namespace vespalib { class asciistream; diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp index f142fedade5..48f40ccff3a 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "identifiable.hpp" #include <cassert> #include <vespa/vespalib/util/stringfmt.h> @@ -11,7 +11,7 @@ #include "objectpredicate.h" #include "objectoperation.h" #include <vespa/vespalib/util/classname.h> -#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/hash_set.hpp> namespace vespalib { diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.h b/staging_vespalib/src/vespa/vespalib/stllike/cache.h index 2304d6389b2..27d6740e7a6 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/cache.h +++ b/staging_vespalib/src/vespa/vespalib/stllike/cache.h @@ -61,21 +61,15 @@ public: * @maxBytes is the maximum limit of bytes the store can hold, before eviction starts. */ cache(BackingStore & b, size_t maxBytes); - + ~cache(); /** * Can be used for controlling max number of elements. */ - cache & maxElements(size_t elems) { - Lru::maxElements(elems); - return *this; - } + cache & maxElements(size_t elems); /** * Can be used for reserving space for elements. */ - cache & reserveElements(size_t elems) { - Lru::reserve(elems); - return *this; - } + cache & reserveElements(size_t elems); size_t capacity() const { return Lru::capacity(); } size_t capacityBytes() const { return _maxBytes; } @@ -91,10 +85,7 @@ public: /** * This simply erases the object from the cache. */ - void invalidate(const K & key) { - vespalib::LockGuard guard(_hashLock); - invalidate(guard, key); - } + void invalidate(const K & key); /** * Return the object with the given key. If it does not exist, the backing store will be consulted. @@ -114,10 +105,7 @@ public: * Tell if an object with given key exists in the cache. * Does not alter the LRU list. */ - bool hasKey(const K & key) const { - vespalib::LockGuard guard(_hashLock); - return hasKey(guard, key); - } + bool hasKey(const K & key) const; size_t getHit() const { return _hit; } size_t getMiss() const { return _miss; } @@ -133,7 +121,7 @@ protected: vespalib::LockGuard getGuard(); void invalidate(const vespalib::LockGuard & guard, const K & key); bool hasKey(const vespalib::LockGuard & guard, const K & key) const; - bool hasLock() const { return TryLock(_hashLock).hasLock(); } + bool hasLock() const; private: /** * Called when an object is inserted, to see if the LRU should be removed. @@ -167,119 +155,4 @@ private: vespalib::Lock _addLocks[113]; }; -template< typename P > -cache<P>::cache(BackingStore & b, size_t maxBytes) : - Lru(Lru::UNLIMITED), - _maxBytes(maxBytes), - _sizeBytes(0), - _hit(0), - _miss(0), - _noneExisting(0), - _race(0), - _insert(0), - _write(0), - _erase(0), - _invalidate(0), - _lookup(0), - _store(b) -{ } - -template< typename P > -bool -cache<P>::removeOldest(const value_type & v) { - bool remove(Lru::removeOldest(v) || (sizeBytes() >= capacityBytes())); - if (remove) { - _sizeBytes -= calcSize(v.first, v.second._value); - } - return remove; } - -template< typename P > -vespalib::LockGuard -cache<P>::getGuard() { - return vespalib::LockGuard(_hashLock); -} - -template< typename P > -typename P::Value -cache<P>::read(const K & key) -{ - { - vespalib::LockGuard guard(_hashLock); - if (Lru::hasKey(key)) { - _hit++; - return (*this)[key]; - } else { - _miss++; - } - } - - vespalib::LockGuard storeGuard(getLock(key)); - { - vespalib::LockGuard guard(_hashLock); - if (Lru::hasKey(key)) { - // Somebody else just fetched it ahead of me. - _race++; - return (*this)[key]; - } - } - V value; - if (_store.read(key, value)) { - vespalib::LockGuard guard(_hashLock); - Lru::insert(key, value); - _sizeBytes += calcSize(key, value); - _insert++; - } else { - vespalib::Atomic::postInc(&_noneExisting); - } - return value; -} - -template< typename P > -void -cache<P>::write(const K & key, const V & value) -{ - vespalib::LockGuard storeGuard(getLock(key)); - { - vespalib::LockGuard guard(_hashLock); - (*this)[key] = value; - _sizeBytes += calcSize(key, value); - _write++; - } - _store.write(key, value); -} - -template< typename P > -void -cache<P>::erase(const K & key) -{ - vespalib::LockGuard storeGuard(getLock(key)); - invalidate(key); - _store.erase(key); -} - -template< typename P > -void -cache<P>::invalidate(const vespalib::LockGuard & guard, const K & key) -{ - assert(guard.locks(_hashLock)); - (void) guard; - if (Lru::hasKey(key)) { - _sizeBytes -= calcSize(key, (*this)[key]); - _invalidate++; - Lru::erase(key); - } -} - -template< typename P > -bool -cache<P>::hasKey(const vespalib::LockGuard & guard, const K & key) const -{ - (void) guard; - assert(guard.locks(_hashLock)); - _lookup++; - return Lru::hasKey(key); -} - -} - diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp b/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp new file mode 100644 index 00000000000..1b608578e8d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "cache.h" +#include "lrucache_map.hpp" + +namespace vespalib { + +template< typename P > +cache<P> & +cache<P>::maxElements(size_t elems) { + Lru::maxElements(elems); + return *this; +} + +template< typename P > +cache<P> & +cache<P>::reserveElements(size_t elems) { + Lru::reserve(elems); + return *this; +} + +template< typename P > +void +cache<P>::invalidate(const K & key) { + vespalib::LockGuard guard(_hashLock); + invalidate(guard, key); +} + +template< typename P > +bool +cache<P>::hasKey(const K & key) const { + vespalib::LockGuard guard(_hashLock); + return hasKey(guard, key); +} + +template< typename P > +bool +cache<P>::hasLock() const { + return TryLock(_hashLock).hasLock(); +} + +template< typename P > +cache<P>::~cache() { } + +template< typename P > +cache<P>::cache(BackingStore & b, size_t maxBytes) : + Lru(Lru::UNLIMITED), + _maxBytes(maxBytes), + _sizeBytes(0), + _hit(0), + _miss(0), + _noneExisting(0), + _race(0), + _insert(0), + _write(0), + _erase(0), + _invalidate(0), + _lookup(0), + _store(b) +{ } + +template< typename P > +bool +cache<P>::removeOldest(const value_type & v) { + bool remove(Lru::removeOldest(v) || (sizeBytes() >= capacityBytes())); + if (remove) { + _sizeBytes -= calcSize(v.first, v.second._value); + } + return remove; +} + +template< typename P > +vespalib::LockGuard +cache<P>::getGuard() { + return vespalib::LockGuard(_hashLock); +} + +template< typename P > +typename P::Value +cache<P>::read(const K & key) +{ + { + vespalib::LockGuard guard(_hashLock); + if (Lru::hasKey(key)) { + _hit++; + return (*this)[key]; + } else { + _miss++; + } + } + + vespalib::LockGuard storeGuard(getLock(key)); + { + vespalib::LockGuard guard(_hashLock); + if (Lru::hasKey(key)) { + // Somebody else just fetched it ahead of me. + _race++; + return (*this)[key]; + } + } + V value; + if (_store.read(key, value)) { + vespalib::LockGuard guard(_hashLock); + Lru::insert(key, value); + _sizeBytes += calcSize(key, value); + _insert++; + } else { + vespalib::Atomic::postInc(&_noneExisting); + } + return value; +} + +template< typename P > +void +cache<P>::write(const K & key, const V & value) +{ + vespalib::LockGuard storeGuard(getLock(key)); + { + vespalib::LockGuard guard(_hashLock); + (*this)[key] = value; + _sizeBytes += calcSize(key, value); + _write++; + } + _store.write(key, value); +} + +template< typename P > +void +cache<P>::erase(const K & key) +{ + vespalib::LockGuard storeGuard(getLock(key)); + invalidate(key); + _store.erase(key); +} + +template< typename P > +void +cache<P>::invalidate(const vespalib::LockGuard & guard, const K & key) +{ + assert(guard.locks(_hashLock)); + (void) guard; + if (Lru::hasKey(key)) { + _sizeBytes -= calcSize(key, (*this)[key]); + _invalidate++; + Lru::erase(key); + } +} + +template< typename P > +bool +cache<P>::hasKey(const vespalib::LockGuard & guard, const K & key) const +{ + (void) guard; + assert(guard.locks(_hashLock)); + _lookup++; + return Lru::hasKey(key); +} + +} + diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h index b48096f184c..747dfcacc56 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h +++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/vespalib/stllike/hashtable.h> +#include <vespa/vespalib/stllike/hash_fun.h> namespace vespalib { @@ -83,6 +84,7 @@ public: * @param maxElements in cache unless you override @ref removeOldest. */ lrucache_map(size_t maxElems); + virtual ~lrucache_map(); lrucache_map & maxElements(size_t elems) { _maxElements = elems; @@ -119,13 +121,7 @@ public: * Object is inserted in cache with given key. * Object is then put at head of LRU list. */ - insert_result insert(const K & key, const V & value) { - insert_result res = insert(value_type(key, LV(value))); - if (res.second) { - onInsert(key); - } - return res; - } + insert_result insert(const K & key, const V & value); /** * Return the object with the given key. If it does not exist an empty one will be created. @@ -155,16 +151,9 @@ public: * The obvious extension is when you are storing pointers and want to cap * on the real size of the object pointed to. */ - virtual bool removeOldest(const value_type & v) { - (void) v; - return (size() > capacity()); - } - virtual void onRemove(const K & key) { - (void) key; - } - virtual void onInsert(const K & key) { - (void) key; - } + virtual bool removeOldest(const value_type & v); + virtual void onRemove(const K & key); + virtual void onInsert(const K & key); /** * Method for testing that internal consitency is good. @@ -198,19 +187,8 @@ private: { _lru._moveRecordingEnabled = true; } - uint32_t movedTo(uint32_t from) { - for (size_t i(0); i < _lru._moved.size(); i++) { - const MoveRecord & mr(_lru._moved[i]); - if (mr.first == from) { - from = mr.second; - } - } - return from; - } - ~RecordMoves() { - _lru._moveRecordingEnabled = false; - _lru._moved.clear(); - } + ~RecordMoves(); + uint32_t movedTo(uint32_t from); private: lrucache_map & _lru; }; @@ -222,212 +200,4 @@ private: MoveRecords _moved; }; -template< typename P > -lrucache_map<P>::lrucache_map(size_t maxElems) : - HashTable(0), - _maxElements(maxElems), - _head(LinkedValueBase::npos), - _tail(LinkedValueBase::npos), - _moveRecordingEnabled(false), - _moved() -{ -} - -template< typename P > -void -lrucache_map<P>::swap(lrucache_map & rhs) { - std::swap(_maxElements, rhs._maxElements); - std::swap(_head, rhs._head); - std::swap(_tail, rhs._tail); - HashTable::swap(rhs); -} - -template< typename P > -void -lrucache_map<P>::move(next_t from, next_t to) { - (void) from; - if (_moveRecordingEnabled) { - _moved.push_back(std::make_pair(from, to)); - } - value_type & moved = HashTable::getByInternalIndex(to); - if (moved.second._prev != LinkedValueBase::npos) { - HashTable::getByInternalIndex(moved.second._prev).second._next = to; - } else { - _head = to; - } - if (moved.second._next != LinkedValueBase::npos) { - HashTable::getByInternalIndex(moved.second._next).second._prev = to; - } else { - _tail = to; - } } - -template< typename P > -void -lrucache_map<P>::erase(const K & key) { - internal_iterator it = HashTable::find(key); - if (it != HashTable::end()) { - onRemove(key); - LV & v = it->second; - if (v._prev != LinkedValueBase::npos) { - HashTable::getByInternalIndex(v._prev).second._next = v._next; - } else { - _head = v._next; - } - if (v._next != LinkedValueBase::npos) { - HashTable::getByInternalIndex(v._next).second._prev = v._prev; - } else { - _tail = v._prev; - } - HashTable::erase(*this, it); - } -} - -template< typename P > -typename lrucache_map<P>::iterator -lrucache_map<P>::erase(const iterator & it) -{ - iterator next(it); - if (it != end()) { - RecordMoves moves(*this); - next++; - const K & key(HashTable::getByInternalIndex(it._current).first); - erase(key); - next = iterator(this, moves.movedTo(next._current)); - } - return next; -} - -template< typename P > -bool -lrucache_map<P>::verifyInternals() -{ - bool retval(true); - assert(_head != LinkedValueBase::npos); - assert(_tail != LinkedValueBase::npos); - assert(HashTable::getByInternalIndex(_head).second._prev == LinkedValueBase::npos); - assert(HashTable::getByInternalIndex(_tail).second._next == LinkedValueBase::npos); - { - size_t i(0); - size_t prev(LinkedValueBase::npos); - size_t c(_head); - for(size_t m(size()); (c != LinkedValueBase::npos) && (i < m); c = HashTable::getByInternalIndex(c).second._next, i++) { - assert((HashTable::getByInternalIndex(c).second._prev == prev)); - prev = c; - } - assert(i == size()); - assert(c == LinkedValueBase::npos); - } - { - size_t i(0); - size_t next(LinkedValueBase::npos); - size_t c(_tail); - for(size_t m(size()); (c != LinkedValueBase::npos) && (i < m); c = HashTable::getByInternalIndex(c).second._prev, i++) { - assert((HashTable::getByInternalIndex(c).second._next == next)); - next = c; - } - assert(i == size()); - assert(c == LinkedValueBase::npos); - } - return retval; -} - -template< typename P > -void -lrucache_map<P>::move(NodeStore && oldStore) -{ - next_t curr(_tail); - _tail = LinkedValueBase::npos; - _head = LinkedValueBase::npos; - - while (curr != LinkedValueBase::npos) { - value_type & v = oldStore[curr].getValue(); - curr = v.second._prev; - v.second._prev = LinkedValueBase::npos; - v.second._next = LinkedValueBase::npos; - insert(std::move(v)); - } -} - -template< typename P > -void -lrucache_map<P>::removeOld() { - if (_tail != LinkedValueBase::npos) { - for (value_type * last(& HashTable::getByInternalIndex(_tail)); - (_tail != _head) && removeOldest(*last); - last = & HashTable::getByInternalIndex(_tail)) - { - _tail = last->second._prev; - HashTable::getByInternalIndex(_tail).second._next = LinkedValueBase::npos; - HashTable::erase(*this, HashTable::find(last->first)); - } - } -} - -template< typename P > -void -lrucache_map<P>::ref(const internal_iterator & it) { - uint32_t me(it.getInternalIndex()); - if (me != _head) { - LV & v = it->second; - LV & oldPrev = HashTable::getByInternalIndex(v._prev).second; - oldPrev._next = v._next; - if (me != _tail) { - LV & oldNext = HashTable::getByInternalIndex(v._next).second; - oldNext._prev = v._prev; - } else { - // I am tail and I am not the only one. - _tail = v._prev; - } - LV & oldHead = HashTable::getByInternalIndex(_head).second; - oldHead._prev = me; - v._next = _head; - v._prev = LinkedValueBase::npos; - _head = me; - } -} - -template< typename P > -typename lrucache_map<P>::insert_result -lrucache_map<P>::insert(value_type && value) { - insert_result res = HashTable::insertInternal(std::forward<value_type>(value)); - uint32_t next(_head); - if ( ! res.second) { - ref(res.first); - } else { - _head = res.first.getInternalIndex(); - HashTable::getByInternalIndex(_head).second._next = next; - if (next != LinkedValueBase::npos) { - HashTable::getByInternalIndex(next).second._prev = _head; - } - if (_tail == LinkedValueBase::npos) { - _tail = _head; - } - removeOld(); - if (_head != res.first.getInternalIndex()) { - res.first.setInternalIndex(_head); - } - } - return res; -} - -template< typename P > -typename P::Value & -lrucache_map<P>::operator [] (const K & key) -{ - return insert(key, V()).first->second._value; -} - -template< typename P > -typename lrucache_map<P>::internal_iterator -lrucache_map<P>::findAndRef(const K & key) -{ - internal_iterator found = HashTable::find(key); - if (found != HashTable::end()) { - ref(found); - } - return found; -} - -} - diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp new file mode 100644 index 00000000000..3301abca3ba --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp @@ -0,0 +1,266 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "lrucache_map.h" +#include <vespa/vespalib/stllike/hashtable.hpp> + +namespace vespalib { + +template< typename P > +typename lrucache_map<P>::insert_result +lrucache_map<P>::insert(const K & key, const V & value) { + insert_result res = insert(value_type(key, LV(value))); + if (res.second) { + onInsert(key); + } + return res; +} + +template< typename P > +bool +lrucache_map<P>::removeOldest(const value_type & v) { + (void) v; + return (size() > capacity()); +} + +template< typename P > +void +lrucache_map<P>::onRemove(const K & key) { + (void) key; +} + +template< typename P > +void +lrucache_map<P>::onInsert(const K & key) { + (void) key; +} + +template< typename P > +uint32_t +lrucache_map<P>::RecordMoves::movedTo(uint32_t from) { + for (size_t i(0); i < _lru._moved.size(); i++) { + const MoveRecord & mr(_lru._moved[i]); + if (mr.first == from) { + from = mr.second; + } + } + return from; +} + +template< typename P > +lrucache_map<P>::RecordMoves::~RecordMoves() { + _lru._moveRecordingEnabled = false; + _lru._moved.clear(); +} + +template< typename P > +lrucache_map<P>::lrucache_map(size_t maxElems) : + HashTable(0), + _maxElements(maxElems), + _head(LinkedValueBase::npos), + _tail(LinkedValueBase::npos), + _moveRecordingEnabled(false), + _moved() +{ } + +template< typename P > +lrucache_map<P>::~lrucache_map() { } + +template< typename P > +void +lrucache_map<P>::swap(lrucache_map & rhs) { + std::swap(_maxElements, rhs._maxElements); + std::swap(_head, rhs._head); + std::swap(_tail, rhs._tail); + HashTable::swap(rhs); +} + +template< typename P > +void +lrucache_map<P>::move(next_t from, next_t to) { + (void) from; + if (_moveRecordingEnabled) { + _moved.push_back(std::make_pair(from, to)); + } + value_type & moved = HashTable::getByInternalIndex(to); + if (moved.second._prev != LinkedValueBase::npos) { + HashTable::getByInternalIndex(moved.second._prev).second._next = to; + } else { + _head = to; + } + if (moved.second._next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(moved.second._next).second._prev = to; + } else { + _tail = to; + } +} + +template< typename P > +void +lrucache_map<P>::erase(const K & key) { + internal_iterator it = HashTable::find(key); + if (it != HashTable::end()) { + onRemove(key); + LV & v = it->second; + if (v._prev != LinkedValueBase::npos) { + HashTable::getByInternalIndex(v._prev).second._next = v._next; + } else { + _head = v._next; + } + if (v._next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(v._next).second._prev = v._prev; + } else { + _tail = v._prev; + } + HashTable::erase(*this, it); + } +} + +template< typename P > +typename lrucache_map<P>::iterator +lrucache_map<P>::erase(const iterator & it) +{ + iterator next(it); + if (it != end()) { + RecordMoves moves(*this); + next++; + const K & key(HashTable::getByInternalIndex(it._current).first); + erase(key); + next = iterator(this, moves.movedTo(next._current)); + } + return next; +} + +template< typename P > +bool +lrucache_map<P>::verifyInternals() +{ + bool retval(true); + assert(_head != LinkedValueBase::npos); + assert(_tail != LinkedValueBase::npos); + assert(HashTable::getByInternalIndex(_head).second._prev == LinkedValueBase::npos); + assert(HashTable::getByInternalIndex(_tail).second._next == LinkedValueBase::npos); + { + size_t i(0); + size_t prev(LinkedValueBase::npos); + size_t c(_head); + for(size_t m(size()); (c != LinkedValueBase::npos) && (i < m); c = HashTable::getByInternalIndex(c).second._next, i++) { + assert((HashTable::getByInternalIndex(c).second._prev == prev)); + prev = c; + } + assert(i == size()); + assert(c == LinkedValueBase::npos); + } + { + size_t i(0); + size_t next(LinkedValueBase::npos); + size_t c(_tail); + for(size_t m(size()); (c != LinkedValueBase::npos) && (i < m); c = HashTable::getByInternalIndex(c).second._prev, i++) { + assert((HashTable::getByInternalIndex(c).second._next == next)); + next = c; + } + assert(i == size()); + assert(c == LinkedValueBase::npos); + } + return retval; +} + +template< typename P > +void +lrucache_map<P>::move(NodeStore && oldStore) +{ + next_t curr(_tail); + _tail = LinkedValueBase::npos; + _head = LinkedValueBase::npos; + + while (curr != LinkedValueBase::npos) { + value_type & v = oldStore[curr].getValue(); + curr = v.second._prev; + v.second._prev = LinkedValueBase::npos; + v.second._next = LinkedValueBase::npos; + insert(std::move(v)); + } +} + +template< typename P > +void +lrucache_map<P>::removeOld() { + if (_tail != LinkedValueBase::npos) { + for (value_type * last(& HashTable::getByInternalIndex(_tail)); + (_tail != _head) && removeOldest(*last); + last = & HashTable::getByInternalIndex(_tail)) + { + _tail = last->second._prev; + HashTable::getByInternalIndex(_tail).second._next = LinkedValueBase::npos; + HashTable::erase(*this, HashTable::find(last->first)); + } + } +} + +template< typename P > +void +lrucache_map<P>::ref(const internal_iterator & it) { + uint32_t me(it.getInternalIndex()); + if (me != _head) { + LV & v = it->second; + LV & oldPrev = HashTable::getByInternalIndex(v._prev).second; + oldPrev._next = v._next; + if (me != _tail) { + LV & oldNext = HashTable::getByInternalIndex(v._next).second; + oldNext._prev = v._prev; + } else { + // I am tail and I am not the only one. + _tail = v._prev; + } + LV & oldHead = HashTable::getByInternalIndex(_head).second; + oldHead._prev = me; + v._next = _head; + v._prev = LinkedValueBase::npos; + _head = me; + } +} + +template< typename P > +typename lrucache_map<P>::insert_result +lrucache_map<P>::insert(value_type && value) { + insert_result res = HashTable::insertInternal(std::forward<value_type>(value)); + uint32_t next(_head); + if ( ! res.second) { + ref(res.first); + } else { + _head = res.first.getInternalIndex(); + HashTable::getByInternalIndex(_head).second._next = next; + if (next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(next).second._prev = _head; + } + if (_tail == LinkedValueBase::npos) { + _tail = _head; + } + removeOld(); + if (_head != res.first.getInternalIndex()) { + res.first.setInternalIndex(_head); + } + } + return res; +} + +template< typename P > +typename P::Value & +lrucache_map<P>::operator [] (const K & key) +{ + return insert(key, V()).first->second._value; +} + +template< typename P > +typename lrucache_map<P>::internal_iterator +lrucache_map<P>::findAndRef(const K & key) +{ + internal_iterator found = HashTable::find(key); + if (found != HashTable::end()) { + ref(found); + } + return found; +} + +} + diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 7552a471256..be189a0bfa9 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -19,6 +19,7 @@ #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/stat.h> #include <vespa/storage/storageutil/distributorstatecache.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/config/config.h> #include <unordered_map> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/bucketdb/judymultimap.h b/storage/src/vespa/storage/bucketdb/judymultimap.h index 44a7c7e712c..1c829a6f1de 100644 --- a/storage/src/vespa/storage/bucketdb/judymultimap.h +++ b/storage/src/vespa/storage/bucketdb/judymultimap.h @@ -37,8 +37,8 @@ template<class Type0, class Type3 = Type2 > class JudyMultiMap : public vespalib::Printable { public: - JudyMultiMap() - : _values0(1), _values1(1), _values2(1), _values3(1), _free(4) {} + JudyMultiMap(); + ~JudyMultiMap(); class Iterator; class ConstIterator; diff --git a/storage/src/vespa/storage/bucketdb/judymultimap.hpp b/storage/src/vespa/storage/bucketdb/judymultimap.hpp index 3bc9263aa37..5ce2c302789 100644 --- a/storage/src/vespa/storage/bucketdb/judymultimap.hpp +++ b/storage/src/vespa/storage/bucketdb/judymultimap.hpp @@ -3,12 +3,21 @@ #include "judymultimap.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/array.hpp> #include <set> #include <ostream> namespace storage { template<class T0, class T1, class T2, class T3> +JudyMultiMap<T0, T1, T2, T3>::JudyMultiMap() + : _values0(1), _values1(1), _values2(1), _values3(1), _free(4) +{ } + +template<class T0, class T1, class T2, class T3> +JudyMultiMap<T0, T1, T2, T3>::~JudyMultiMap() { } + +template<class T0, class T1, class T2, class T3> bool JudyMultiMap<T0, T1, T2, T3>:: operator==(const JudyMultiMap<T0, T1, T2, T3>& map) const diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h index 3a7843405c8..6301014d26b 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.h +++ b/storage/src/vespa/storage/bucketdb/lockablemap.h @@ -212,7 +212,7 @@ private: ~LockIdSet(); void print(std::ostream& out, bool verbose, const std::string& indent) const; bool exist(const LockId & lid) const { return this->find(lid) != Hash::end(); } - size_t getMemoryUsage() const { return Hash::getMemoryConsumption(); } + size_t getMemoryUsage() const; }; class LockWaiters { diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp index c84ea68cd33..7510b5d1a87 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ b/storage/src/vespa/storage/bucketdb/lockablemap.hpp @@ -3,6 +3,8 @@ #include "lockablemap.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/hash_set.hpp> #include <thread> #include <chrono> @@ -15,6 +17,12 @@ template<typename Map> LockableMap<Map>::LockIdSet::~LockIdSet() { } template<typename Map> +size_t +LockableMap<Map>::LockIdSet::getMemoryUsage() const { + return Hash::getMemoryConsumption(); +} + +template<typename Map> LockableMap<Map>::LockWaiters::LockWaiters() : _id(0), _map() { } template<typename Map> diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp index 1021cf7f212..e5db345505c 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp @@ -10,10 +10,10 @@ #include <vespa/config-stor-filestor.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/config/config.h> -#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <iomanip> -#include <vespa/log/log.h> +#include <vespa/log/log.h> LOG_SETUP(".storage.bucketdb.initializer"); namespace storage { @@ -106,6 +106,14 @@ StorageBucketDBInitializer::Metrics::Metrics(framework::Component& component) component.registerMetric(*this); } +StorageBucketDBInitializer::GlobalState::GlobalState() + : _insertedCount(0), _infoReadCount(0), + _infoSetByLoad(0), _dirsListed(0), _dirsToList(0), + _gottenInitProgress(false), _doneListing(false), + _doneInitializing(false) +{ } +StorageBucketDBInitializer::GlobalState::~GlobalState() { } + StorageBucketDBInitializer::StorageBucketDBInitializer( const config::ConfigUri & configUri, const spi::PartitionStateList& partitions, diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h index 01445529865..a0f4345a2a6 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h @@ -100,10 +100,8 @@ class StorageBucketDBInitializer : public StorageLink, Metrics(framework::Component&); }; struct GlobalState { - vespalib::hash_map<api::StorageMessage::Id, - ReadBucketList::SP> _lists; - vespalib::hash_map<api::StorageMessage::Id, - InternalBucketJoinCommand::SP> _joins; + vespalib::hash_map<api::StorageMessage::Id, ReadBucketList::SP> _lists; + vespalib::hash_map<api::StorageMessage::Id, InternalBucketJoinCommand::SP> _joins; IdDiskMap _infoRequests; std::list<api::StorageMessage::SP> _replies; uint64_t _insertedCount; @@ -121,11 +119,8 @@ class StorageBucketDBInitializer : public StorageLink, // This lock protects the reply list. vespalib::Monitor _replyLock; - GlobalState() - : _insertedCount(0), _infoReadCount(0), - _infoSetByLoad(0), _dirsListed(0), _dirsToList(0), - _gottenInitProgress(false), _doneListing(false), - _doneInitializing(false) {} + GlobalState(); + ~GlobalState(); }; diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 95322ad663a..623312a8049 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/config/distributorconfiguration.h> -#include <vespa/log/log.h> +#include "distributorconfiguration.h" #include <vespa/document/select/parser.h> -#include <vespa/document/select/node.h> #include <vespa/document/select/traversingvisitor.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/log/log.h> LOG_SETUP(".distributorconfiguration"); namespace storage { @@ -34,9 +33,9 @@ DistributorConfiguration::DistributorConfiguration(StorageComponent& component) _enableHostInfoReporting(true), _disableBucketActivation(false), _minimumReplicaCountingMode(ReplicaCountingMode::TRUSTED) -{ - -} +{ } + +DistributorConfiguration::~DistributorConfiguration() { } namespace { diff --git a/storage/src/vespa/storage/config/distributorconfiguration.h b/storage/src/vespa/storage/config/distributorconfiguration.h index b18c09955fa..fd8ed0c272b 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.h +++ b/storage/src/vespa/storage/config/distributorconfiguration.h @@ -16,6 +16,7 @@ namespace distributor { class DistributorConfiguration { public: DistributorConfiguration(StorageComponent& component); + ~DistributorConfiguration(); struct MaintenancePriorities { diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp index 1b82cdfeaf3..6de7722496b 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> - +#include "bucketdbmetricupdater.h" #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/idealstatemetricsset.h> -#include <algorithm> - namespace storage { namespace distributor { diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h index 9f59426f983..3c4dfa6f0fc 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h @@ -94,8 +94,7 @@ public: } private: - void updateMinReplicationStats(const BucketDatabase::Entry& entry, - uint32_t trustedCopies); + void updateMinReplicationStats(const BucketDatabase::Entry& entry, uint32_t trustedCopies); void resetStats(); }; diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp index 30ff11fc85f..bc0429f806c 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp @@ -1,13 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/externaloperationhandler.h> - -#include <vespa/document/bucket/bucketidfactory.h> +#include "externaloperationhandler.h" +#include "distributor.h" #include <vespa/document/base/documentid.h> -#include <vespa/log/log.h> -#include <vespa/storage/config/config-stor-server.h> -#include <vespa/storage/storageutil/log.h> #include <vespa/storage/distributor/operations/external/putoperation.h> #include <vespa/storage/distributor/operations/external/twophaseupdateoperation.h> #include <vespa/storage/distributor/operations/external/updateoperation.h> @@ -20,18 +15,13 @@ #include <vespa/storage/distributor/operations/external/visitoroperation.h> #include <vespa/document/util/stringutil.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/batch.h> -#include <vespa/storage/storageserver/storagemetricsset.h> -#include <vespa/storage/distributor/distributormetricsset.h> -#include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/stat.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/storage/distributor/distributor.h> -#include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".distributor.manager"); namespace storage { diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index eba37435462..e4e40f32be5 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -1,25 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/idealstatemanager.h> -#include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> -#include <vespa/log/log.h> -#include <vespa/storageapi/message/state.h> -#include <vespa/vespalib/util/stringfmt.h> +#include "idealstatemanager.h" +#include "statecheckers.h" +#include "distributor.h" +#include "idealstatemetricsset.h" #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/storage/storageserver/storagemetricsset.h> -#include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/distributor/statecheckers.h> -#include <vespa/storageapi/message/bucket.h> -#include <vespa/storage/distributor/distributormetricsset.h> -#include <vespa/storageapi/message/stat.h> -#include <vespa/storage/distributor/distributor.h> -#include <vespa/storageframework/storageframework.h> -#include <vespa/storage/distributor/idealstatemetricsset.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".distributor.operation.queue"); using storage::lib::Node; diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 9997cc029d9..af0e016a77c 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -9,6 +9,7 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storageapi/message/batch.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.twophaseupdate"); diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.cpp b/storage/src/vespa/storage/persistence/bucketprocessor.cpp index d185d94f11e..c15b4ac58e1 100644 --- a/storage/src/vespa/storage/persistence/bucketprocessor.cpp +++ b/storage/src/vespa/storage/persistence/bucketprocessor.cpp @@ -4,8 +4,6 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <stdexcept> - namespace storage { namespace { @@ -24,7 +22,7 @@ public: : _spi(spi), _iteratorId(iteratorId), _context(context) - {} + { } ~IteratorGuard() { assert(_iteratorId != 0); diff --git a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp b/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp index aaadb8a7f7d..6040a4b58ab 100644 --- a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp +++ b/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/persistence/diskmoveoperationhandler.h> -#include <vespa/log/log.h> +#include "diskmoveoperationhandler.h" + +#include <vespa/log/log.h> LOG_SETUP(".persistence.diskmoveoperationhandler"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 36d0c477b39..fb9b18c7341 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/persistence/filestorage/filestorhandlerimpl.h> -#include <vespa/log/log.h> -#include <vespa/storageapi/messageapi/storagemessage.h> -#include <vespa/storageapi/message/bucket.h> +#include "filestorhandlerimpl.h" +#include "filestormetrics.h" #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/storageapi/message/persistence.h> @@ -16,12 +13,13 @@ #include <vespa/storage/common/statusmessages.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storage/common/messagebucketid.h> -#include <vespa/storage/persistence/filestorage/filestormetrics.h> #include <vespa/storage/persistence/messages.h> #include <vespa/vespalib/util/random.h> #include <vespa/storageapi/message/stat.h> #include <vespa/storageapi/message/batch.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".persistence.filestor.handler.impl"); namespace storage { @@ -57,13 +55,10 @@ FileStorHandlerImpl::FileStorHandlerImpl( _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); } -FileStorHandlerImpl::~FileStorHandlerImpl() -{ -} +FileStorHandlerImpl::~FileStorHandlerImpl() { } void -FileStorHandlerImpl::addMergeStatus(const document::BucketId& bucket, - MergeStatus::SP status) +FileStorHandlerImpl::addMergeStatus(const document::BucketId& bucket, MergeStatus::SP status) { vespalib::LockGuard mlock(_mergeStatesLock); if (_mergeStates.find(bucket) != _mergeStates.end()) {; @@ -1197,8 +1192,7 @@ FileStorHandlerImpl::failOperations( } void -FileStorHandlerImpl::sendCommand( - const std::shared_ptr<api::StorageCommand>& msg) +FileStorHandlerImpl::sendCommand(const std::shared_ptr<api::StorageCommand>& msg) { _messageSender.sendCommand(msg); } @@ -1209,18 +1203,35 @@ FileStorHandlerImpl::sendReply(const std::shared_ptr<api::StorageReply>& msg) _messageSender.sendReply(msg); } +FileStorHandlerImpl::MessageEntry::MessageEntry(const std::shared_ptr<api::StorageMessage>& cmd, + const document::BucketId& bId) + : _command(cmd), + _timer(), + _bucketId(bId), + _priority(cmd->getPriority()) +{ } + +FileStorHandlerImpl::MessageEntry::MessageEntry(const MessageEntry& entry) + : _command(entry._command), + _timer(entry._timer), + _bucketId(entry._bucketId), + _priority(entry._priority) +{ } + +FileStorHandlerImpl::MessageEntry::~MessageEntry() { } + FileStorHandlerImpl::Disk::Disk() : lock(), queue(), lockedBuckets(100), metrics(0), state(FileStorHandler::AVAILABLE) -{ -} +{ } + +FileStorHandlerImpl::Disk::~Disk() { } bool -FileStorHandlerImpl::Disk::isLocked( - const document::BucketId& bucket) const noexcept +FileStorHandlerImpl::Disk::isLocked(const document::BucketId& bucket) const noexcept { return (lockedBuckets.find(bucket) != lockedBuckets.end()); } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 5abf033be2b..27bb537ab18 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -50,20 +50,10 @@ public: document::BucketId _bucketId; uint8_t _priority; - MessageEntry(const std::shared_ptr<api::StorageMessage>& cmd, - const document::BucketId& bId) - : _command(cmd), - _bucketId(bId), - _priority(cmd->getPriority()) - {} - - MessageEntry(const MessageEntry& entry) - : _command(entry._command), - _timer(entry._timer), - _bucketId(entry._bucketId), - _priority(entry._priority) - { - } + MessageEntry(const std::shared_ptr<api::StorageMessage>& cmd, const document::BucketId& bId); + MessageEntry(const MessageEntry &); + MessageEntry & operator = (const MessageEntry &) = delete; + ~MessageEntry(); bool operator<(const MessageEntry& entry) const { return (_priority < entry._priority); @@ -101,23 +91,16 @@ public: LockEntry() : timestamp(0), priority(0), statusString() - { - } + { } - LockEntry(uint8_t priority_, - vespalib::stringref status) + LockEntry(uint8_t priority_, vespalib::stringref status) : timestamp(time(NULL)), priority(priority_), statusString(status) - { - } + { } }; - typedef vespalib::hash_map< - document::BucketId, - LockEntry, - document::BucketId::hash - > LockedBuckets; + typedef vespalib::hash_map<document::BucketId, LockEntry, document::BucketId::hash> LockedBuckets; LockedBuckets lockedBuckets; FileStorDiskMetrics* metrics; @@ -137,6 +120,7 @@ public: } Disk(); + ~Disk(); bool isLocked(const document::BucketId&) const noexcept; uint32_t getQueueSize() const noexcept; @@ -146,15 +130,11 @@ public: class BucketLock : public FileStorHandler::BucketLockInterface { public: - BucketLock(Disk& disk, - const document::BucketId& id, - uint8_t priority, + BucketLock(Disk& disk, const document::BucketId& id, uint8_t priority, const vespalib::stringref & statusString); ~BucketLock(); - const document::BucketId& getBucketId() const { - return _id; - } + const document::BucketId& getBucketId() const { return _id; } private: Disk& _disk; @@ -169,38 +149,26 @@ public: uint8_t minPriorityToBeBlocking); ~FileStorHandlerImpl(); - void setGetNextMessageTimeout(uint32_t timeout) - { _getNextMessageTimeout = timeout; } + void setGetNextMessageTimeout(uint32_t timeout) { _getNextMessageTimeout = timeout; } void flush(bool killPendingMerges); void setDiskState(uint16_t disk, DiskState state); DiskState getDiskState(uint16_t disk) const; void close(); - bool schedule(const std::shared_ptr<api::StorageMessage>&, - uint16_t disk); + bool schedule(const std::shared_ptr<api::StorageMessage>&, uint16_t disk); void pause(uint16_t disk, uint8_t priority) const; + FileStorHandler::LockedMessage getNextMessage(uint16_t disk, uint8_t lowestPriority); - FileStorHandler::LockedMessage getNextMessage(uint16_t disk, - uint8_t lowestPriority); - - FileStorHandler::LockedMessage & getNextMessage(uint16_t disk, - FileStorHandler::LockedMessage& lock, - uint8_t lowestPriority); + FileStorHandler::LockedMessage & getNextMessage(uint16_t disk, FileStorHandler::LockedMessage& lock, + uint8_t lowestPriority); enum Operation { MOVE, SPLIT, JOIN }; - void remapQueue( - const RemapInfo& source, - RemapInfo& target, - Operation op); - - void remapQueue(const RemapInfo& source, - RemapInfo& target1, - RemapInfo& target2, - Operation op); - - void failOperations(const document::BucketId&, uint16_t fromDisk, - const api::ReturnCode&); + void remapQueue(const RemapInfo& source, RemapInfo& target, Operation op); + + void remapQueue(const RemapInfo& source, RemapInfo& target1, RemapInfo& target2, Operation op); + + void failOperations(const document::BucketId&, uint16_t fromDisk, const api::ReturnCode&); void sendCommand(const std::shared_ptr<api::StorageCommand>&); void sendReply(const std::shared_ptr<api::StorageReply>&); @@ -214,19 +182,13 @@ public: void addMergeStatus(const document::BucketId&, MergeStatus::SP); MergeStatus& editMergeStatus(const document::BucketId&); - bool isMerging(const document::BucketId&) const; - uint32_t getNumActiveMerges() const; - void clearMergeStatus(const document::BucketId&, const api::ReturnCode*); std::string dumpQueue(uint16_t disk) const; - ResumeGuard pause(); - void resume(); - void abortQueuedOperations(const AbortBucketOperationsCommand& cmd); private: @@ -251,9 +213,7 @@ private: static document::BucketId getBucketId(const api::StorageMessage&); // Returns the index in the targets array we are sending to, or -1 if none of them match. - int calculateTargetBasedOnDocId( - const api::StorageMessage& msg, - std::vector<RemapInfo*>& targets); + int calculateTargetBasedOnDocId(const api::StorageMessage& msg, std::vector<RemapInfo*>& targets); /** * If FileStor layer is explicitly paused, try to wait a single time, then @@ -281,59 +241,38 @@ private: * Often, operations such as streaming searches will have dedicated threads * that refuse lower priority operations such as Puts etc. */ - bool operationHasHighEnoughPriorityToBeRun( - const api::StorageMessage& msg, - uint8_t maxPriority) const; + bool operationHasHighEnoughPriorityToBeRun(const api::StorageMessage& msg, uint8_t maxPriority) const; /** * Return whether an already running high priority operation pre-empts * (blocks) the operation in msg from even starting in the current thread. */ - bool operationBlockedByHigherPriorityThread( - const api::StorageMessage& msg, - const Disk& disk) const; + bool operationBlockedByHigherPriorityThread(const api::StorageMessage& msg, const Disk& disk) const; /** * Return whether msg has timed out based on waitTime and the message's * specified timeout. */ - bool messageTimedOutInQueue(const api::StorageMessage& msg, - uint64_t waitTime) const; + bool messageTimedOutInQueue(const api::StorageMessage& msg, uint64_t waitTime) const; /** * Assume ownership of lock for a given bucket on a given disk. * Disk lock MUST have been taken prior to calling this function. */ std::unique_ptr<FileStorHandler::BucketLockInterface> - takeDiskBucketLockOwnership(Disk& disk, - const document::BucketId& id, - const api::StorageMessage& msg); + takeDiskBucketLockOwnership(Disk& disk, const document::BucketId& id, const api::StorageMessage& msg); /** * Creates and returns a reply with api::TIMEOUT return code for msg. * Swaps (invalidates) context from msg into reply. */ - std::unique_ptr<api::StorageReply> - makeQueueTimeoutReply(api::StorageMessage& msg) const; - - bool bucketIsLockedOnDisk(const document::BucketId&, - const Disk&) const; - + std::unique_ptr<api::StorageReply> makeQueueTimeoutReply(api::StorageMessage& msg) const; + bool bucketIsLockedOnDisk(const document::BucketId&, const Disk&) const; bool messageMayBeAborted(const api::StorageMessage& msg) const; - bool hasBlockingOperations(const Disk& t) const; - - void abortQueuedCommandsForBuckets( - Disk& disk, - const AbortBucketOperationsCommand& cmd); - - bool diskHasActiveOperationForAbortedBucket( - const Disk& disk, - const AbortBucketOperationsCommand& cmd) const; - - void waitUntilNoActiveOperationsForAbortedBuckets( - Disk& disk, - const AbortBucketOperationsCommand& cmd); + void abortQueuedCommandsForBuckets(Disk& disk, const AbortBucketOperationsCommand& cmd); + bool diskHasActiveOperationForAbortedBucket(const Disk& disk, const AbortBucketOperationsCommand& cmd) const; + void waitUntilNoActiveOperationsForAbortedBuckets(Disk& disk, const AbortBucketOperationsCommand& cmd); // Update hook void updateMetrics(const MetricLockGuard &) override; @@ -345,11 +284,7 @@ private: uint16_t& targetDisk, api::ReturnCode& returnCode); - void remapQueueNoLock( - Disk& from, - const RemapInfo& source, - std::vector<RemapInfo*>& targets, - Operation op); + void remapQueueNoLock(Disk& from, const RemapInfo& source, std::vector<RemapInfo*>& targets, Operation op); /** * Waits until the queue has no pending operations (i.e. no locks are diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 12d0e4738f9..d77b62c3c6e 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -2,31 +2,21 @@ #include "filestormanager.h" -#include <vespa/document/bucket/bucketidfactory.h> -#include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/multioperation.h> -#include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/state.h> -#include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/persistence/persistencethread.h> -#include <vespa/storage/persistence/messages.h> -#include <vespa/storage/storageserver/statemanager.h> -#include <vespa/storage/storageserver/storagemetricsset.h> #include <vespa/storage/storageutil/log.h> #include <vespa/storage/common/messagebucketid.h> #include <vespa/storage/persistence/bucketownershipnotifier.h> -#include <vespa/vdslib/state/random.h> -#include <vespa/vdslib/state/nodestate.h> -#include <vespa/storageapi/message/stat.h> #include <vespa/storageapi/message/batch.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storage/bucketdb/lockablemap.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> LOG_SETUP(".persistence.filestor.manager"); diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp index 76744dc2a5b..f0e640afe27 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> +#include "modifiedbucketchecker.h" +#include "filestormanager.h" #include <vespa/log/log.h> -#include <vespa/storage/persistence/filestorage/filestormanager.h> LOG_SETUP(".persistence.filestor.modifiedbucketchecker"); diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index 8978c504a41..f84f9d38386 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -189,4 +189,4 @@ AbortBucketOperationsCommand::makeReply() { return std::make_unique<AbortBucketOperationsReply>(*this); } -} +}
\ No newline at end of file diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 8a6f320ac3e..a8f74af539d 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -1,19 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/persistence/persistencethread.h> - -#include <vespa/log/log.h> +#include "persistencethread.h" +#include "splitbitdetector.h" +#include "bucketownershipnotifier.h" +#include "testandsethelper.h" #include <vespa/storageapi/message/multioperation.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storage/persistence/splitbitdetector.h> -#include <vespa/storage/persistence/bucketownershipnotifier.h> -#include <vespa/storage/persistence/testandsethelper.h> -#include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/document/fieldset/fieldsetrepo.h> - +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".persistence.thread"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index c4ed6031d64..23990cd2108 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> +#include "processallhandler.h" +#include "bucketprocessor.h" #include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/persistence/processallhandler.h> -#include <vespa/storage/persistence/bucketprocessor.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".persistence.processall"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/types.cpp b/storage/src/vespa/storage/persistence/types.cpp index ca8141a1ac6..beeb9d70748 100644 --- a/storage/src/vespa/storage/persistence/types.cpp +++ b/storage/src/vespa/storage/persistence/types.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <iostream> -#include <vespa/storage/persistence/types.h> +#include "types.h" namespace storage { diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index b3ed31dcb88..e9b7013d04a 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -1,17 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "communicationmanager.h" -#include "storagemetricsset.h" -#include "documentapiconverter.h" #include <vespa/storageapi/message/state.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/storageapi/message/persistence.h> #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/rpcmessagebus.h> -#include <vespa/messagebus/sourcesessionparams.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/common/nodestateupdater.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".communication.manager"); diff --git a/storage/src/vespa/storage/visiting/dumpvisitor.cpp b/storage/src/vespa/storage/visiting/dumpvisitor.cpp index 051022521e8..bbb23d295f1 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitor.cpp +++ b/storage/src/vespa/storage/visiting/dumpvisitor.cpp @@ -4,7 +4,7 @@ #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/document/update/documentupdate.h> #include <vespa/vdslib/container/mutabledocumentlist.h> -#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.dumpvisitor"); diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index f133b2a7c0c..705b3f8d45e 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/visiting/recoveryvisitor.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/log/log.h> +#include "recoveryvisitor.h" + #include <vespa/documentapi/messagebus/messages/visitor.h> -#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".visitor.instance.recoveryvisitor"); namespace storage { diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 7214ed005e9..28694185d77 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -2,18 +2,14 @@ #include "visitor.h" #include "visitormetrics.h" -#include <vespa/log/log.h> #include <vespa/storageapi/message/datagram.h> -#include <vespa/storageapi/message/visitor.h> #include <vespa/storage/persistence/messages.h> -#include <vespa/storageframework/storageframework.h> -#include <vespa/documentapi/messagebus/messages/documentmessage.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/document/select/node.h> -#include <sstream> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/asciistream.h> #include <unordered_map> -#include <cmath> - +#include <vespa/log/log.h> LOG_SETUP(".visitor.instance"); namespace storage { @@ -560,7 +556,7 @@ Visitor::reportProblem(const std::string& problem) void Visitor::reportProblem(const api::ReturnCode& problemCode) { - std::ostringstream os; + vespalib::asciistream os; os << "[From content node " << _ownNodeIndex << "] "; os << api::ReturnCode::getResultString(problemCode.getResult()) << ": " << problemCode.getMessage(); diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index 948783c438a..33beae14c4c 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -1,22 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/visiting/visitormanager.h> - -#include <cstdlib> -#include <vespa/log/log.h> +#include "visitormanager.h" +#include "messages.h" +#include "dumpvisitor.h" +#include "dumpvisitorsingle.h" +#include "countvisitor.h" +#include "testvisitor.h" +#include "recoveryvisitor.h" #include <vespa/storageapi/message/visitor.h> #include <vespa/storage/common/statusmessages.h> #include <vespa/storage/storageserver/storagemetricsset.h> -#include <vespa/storage/visiting/messages.h> - -#include <vespa/storage/visiting/dumpvisitor.h> -#include <vespa/storage/visiting/dumpvisitorsingle.h> -#include <vespa/storage/visiting/countvisitor.h> -#include <vespa/storage/visiting/testvisitor.h> -#include <vespa/storage/visiting/recoveryvisitor.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> +#include <vespa/log/log.h> LOG_SETUP(".visitor.manager"); namespace storage { diff --git a/storage/src/vespa/storage/visiting/visitormetrics.cpp b/storage/src/vespa/storage/visiting/visitormetrics.cpp index 1ea9e7c9cd4..f36bda151e6 100644 --- a/storage/src/vespa/storage/visiting/visitormetrics.cpp +++ b/storage/src/vespa/storage/visiting/visitormetrics.cpp @@ -4,7 +4,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> - namespace storage { VisitorMetrics::VisitorMetrics() diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 9b1faf46ef7..61cdfba1a11 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "visitorthread.h" - +#include "messages.h" #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/bodyfielddetector.h> #include <vespa/document/select/orderingselector.h> @@ -9,14 +9,12 @@ #include <vespa/messagebus/rpcmessagebus.h> #include <vespa/storage/common/statusmessages.h> #include <vespa/storage/config/config-stor-server.h> -#include <vespa/storage/visiting/messages.h> #include <vespa/storageapi/message/datagram.h> -#include <vespa/storageapi/message/visitor.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/base/exceptions.h> -#include <algorithm> #include <locale> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".visitor.thread"); diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp index 7b74d17ada1..341a11ba5ee 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -2,6 +2,7 @@ #include "bucket.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/array.hpp> namespace storage { @@ -27,8 +28,7 @@ IMPLEMENT_REPLY(SetBucketStateReply) CreateBucketCommand::CreateBucketCommand(const document::BucketId& id) : MaintenanceCommand(MessageType::CREATEBUCKET, id), _active(false) -{ -} +{ } void CreateBucketCommand::print(std::ostream& out, bool verbose, @@ -66,8 +66,7 @@ CreateBucketReply::print(std::ostream& out, bool verbose, DeleteBucketCommand::DeleteBucketCommand(const document::BucketId& id) : MaintenanceCommand(MessageType::DELETEBUCKET, id) -{ -} +{ } void DeleteBucketCommand::print(std::ostream& out, bool verbose, @@ -496,8 +495,9 @@ RequestBucketInfoReply::RequestBucketInfoReply( const RequestBucketInfoCommand& cmd) : StorageReply(cmd), _buckets() -{ -} +{ } + +RequestBucketInfoReply::~RequestBucketInfoReply() { } uint32_t RequestBucketInfoReply::getMemoryFootprint() const @@ -618,3 +618,5 @@ SetBucketStateReply::print(std::ostream& out, } // api } // storage + +template class vespalib::Array<storage::api::RequestBucketInfoReply::Entry>; diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index 7f5ba157d3d..d5c4584fe4e 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -400,8 +400,8 @@ public: document::BucketId _bucketId; BucketInfo _info; - bool operator==(const Entry& e) const - { return (_bucketId == e._bucketId && _info == e._info); } + bool operator==(const Entry& e) const { return (_bucketId == e._bucketId && _info == e._info); } + bool operator!=(const Entry& e) const { return !(*this == e); } Entry() : _bucketId(), _info() {} Entry(const document::BucketId& id, const BucketInfo& info) : _bucketId(id), _info(info) {} @@ -414,6 +414,7 @@ private: public: explicit RequestBucketInfoReply(const RequestBucketInfoCommand& cmd); + ~RequestBucketInfoReply(); const EntryVector & getBucketInfo() const { return _buckets; } EntryVector & getBucketInfo() { return _buckets; } diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index 3dee0e2e4fb..9d2ec3624e8 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* $Id$ */ - - -#include <vespa/fastos/fastos.h> -#include <vespa/storageapi/message/visitor.h> +#include "visitor.h" +#include <vespa/vespalib/util/array.hpp> namespace storage { namespace api { diff --git a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp index df40062ce07..76a1a832ca6 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "searchenvironment.h" +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> LOG_SETUP(".visitor.instance.searchenvironment"); using search::docsummary::JuniperProperties; @@ -55,8 +55,7 @@ SearchEnvironment::SearchEnvironment(const config::ConfigUri & configUri) : VisitorEnvironment(), _envMap(), _configUri(configUri) -{ -} +{ } SearchEnvironment::~SearchEnvironment() { diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index d24dcb6b3a4..d82f89da29d 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -3,6 +3,7 @@ #include "parameters.hpp" #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/objects/hexdump.h> +#include <vespa/vespalib/stllike/hash_map.hpp> using namespace vdslib; @@ -168,4 +169,6 @@ template int32_t vdslib::Parameters::get(const vespalib::stringref &, int32_t) c template int64_t vdslib::Parameters::get(const vespalib::stringref &, int64_t) const; template uint64_t vdslib::Parameters::get(const vespalib::stringref &, uint64_t) const; template double vdslib::Parameters::get(const vespalib::stringref &, double) const; -template std::string vdslib::Parameters::get(const vespalib::stringref &, std::string) const;
\ No newline at end of file +template std::string vdslib::Parameters::get(const vespalib::stringref &, std::string) const; + +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vdslib::Parameters::Value);
\ No newline at end of file diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp index 824a8adf1d6..73b0d559a15 100644 --- a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp +++ b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/data/memorydatastore.h> +#include <vespa/vespalib/util/array.hpp> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp index d3026211c61..be2089b4076 100644 --- a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp @@ -3,6 +3,7 @@ #include "binary_format.h" #include "inserter.h" #include "slime.h" +#include <vespa/vespalib/util/array.hpp> namespace vespalib { namespace slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp b/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp index 57b0b2e843a..fe1d0a1a78b 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "symbol_table.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace vespalib { namespace slime { @@ -18,5 +19,25 @@ SymbolTable::clear() { _symbols.clear(); } +Symbol +SymbolTable::insert(const Memory &name) { + SymbolMap::const_iterator pos = _symbols.find(name); + if (pos == _symbols.end()) { + Symbol symbol(_names.size()); + SymbolVector::Reference r(_names.push_back(name.data, name.size)); + _symbols.insert(std::make_pair(Memory(r.c_str(), r.size()), symbol)); + return symbol; + } + return pos->second; +} +Symbol +SymbolTable::lookup(const Memory &name) const { + SymbolMap::const_iterator pos = _symbols.find(name); + if (pos == _symbols.end()) { + return Symbol(); + } + return pos->second; +} + } // namespace vespalib::slime } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h index 23cdbecdb5e..e4cafa2b9d6 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h +++ b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h @@ -21,8 +21,8 @@ private: return lcm.hash(); } }; - typedef hash_map<Memory, Symbol, hasher> SymbolMap; - typedef VariableSizeVector SymbolVector; + using SymbolMap = hash_map<Memory, Symbol, hasher>; + using SymbolVector = VariableSizeVector; SymbolMap _symbols; SymbolVector _names; @@ -38,23 +38,8 @@ public: SymbolVector::Reference r(_names[symbol.getValue()]); return Memory(r.c_str(), r.size()); } - Symbol insert(const Memory &name) { - SymbolMap::const_iterator pos = _symbols.find(name); - if (pos == _symbols.end()) { - Symbol symbol(_names.size()); - SymbolVector::Reference r(_names.push_back(name.data, name.size)); - _symbols.insert(std::make_pair(Memory(r.c_str(), r.size()), symbol)); - return symbol; - } - return pos->second; - } - Symbol lookup(const Memory &name) const { - SymbolMap::const_iterator pos = _symbols.find(name); - if (pos == _symbols.end()) { - return Symbol(); - } - return pos->second; - } + Symbol insert(const Memory &name); + Symbol lookup(const Memory &name) const; void clear(); }; diff --git a/vespalib/src/vespa/vespalib/eval/llvm/llvm_wrapper.cpp b/vespalib/src/vespa/vespalib/eval/llvm/llvm_wrapper.cpp index c5ceacb3a5a..1222eeae837 100644 --- a/vespalib/src/vespa/vespalib/eval/llvm/llvm_wrapper.cpp +++ b/vespalib/src/vespa/vespalib/eval/llvm/llvm_wrapper.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <cmath> #include "llvm_wrapper.h" #include <vespa/vespalib/eval/node_visitor.h> diff --git a/vespalib/src/vespa/vespalib/stllike/CMakeLists.txt b/vespalib/src/vespa/vespalib/stllike/CMakeLists.txt index c03dc4d1703..3900c6210d4 100644 --- a/vespalib/src/vespa/vespalib/stllike/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/stllike/CMakeLists.txt @@ -3,6 +3,9 @@ vespa_add_library(vespalib_vespalib_stllike OBJECT SOURCES asciistream.cpp hashtable.cpp + hashtable.cpp + hash_set.cpp + hash_map.cpp string.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.cpp b/vespalib/src/vespa/vespalib/stllike/hash_map.cpp new file mode 100644 index 00000000000..bbc8ea17991 --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hash_map.hpp" + +namespace vespalib { +} + +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vespalib::string); +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, int32_t); +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, uint32_t); +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, uint64_t); +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, double); +VESPALIB_HASH_MAP_INSTANTIATE(int64_t, int32_t); +VESPALIB_HASH_MAP_INSTANTIATE(uint32_t, uint32_t); +VESPALIB_HASH_MAP_INSTANTIATE(uint64_t, uint32_t); +VESPALIB_HASH_MAP_INSTANTIATE(double, uint32_t);
\ No newline at end of file diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h index 22f352fcd99..12c6d3574e4 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h @@ -1,7 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/stllike/hashtable.h> +#include "hashtable.h" +#include "hash_fun.h" namespace vespalib { @@ -12,15 +13,20 @@ public: typedef std::pair<K, V> value_type; typedef K key_type; typedef V mapped_type; + using HashTable = hashtable< K, value_type, H, EQ, std::_Select1st< value_type >, M >; private: - typedef hashtable< K, value_type, H, EQ, std::_Select1st< value_type >, M > HashTable; HashTable _ht; public: typedef typename HashTable::iterator iterator; typedef typename HashTable::const_iterator const_iterator; typedef typename HashTable::insert_result insert_result; public: - hash_map(size_t reserveSize=0) : _ht(reserveSize) { } + hash_map(hash_map &&) = default; + hash_map & operator = (hash_map &&) = default; + hash_map(const hash_map &) = default; + hash_map & operator = (const hash_map &) = default; + hash_map(size_t reserveSize=0); + ~hash_map(); iterator begin() { return _ht.begin(); } iterator end() { return _ht.end(); } const_iterator begin() const { return _ht.begin(); } @@ -28,52 +34,28 @@ public: size_t capacity() const { return _ht.capacity(); } size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } - insert_result insert(const value_type & value) { return _ht.insert(value); } + insert_result insert(const value_type & value); template <typename InputIt> void insert(InputIt first, InputIt last); const V & operator [] (const K & key) const { return _ht.find(key)->second; } V & operator [] (const K & key) { return _ht.insert(value_type(key, V())).first->second; } - void erase(const K & key) { return _ht.erase(key); } - void erase(iterator it) { return _ht.erase(it->first); } - void erase(const_iterator it) { return _ht.erase(it->first); } + void erase(const K & key); + void erase(iterator it) { return erase(it->first); } + void erase(const_iterator it) { return erase(it->first); } iterator find(const K & key) { return _ht.find(key); } const_iterator find(const K & key) const { return _ht.find(key); } - void clear() { _ht.clear(); } - void resize(size_t newSize) { _ht.resize(newSize); } - void swap(hash_map & rhs) { _ht.swap(rhs._ht); } + void clear(); + void resize(size_t newSize); + void swap(hash_map & rhs); bool operator == (const hash_map & rhs) const; - size_t getMemoryConsumption() const { return _ht.getMemoryConsumption(); } - size_t getMemoryUsed() const { return _ht.getMemoryUsed(); } + size_t getMemoryConsumption() const; + size_t getMemoryUsed() const; }; -template <typename K, typename V, typename H, typename EQ, typename M> -bool hash_map<K, V, H, EQ, M>::operator ==(const hash_map & rhs) const { - bool identical(rhs.size() == size()); - if (identical) { - for(const_iterator at(begin()), mat(end()); identical && at != mat; at++) { - const_iterator bt = rhs.find(at->first); - identical = (bt != rhs.end()) && (*at == *bt); - } - } - return identical; -} - -template <typename K, typename V, typename H, typename EQ, typename M> -template <typename InputIt> -void hash_map<K, V, H, EQ, M>::insert(InputIt first, InputIt last) { - while (first != last) { - _ht.insert(*first); - ++first; - } -} - template< typename K, typename V, typename H, typename EQ, typename M > void swap(hash_map<K, V, H, EQ, M> & a, hash_map<K, V, H, EQ, M> & b) { a.swap(b); } - } - - diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp new file mode 100644 index 00000000000..0a97a1f17f1 --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "hash_map.h" +#include "hashtable.hpp" + +namespace vespalib { + +template <typename K, typename V, typename H, typename EQ, typename M> +hash_map<K, V, H, EQ, M>::hash_map(size_t reserveSize) : + _ht(reserveSize) +{ } + +template <typename K, typename V, typename H, typename EQ, typename M> +hash_map<K, V, H, EQ, M>::~hash_map() { } + +template <typename K, typename V, typename H, typename EQ, typename M> +bool +hash_map<K, V, H, EQ, M>::operator ==(const hash_map & rhs) const { + bool identical(rhs.size() == size()); + if (identical) { + for(const_iterator at(begin()), mat(end()); identical && at != mat; at++) { + const_iterator bt = rhs.find(at->first); + identical = (bt != rhs.end()) && (*at == *bt); + } + } + return identical; +} + +template <typename K, typename V, typename H, typename EQ, typename M> +typename hash_map<K, V, H, EQ, M>::insert_result +hash_map<K, V, H, EQ, M>::insert(const value_type & value) { + return _ht.insert(value); +} + +template <typename K, typename V, typename H, typename EQ, typename M> +void +hash_map<K, V, H, EQ, M>::erase(const K & key) { + return _ht.erase(key); +} + +template <typename K, typename V, typename H, typename EQ, typename M> +template <typename InputIt> +void +hash_map<K, V, H, EQ, M>::insert(InputIt first, InputIt last) { + while (first != last) { + _ht.insert(*first); + ++first; + } +} + +template <typename K, typename V, typename H, typename EQ, typename M> +void +hash_map<K, V, H, EQ, M>::clear() { + _ht.clear(); +} + +template <typename K, typename V, typename H, typename EQ, typename M> +void +hash_map<K, V, H, EQ, M>::resize(size_t newSize) { + _ht.resize(newSize); +} + +template <typename K, typename V, typename H, typename EQ, typename M> +void +hash_map<K, V, H, EQ, M>::swap(hash_map & rhs) { + _ht.swap(rhs._ht); +} + +template <typename K, typename V, typename H, typename EQ, typename M> +size_t +hash_map<K, V, H, EQ, M>::getMemoryConsumption() const { + return _ht.getMemoryConsumption(); +} + +template <typename K, typename V, typename H, typename EQ, typename M> +size_t +hash_map<K, V, H, EQ, M>::getMemoryUsed() const +{ + return _ht.getMemoryUsed(); +} + +} + +#define VESPALIB_HASH_MAP_INSTANTIATE(K, V) \ + template class vespalib::hash_map<K, V>; \ + template class vespalib::hashtable<K, std::pair<K,V>, vespalib::hash<K>, std::equal_to<K>, std::_Select1st<std::pair<K,V>>>; \ + template vespalib::hashtable<K, std::pair<K,V>, vespalib::hash<K>, std::equal_to<K>, std::_Select1st<std::pair<K,V>>>::insert_result \ + vespalib::hashtable<K, std::pair<K,V>, vespalib::hash<K>, std::equal_to<K>, std::_Select1st<std::pair<K,V>>>::insert(std::pair<K,V> &&); \ + template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.cpp b/vespalib/src/vespa/vespalib/stllike/hash_set.cpp new file mode 100644 index 00000000000..ef0c0e6188e --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.cpp @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hash_set.hpp" + +namespace vespalib { +} + +VESPALIB_HASH_SET_INSTANTIATE(int32_t); +VESPALIB_HASH_SET_INSTANTIATE(uint32_t); +VESPALIB_HASH_SET_INSTANTIATE(uint64_t); +VESPALIB_HASH_SET_INSTANTIATE(double); +VESPALIB_HASH_SET_INSTANTIATE(vespalib::string); +VESPALIB_HASH_SET_INSTANTIATE(std::string); +VESPALIB_HASH_SET_INSTANTIATE(const void *);
\ No newline at end of file diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.h b/vespalib/src/vespa/vespalib/stllike/hash_set.h index ff7049b7784..c74a377cc18 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.h @@ -1,7 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/stllike/hashtable.h> +#include "hashtable.h" +#include "hash_fun.h" #include <initializer_list> namespace vespalib { @@ -10,26 +11,24 @@ template< typename K, typename H = vespalib::hash<K>, typename EQ = std::equal_t class hash_set { private: - typedef hashtable< K, K, H, EQ, std::_Identity<K>, M> HashTable; + using HashTable = hashtable< K, K, H, EQ, std::_Identity<K>, M>; HashTable _ht; public: typedef typename HashTable::iterator iterator; typedef typename HashTable::const_iterator const_iterator; typedef typename HashTable::insert_result insert_result; public: - hash_set(size_t reserveSize=0) : _ht(reserveSize) { } - hash_set(size_t reserveSize, const H & hasher, const EQ & equal) : _ht(reserveSize, hasher, equal) { } + hash_set(hash_set &&) = default; + hash_set & operator = (hash_set &&) = default; + hash_set(const hash_set &) = default; + hash_set & operator = (const hash_set &) = default; + hash_set(size_t reserveSize=0); + hash_set(size_t reserveSize, const H & hasher, const EQ & equal); template <typename InputIterator> - hash_set(InputIterator first, InputIterator last) - : _ht(0) - { - insert(first, last); - } - hash_set(std::initializer_list<K> input) - : _ht(0) - { - insert(input.begin(), input.end()); - } + hash_set(InputIterator first, InputIterator last); + + hash_set(std::initializer_list<K> input); + ~hash_set(); iterator begin() { return _ht.begin(); } iterator end() { return _ht.end(); } const_iterator begin() const { return _ht.begin(); } @@ -37,16 +36,10 @@ public: size_t capacity() const { return _ht.capacity(); } size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } - insert_result insert(const K & value) { return _ht.insert(value); } + insert_result insert(const K & value); template<typename InputIt> - void insert(InputIt first, InputIt last) { - _ht.resize(last-first + capacity()); - for(;first < last; first++) { - _ht.insert(*first); - } - } - void erase(const K & key) { return _ht.erase(key); } - void erase(const iterator & it) { return _ht.erase(it); } + void insert(InputIt first, InputIt last); + void erase(const K & key); iterator find(const K & key) { return _ht.find(key); } const_iterator find(const K & key) const { return _ht.find(key); } @@ -66,25 +59,17 @@ public: return _ht.template find<AltKey, AltExtract, AltHash, AltEqual>(key, altExtract); } - void clear() { _ht.clear(); } - void resize(size_t newSize) { _ht.resize(newSize); } - void swap(hash_set & rhs) { _ht.swap(rhs._ht); } + void clear(); + void resize(size_t newSize); + void swap(hash_set & rhs); - bool operator==(const hash_set &rhs) const { - bool equal = (size() == rhs.size()); - if (equal) { - for (auto itr = begin(), endItr = end(); equal && itr != endItr; ++itr) { - equal = (rhs.find(*itr) != rhs.end()); - } - } - return equal; - } + bool operator==(const hash_set &rhs) const; /** * Get an approximate number of memory consumed by hash set. Not including * any data K would store outside of sizeof(K) of course. */ - size_t getMemoryConsumption() const { return _ht.getMemoryConsumption(); } + size_t getMemoryConsumption() const; }; template< typename K, typename H, typename EQ, typename M > diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp new file mode 100644 index 00000000000..98c57a76358 --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "hash_set.h" +#include "hashtable.hpp" + +namespace vespalib { + +template<typename K, typename H, typename EQ, typename M> +hash_set<K, H, EQ, M>::hash_set(size_t reserveSize) + : _ht(reserveSize) +{ } + +template<typename K, typename H, typename EQ, typename M> +hash_set<K, H, EQ, M>::hash_set(size_t reserveSize, const H &hasher, const EQ &equal) + : _ht(reserveSize, hasher, equal) +{ } + +template<typename K, typename H, typename EQ, typename M> +template<typename InputIterator> +hash_set<K, H, EQ, M>::hash_set(InputIterator first, InputIterator last) + : _ht(0) +{ + insert(first, last); +} + +template<typename K, typename H, typename EQ, typename M> +hash_set<K, H, EQ, M>::hash_set(std::initializer_list<K> input) + : _ht(0) +{ + insert(input.begin(), input.end()); +} + +template<typename K, typename H, typename EQ, typename M> +hash_set<K, H, EQ, M>::~hash_set() {} + +template<typename K, typename H, typename EQ, typename M> +bool +hash_set<K, H, EQ, M>::operator==(const hash_set &rhs) const { + bool equal = (size() == rhs.size()); + if (equal) { + for (auto itr = begin(), endItr = end(); equal && itr != endItr; ++itr) { + equal = (rhs.find(*itr) != rhs.end()); + } + } + return equal; +} + +template<typename K, typename H, typename EQ, typename M> +void +hash_set<K, H, EQ, M>::clear() { + _ht.clear(); +} + +template<typename K, typename H, typename EQ, typename M> +void +hash_set<K, H, EQ, M>::resize(size_t newSize) { + _ht.resize(newSize); +} + +template<typename K, typename H, typename EQ, typename M> +void +hash_set<K, H, EQ, M>::swap(hash_set &rhs) { + _ht.swap(rhs._ht); +} + +template<typename K, typename H, typename EQ, typename M> +size_t +hash_set<K, H, EQ, M>::getMemoryConsumption() const { + return _ht.getMemoryConsumption(); +} + +template<typename K, typename H, typename EQ, typename M> +template<typename InputIt> +void +hash_set<K, H, EQ, M>::insert(InputIt first, InputIt last) { + _ht.resize(last - first + capacity()); + for (; first < last; first++) { + _ht.insert(*first); + } +} + +template<typename K, typename H, typename EQ, typename M> +void +hash_set<K, H, EQ, M>::erase(const K &key) { + return _ht.erase(key); +} + +template<typename K, typename H, typename EQ, typename M> +typename hash_set<K, H, EQ, M>::insert_result +hash_set<K, H, EQ, M>::insert(const K & value) { + return _ht.insert(value); +} + +} + +#define VESPALIB_HASH_SET_INSTANTIATE(K) \ + template class vespalib::hash_set<K>; \ + template class vespalib::hashtable<K, K, vespalib::hash<K>, std::equal_to<K>, std::_Identity<K>>; \ + template class vespalib::Array<vespalib::hash_node<K>>; + +#define VESPALIB_HASH_SET_INSTANTIATE_H(K, H) \ + template class vespalib::hash_set<K, H>; \ + template class vespalib::hashtable<K, K, H, std::equal_to<K>, std::_Identity<K>>; \ + template class vespalib::Array<vespalib::hash_node<K>>; + diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.cpp b/vespalib/src/vespa/vespalib/stllike/hashtable.cpp index e2da6b255af..28423e5263e 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/stllike/hashtable.h> +#include <vespa/vespalib/stllike/hashtable.hpp> namespace { diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index a1eeb289c31..ae30f04d1d1 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -1,15 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> #include <iterator> -#include <cstddef> -#include <bits/stl_algo.h> -#include <bits/stl_function.h> #include <vespa/vespalib/util/array.h> -#include "hash_fun.h" - namespace vespalib { /** @@ -113,6 +107,9 @@ public: hash_node &operator=(hash_node &&) = default; hash_node(const hash_node &) = default; // These will not be created hash_node &operator=(const hash_node &) = default; // if V is non-copyable. + bool operator == (const hash_node & rhs) const { + return (_next == rhs._next) && (_node == rhs._node); + } V & getValue() { return _node; } const V & getValue() const { return _node; } next_t getNext() const { return _next; } @@ -227,6 +224,10 @@ public: typedef std::pair<iterator, bool> insert_result; public: + hashtable(hashtable &&) = default; + hashtable & operator = (hashtable &&) = default; + hashtable(const hashtable &); + hashtable & operator = (const hashtable &); hashtable(size_t reservedSpace); hashtable(size_t reservedSpace, const Hash & hasher, const Equal & equal); virtual ~hashtable(); @@ -248,20 +249,14 @@ public: const_iterator find(const AltKey & key) const { return find<AltKey, AltExtract, AltHash, AltEqual>(key, AltExtract()); } const_iterator find(const Key & key) const; template <typename V> - insert_result insert(V && node) { return insertInternal(std::forward<V>(node)); } - void erase(const Key & key) { - const_iterator found(find(key)); - if (found != end()) { - DefaultMoveHandler moveHandler; - erase(moveHandler, found); - } - } + insert_result insert(V && node); + void erase(const Key & key); void reserve(size_t sz) { if (sz > _nodes.capacity()) { resize(sz); } } - void clear() { _nodes.clear(); resize(getTableSize()); } + void clear(); void resize(size_t newSize); void swap(hashtable & rhs); @@ -307,244 +302,4 @@ private: void reclaim(MoveHandler & moveHandler, next_t node); }; -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -void hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::swap(hashtable & rhs) -{ - std::swap(_modulator, rhs._modulator); - std::swap(_count, rhs._count); - _nodes.swap(rhs._nodes); - std::swap(_hasher, rhs._hasher); - std::swap(_equal, rhs._equal); - std::swap(_keyExtractor, rhs._keyExtractor); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::hashtable(size_t reservedSpace) : - _modulator(1), - _count(0), - _nodes(1) -{ - if (reservedSpace > 0) { - resize(reservedSpace); - } } - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::hashtable(size_t reservedSpace, const Hash & hasher, const Equal & equal) : - _modulator(1), - _count(0), - _nodes(1), - _hasher(hasher), - _equal(equal) -{ - if (reservedSpace > 0) { - resize(reservedSpace); - } -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::~hashtable() -{ -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::iterator -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) -{ - next_t h = hash(key); - if (_nodes[h].valid()) { - next_t start(h); - do { - if (_equal(_keyExtractor(_nodes[h].getValue()), key)) { - return iterator(this, start, h); - } - h = _nodes[h].getNext(); - } while (h != Node::npos); - } - return end(); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::const_iterator -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) const -{ - next_t h = hash(key); - if (_nodes[h].valid()) { - next_t start(h); - do { - if (_equal(_keyExtractor(_nodes[h].getValue()), key)) { - return const_iterator(this, start, h); - } - h = _nodes[h].getNext(); - } while (h != Node::npos); - } - return end(); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -template< typename AltKey, typename AltExtract, typename AltHash, typename AltEqual> -typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::const_iterator -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & key, const AltExtract & altExtract) const -{ - AltHash altHasher; - next_t h = modulator(altHasher(key)); - if (_nodes[h].valid()) { - next_t start(h); - AltEqual altEqual; - do { - if (altEqual(altExtract(_keyExtractor(_nodes[h].getValue())), key)) { - return const_iterator(this, start, h); - } - h = _nodes[h].getNext(); - } while (h != Node::npos); - } - return end(); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -template< typename AltKey, typename AltExtract, typename AltHash, typename AltEqual> -typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::iterator -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & key, const AltExtract & altExtract) -{ - AltHash altHasher; - next_t h = modulator(altHasher(key)); - if (_nodes[h].valid()) { - next_t start(h); - AltEqual altEqual; - do { - if (altEqual(altExtract(_keyExtractor(_nodes[h].getValue())), key)) { - return iterator(this, start, h); - } - h = _nodes[h].getNext(); - } while (h != Node::npos); - } - return end(); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -template< typename V > -typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insert_result -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insertInternal(V && node) -{ - const next_t h = hash(_keyExtractor(node)); - if ( ! _nodes[h].valid() ) { - _nodes[h] = std::forward<V>(node); - _count++; - return insert_result(iterator(this, h, h), true); - } else if (_nodes.size() <= _nodes.capacity()) { - for (next_t c(h); c != Node::npos; c = _nodes[c].getNext()) { - if (_equal(_keyExtractor(_nodes[c].getValue()), _keyExtractor(node))) { - return insert_result(iterator(this, h, c), false); - } - } - if (_nodes.size() < _nodes.capacity()) { - const next_t p(_nodes[h].getNext()); - const next_t newIdx(_nodes.size()); - _nodes[h].setNext(newIdx); - new (_nodes.push_back_fast()) Node(std::forward<V>(node), p); - _count++; - return insert_result(iterator(this, h, newIdx), true); - } else { - resize(_nodes.capacity()*2); - return insertInternal(std::forward<V>(node)); - } - } else { - resize(_nodes.capacity()*2); - return insertInternal(std::forward<V>(node)); - } -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -template<typename MoveHandler> -void hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::reclaim(MoveHandler & moveHandler, next_t node) -{ - size_t last(_nodes.size()-1); - if (last >= getTableSize()) { - if (last != node) { - next_t h = hash(_keyExtractor(_nodes[last].getValue())); - for (next_t n(_nodes[h].getNext()); n != last; n=_nodes[h].getNext()) { - h = n; - } - move(moveHandler, last, node); - _nodes[h].setNext(node); - } - _nodes.resize(last); - } -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -template <typename MoveHandler> -void -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::erase(MoveHandler & moveHandler, const const_iterator & it) -{ - next_t h = it.getHash(); - next_t prev = Node::npos; - do { - if (h == it.getInternalIndex()) { - if (prev != Node::npos) { - _nodes[prev].setNext(_nodes[h].getNext()); - reclaim(moveHandler, h); - } else { - if (_nodes[h].hasNext()) { - next_t next = _nodes[h].getNext(); - move(moveHandler, next, h); - reclaim(moveHandler, next); - } else { - _nodes[h].invalidate(); - } - } - _count--; - return; - } - prev = h; - h = _nodes[h].getNext(); - } while (h != Node::npos); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -void -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::resize(size_t newSize) -{ - newSize = roundUp2inN(newSize); - next_t newModulo = Modulator::selectHashTableSize(newSize/3); - if (newModulo > newSize) { - newSize = newModulo; - } - NodeStore newStore; - newStore.reserve(roundUp2inN(newSize)); - newStore.resize(newModulo); - _modulator = Modulator(newModulo); - _count = 0; - _nodes.swap(newStore); - move(std::move(newStore)); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -void -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::move(NodeStore && oldStore) -{ - for(typename NodeStore::iterator it(oldStore.begin()), mt(oldStore.end()); it != mt; it++) { - if (it->valid()) { - insert(std::move(it->getValue())); - } - } -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -size_t -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::getMemoryConsumption() const -{ - return sizeof(hashtable<Key, Value, Hash, Equal, KeyExtract>) - + _nodes.capacity() * sizeof(Node); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -size_t -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::getMemoryUsed() const -{ - return sizeof(hashtable<Key, Value, Hash, Equal, KeyExtract>) - + _nodes.size() * sizeof(Node); -} - -} - diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp new file mode 100644 index 00000000000..76454ff7fff --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp @@ -0,0 +1,289 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "hashtable.h" +#include <vespa/vespalib/util/array.hpp> + +namespace vespalib { + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +void hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::swap(hashtable & rhs) +{ + std::swap(_modulator, rhs._modulator); + std::swap(_count, rhs._count); + _nodes.swap(rhs._nodes); + std::swap(_hasher, rhs._hasher); + std::swap(_equal, rhs._equal); + std::swap(_keyExtractor, rhs._keyExtractor); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::hashtable(size_t reservedSpace) : + _modulator(1), + _count(0), + _nodes(1) +{ + if (reservedSpace > 0) { + resize(reservedSpace); + } +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::hashtable(size_t reservedSpace, const Hash & hasher, const Equal & equal) : + _modulator(1), + _count(0), + _nodes(1), + _hasher(hasher), + _equal(equal) +{ + if (reservedSpace > 0) { + resize(reservedSpace); + } +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::hashtable(const hashtable & rhs) + : _modulator(rhs._modulator), + _count(rhs._count), + _nodes(rhs._nodes), + _hasher(rhs._hasher), + _equal(rhs._equal) +{ } + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator> & +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::operator = (const hashtable & rhs) { + hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>(rhs).swap(*this); + return *this; +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::~hashtable() +{ +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::iterator +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) +{ + next_t h = hash(key); + if (_nodes[h].valid()) { + next_t start(h); + do { + if (_equal(_keyExtractor(_nodes[h].getValue()), key)) { + return iterator(this, start, h); + } + h = _nodes[h].getNext(); + } while (h != Node::npos); + } + return end(); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::const_iterator +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) const +{ + next_t h = hash(key); + if (_nodes[h].valid()) { + next_t start(h); + do { + if (_equal(_keyExtractor(_nodes[h].getValue()), key)) { + return const_iterator(this, start, h); + } + h = _nodes[h].getNext(); + } while (h != Node::npos); + } + return end(); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +template< typename AltKey, typename AltExtract, typename AltHash, typename AltEqual> +typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::const_iterator +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & key, const AltExtract & altExtract) const +{ + AltHash altHasher; + next_t h = modulator(altHasher(key)); + if (_nodes[h].valid()) { + next_t start(h); + AltEqual altEqual; + do { + if (altEqual(altExtract(_keyExtractor(_nodes[h].getValue())), key)) { + return const_iterator(this, start, h); + } + h = _nodes[h].getNext(); + } while (h != Node::npos); + } + return end(); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +template< typename AltKey, typename AltExtract, typename AltHash, typename AltEqual> +typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::iterator +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & key, const AltExtract & altExtract) +{ + AltHash altHasher; + next_t h = modulator(altHasher(key)); + if (_nodes[h].valid()) { + next_t start(h); + AltEqual altEqual; + do { + if (altEqual(altExtract(_keyExtractor(_nodes[h].getValue())), key)) { + return iterator(this, start, h); + } + h = _nodes[h].getNext(); + } while (h != Node::npos); + } + return end(); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +template<typename V> +typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insert_result +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insert(V && node) { + return insertInternal(std::forward<V>(node)); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +void +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::erase(const Key & key) { + const_iterator found(find(key)); + if (found != end()) { + DefaultMoveHandler moveHandler; + erase(moveHandler, found); + } +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +void +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::clear() { + _nodes.clear(); + resize(getTableSize()); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +template< typename V > +typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insert_result +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insertInternal(V && node) +{ + const next_t h = hash(_keyExtractor(node)); + if ( ! _nodes[h].valid() ) { + _nodes[h] = std::forward<V>(node); + _count++; + return insert_result(iterator(this, h, h), true); + } else if (_nodes.size() <= _nodes.capacity()) { + for (next_t c(h); c != Node::npos; c = _nodes[c].getNext()) { + if (_equal(_keyExtractor(_nodes[c].getValue()), _keyExtractor(node))) { + return insert_result(iterator(this, h, c), false); + } + } + if (_nodes.size() < _nodes.capacity()) { + const next_t p(_nodes[h].getNext()); + const next_t newIdx(_nodes.size()); + _nodes[h].setNext(newIdx); + new (_nodes.push_back_fast()) Node(std::forward<V>(node), p); + _count++; + return insert_result(iterator(this, h, newIdx), true); + } else { + resize(_nodes.capacity()*2); + return insertInternal(std::forward<V>(node)); + } + } else { + resize(_nodes.capacity()*2); + return insertInternal(std::forward<V>(node)); + } +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +template<typename MoveHandler> +void hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::reclaim(MoveHandler & moveHandler, next_t node) +{ + size_t last(_nodes.size()-1); + if (last >= getTableSize()) { + if (last != node) { + next_t h = hash(_keyExtractor(_nodes[last].getValue())); + for (next_t n(_nodes[h].getNext()); n != last; n=_nodes[h].getNext()) { + h = n; + } + move(moveHandler, last, node); + _nodes[h].setNext(node); + } + _nodes.resize(last); + } +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +template <typename MoveHandler> +void +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::erase(MoveHandler & moveHandler, const const_iterator & it) +{ + next_t h = it.getHash(); + next_t prev = Node::npos; + do { + if (h == it.getInternalIndex()) { + if (prev != Node::npos) { + _nodes[prev].setNext(_nodes[h].getNext()); + reclaim(moveHandler, h); + } else { + if (_nodes[h].hasNext()) { + next_t next = _nodes[h].getNext(); + move(moveHandler, next, h); + reclaim(moveHandler, next); + } else { + _nodes[h].invalidate(); + } + } + _count--; + return; + } + prev = h; + h = _nodes[h].getNext(); + } while (h != Node::npos); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +void +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::resize(size_t newSize) +{ + newSize = roundUp2inN(newSize); + next_t newModulo = Modulator::selectHashTableSize(newSize/3); + if (newModulo > newSize) { + newSize = newModulo; + } + NodeStore newStore; + newStore.reserve(roundUp2inN(newSize)); + newStore.resize(newModulo); + _modulator = Modulator(newModulo); + _count = 0; + _nodes.swap(newStore); + move(std::move(newStore)); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +void +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::move(NodeStore && oldStore) +{ + for(typename NodeStore::iterator it(oldStore.begin()), mt(oldStore.end()); it != mt; it++) { + if (it->valid()) { + insert(std::move(it->getValue())); + } + } +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +size_t +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::getMemoryConsumption() const +{ + return sizeof(hashtable<Key, Value, Hash, Equal, KeyExtract>) + + _nodes.capacity() * sizeof(Node); +} + +template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > +size_t +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::getMemoryUsed() const +{ + return sizeof(hashtable<Key, Value, Hash, Equal, KeyExtract>) + + _nodes.size() * sizeof(Node); +} + +} + diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.cpp b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.cpp index fe626e626cc..8b41d349d43 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.cpp +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor_builder.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> -#include <memory> -#include <string> +#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::IllegalArgumentException; using vespalib::make_string; diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_cells_iterator.h b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_cells_iterator.h index 446db249e02..c3d00fdb28d 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_cells_iterator.h +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_cells_iterator.h @@ -6,6 +6,7 @@ #include <vespa/vespalib/tensor/types.h> #include <vespa/vespalib/eval/value_type.h> #include <vespa/vespalib/tensor/tensor.h> +#include <vespa/vespalib/util/arrayref.h> namespace vespalib { namespace tensor { diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.cpp index 24c48bfb92c..7a0214e2ca1 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sparse_tensor.h" #include "sparse_tensor_address_builder.h" #include "sparse_tensor_match.h" @@ -10,6 +9,7 @@ #include <vespa/vespalib/tensor/tensor_apply.h> #include <vespa/vespalib/tensor/tensor_visitor.h> #include <vespa/vespalib/eval/operation.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <sstream> using vespalib::eval::TensorSpec; @@ -306,4 +306,7 @@ SparseTensor::reduce(const eval::BinaryOperation &op, } } // namespace vespalib::tensor + +template class hash_map<tensor::SparseTensorAddressRef, double>; + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.h b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.h index 5ed3d16b29c..e6682011ba2 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.h +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.h @@ -22,7 +22,7 @@ namespace tensor { class SparseTensor : public Tensor { public: - typedef vespalib::hash_map<SparseTensorAddressRef, double> Cells; + using Cells = vespalib::hash_map<SparseTensorAddressRef, double>; static constexpr size_t STASH_CHUNK_SIZE = 16384u; diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.cpp index 6073acc4669..277bf7963e0 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sparse_tensor_address_reducer.h" #include <vespa/vespalib/eval/value_type.h> +#include <vespa/vespalib/stllike/hash_set.hpp> namespace vespalib { namespace tensor { diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.cpp index 537da7d8085..4092499bf44 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.cpp @@ -2,6 +2,7 @@ #include "sparse_tensor_builder.h" #include <vespa/vespalib/tensor/tensor.h> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace vespalib { namespace tensor { diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_match.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_match.cpp index 30cbad770a3..4b4378796d0 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_match.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_match.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sparse_tensor_match.h" #include "sparse_tensor_address_decoder.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace vespalib { namespace tensor { diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_apply.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_apply.cpp index 23edf418c0b..ab8c1534a39 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_apply.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_apply.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor_apply.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace vespalib { namespace tensor { diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp index 9c263fb3ffa..646377c1d1a 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp @@ -1,7 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - #include "tensor_mapper.h" #include "tensor.h" #include "tensor_visitor.h" @@ -9,6 +7,7 @@ #include <vespa/vespalib/tensor/dense/dense_tensor.h> #include "tensor_address_element_iterator.h" #include "default_tensor.h" +#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::eval::ValueType; diff --git a/vespalib/src/vespa/vespalib/tensor/types.h b/vespalib/src/vespa/vespalib/tensor/types.h index 6dcdfb5f0d8..7bdb37b8ac2 100644 --- a/vespalib/src/vespa/vespalib/tensor/types.h +++ b/vespalib/src/vespa/vespalib/tensor/types.h @@ -4,7 +4,6 @@ #include <vespa/vespalib/stllike/string.h> #include <map> -#include <set> #include <vespa/vespalib/stllike/hash_set.h> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 261c33c8135..6fb3626a3a1 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT alignedmemory.cpp alloc.cpp approx.cpp + array.cpp atomic.cpp backtrace.cpp barrier.cpp diff --git a/vespalib/src/vespa/vespalib/util/array.cpp b/vespalib/src/vespa/vespalib/util/array.cpp new file mode 100644 index 00000000000..364e87d8a66 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/array.cpp @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "array.hpp" + +namespace vespalib { + +template class Array<signed char>; +template class Array<char>; +template class Array<int16_t>; +template class Array<int32_t>; +template class Array<int64_t>; +template class Array<unsigned char>; +template class Array<uint32_t>; +template class Array<uint64_t>; +template class Array<float>; +template class Array<double>; + +} diff --git a/vespalib/src/vespa/vespalib/util/array.h b/vespalib/src/vespa/vespalib/util/array.h index 4016a1dbf3c..4d0699c7886 100644 --- a/vespalib/src/vespa/vespalib/util/array.h +++ b/vespalib/src/vespa/vespalib/util/array.h @@ -2,59 +2,14 @@ #pragma once #include <stdint.h> -#include <stdlib.h> -#include <string.h> #include <sys/types.h> #include <algorithm> -#include <vector> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/optimized.h> #include <tr1/type_traits> namespace vespalib { -template <typename T> class Array; - -/** - * This is a simple wrapper class for a typed array with no memory ownership. - * It is similar to vespalib::stringref - **/ -template <typename T> -class ArrayRef { -public: - ArrayRef(T * v, size_t sz) : _v(v), _sz(sz) { } - ArrayRef(std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { } - inline ArrayRef(Array<T> &v); - ArrayRef() : _v(nullptr), _sz(0) {} - T & operator [] (size_t i) { return _v[i]; } - const T & operator [] (size_t i) const { return _v[i]; } - size_t size() const { return _sz; } - T *begin() { return _v; } - T *end() { return _v + _sz; } -private: - T * _v; - size_t _sz; -}; - -template <typename T> -class ConstArrayRef { -public: - ConstArrayRef(const T *v, size_t sz) : _v(v), _sz(sz) { } - ConstArrayRef(const std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { } - ConstArrayRef(const ArrayRef<T> & v) : _v(&v[0]), _sz(v.size()) { } - inline ConstArrayRef(const Array<T> &v); - ConstArrayRef() : _v(nullptr), _sz(0) {} - const T & operator [] (size_t i) const { return _v[i]; } - size_t size() const { return _sz; } - const T *cbegin() const { return _v; } - const T *cend() const { return _v + _sz; } - const T *begin() const { return _v; } - const T *end() const { return _v + _sz; } -private: - const T *_v; - size_t _sz; -}; - /** * This is a small and compact implementation of a resizeable array. * It has a smaller footprint than std::vector and most important, @@ -134,42 +89,23 @@ public: typedef T value_type; typedef size_t size_type; - Array(const Alloc & initial=Alloc::alloc()) : _array(initial.create(0)), _sz(0) { } + Array(const Alloc & initial=Alloc::alloc()); Array(size_t sz, const Alloc & initial=Alloc::alloc()); Array(Alloc && buf, size_t sz); Array(Array &&rhs); Array(size_t sz, T value, const Alloc & initial=Alloc::alloc()); Array(const_iterator begin, const_iterator end, const Alloc & initial=Alloc::alloc()); Array(const Array & rhs); - Array & operator =(const Array & rhs) { - if (&rhs != this) { - Array t(rhs); - swap(t); - } - return *this; - } - Array & operator =(Array && rhs) { - if (&rhs != this) { - Array t(std::move(rhs)); - swap(t); - } - return *this; - } + Array & operator =(const Array & rhs); + Array & operator =(Array && rhs); ~Array(); void swap(Array & rhs) { _array.swap(rhs._array); std::swap(_sz, rhs._sz); } void resize(size_t n); - void assign(const_iterator begin_, const_iterator end_) { - Array tmp(begin_, end_); - swap(tmp); - } - void reserve(size_t n) { - if (capacity() < n) { - increase(n); - } - } + void assign(const_iterator begin_, const_iterator end_); + void reserve(size_t n); void push_back(const T & v) { std::_Construct(push_back(), v); } iterator push_back() { extend(size()+1); return array(_sz++); } iterator push_back_fast() { return array(_sz++); } @@ -216,179 +152,4 @@ private: size_t _sz; }; -template <typename T> -ArrayRef<T>::ArrayRef(Array<T> &v) - : _v(&v[0]), - _sz(v.size()) -{ -} - -template <typename T> -ConstArrayRef<T>::ConstArrayRef(const Array<T> &v) - : _v(&v[0]), - _sz(v.size()) -{ -} - -template <typename T> -void construct(T * dest, const T * source, size_t sz, std::tr1::false_type) -{ - for (size_t i(0); i < sz; i++) { - std::_Construct(dest + i, *(source + i)); - } -} - -template <typename T> -void construct(T * dest, const T * source, size_t sz, std::tr1::true_type) -{ - memcpy(dest, source, sz*sizeof(T)); -} - -template <typename T> -void construct(T * dest, size_t sz, std::tr1::false_type) -{ - for (size_t i(0); i < sz; i++) { - void *ptr = &dest[i]; - new(ptr) T(); - } -} - -template <typename T> -void construct(T * dest, size_t sz, std::tr1::true_type) -{ - (void) dest; - (void) sz; -} - -template <typename T> -void construct(T * dest, size_t sz, T val, std::tr1::false_type) -{ - for (size_t i(0); i < sz; i++) { - void *ptr = &dest[i]; - new(ptr) T(val); - } -} - -template <typename T> -void construct(T * dest, size_t sz, T val, std::tr1::true_type) -{ - for (size_t i(0); i < sz; i++) { - dest[i] = val; - } -} - -template <typename T> -Array<T>::Array(const Array & rhs) - : _array(rhs._array.create(rhs.size() * sizeof(T))), - _sz(rhs.size()) -{ - construct(array(0), rhs.array(0), _sz, std::tr1::has_trivial_destructor<T>()); -} - -template <typename T> -bool Array<T>::operator ==(const Array & rhs) const -{ - bool retval(size() == rhs.size()); - for (size_t i(0); retval && (i < _sz); i++) { - if (*array(i) != rhs[i]) { - retval = false; - } - } - return retval; -} - -template <typename T> -void Array<T>::resize(size_t n) -{ - if (n > capacity()) { - reserve(n); - } - if (n > _sz) { - construct(array(_sz), n-_sz, std::tr1::has_trivial_destructor<T>()); - } else if (n < _sz) { - std::_Destroy(array(n), array(_sz)); - } - _sz = n; -} - -template <typename T> -void move(T * dest, T * source, size_t sz, std::tr1::false_type) -{ - for (size_t i(0); i < sz; i++) { - std::_Construct(dest + i, std::move(*(source + i))); - std::_Destroy(source + i); - } -} - -template <typename T> -void move(T * dest, const T * source, size_t sz, std::tr1::true_type) -{ - memcpy(dest, source, sz*sizeof(T)); -} - -template <typename T> -void Array<T>::increase(size_t n) -{ - Alloc newArray(_array.create(sizeof(T)*n)); - if (capacity() > 0) { - move(static_cast<T *>(newArray.get()), array(0), _sz, std::tr1::has_trivial_destructor<T>()); - } - _array.swap(newArray); -} - -template <typename T> -Array<T>::Array(Alloc && buf, size_t sz) : - _array(std::move(buf)), - _sz(sz) -{ -} - - -template <typename T> -Array<T>::Array(Array &&rhs) - : _array(std::move(rhs._array)), - _sz(rhs._sz) -{ - rhs._sz = 0; -} - -template <typename T> -Array<T>::Array(size_t sz, const Alloc & initial) : - _array(initial.create(sz * sizeof(T))), - _sz(sz) -{ - construct(array(0), _sz, std::tr1::has_trivial_destructor<T>()); -} - -template <typename T> -Array<T>::Array(size_t sz, T value, const Alloc & initial) : - _array(initial.create(sz * sizeof(T))), - _sz(sz) -{ - construct(array(0), _sz, value, std::tr1::has_trivial_destructor<T>()); -} - -template <typename T> -Array<T>::Array(const_iterator begin_, const_iterator end_, const Alloc & initial) : - _array(initial.create(begin_ != end_ ? sizeof(T) * (end_-begin_) : 0)), - _sz(end_-begin_) -{ - construct(array(0), begin_, _sz, std::tr1::has_trivial_destructor<T>()); -} - -template <typename T> -Array<T>::~Array() -{ - cleanup(); -} - -template <typename T> -void Array<T>::cleanup() -{ - std::_Destroy(array(0), array(_sz)); - _sz = 0; - Alloc().swap(_array); -} - } - diff --git a/vespalib/src/vespa/vespalib/util/array.hpp b/vespalib/src/vespa/vespalib/util/array.hpp new file mode 100644 index 00000000000..a7194c95014 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/array.hpp @@ -0,0 +1,208 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "array.h" +#include <stdlib.h> +#include <string.h> + +namespace vespalib { + +template <typename T> +void construct(T * dest, const T * source, size_t sz, std::tr1::false_type) +{ + for (size_t i(0); i < sz; i++) { + std::_Construct(dest + i, *(source + i)); + } +} + +template <typename T> +void construct(T * dest, const T * source, size_t sz, std::tr1::true_type) +{ + memcpy(dest, source, sz*sizeof(T)); +} + +template <typename T> +void construct(T * dest, size_t sz, std::tr1::false_type) +{ + for (size_t i(0); i < sz; i++) { + void *ptr = &dest[i]; + new(ptr) T(); + } +} + +template <typename T> +void construct(T * dest, size_t sz, std::tr1::true_type) +{ + (void) dest; + (void) sz; +} + +template <typename T> +void construct(T * dest, size_t sz, T val, std::tr1::false_type) +{ + for (size_t i(0); i < sz; i++) { + void *ptr = &dest[i]; + new(ptr) T(val); + } +} + +template <typename T> +void construct(T * dest, size_t sz, T val, std::tr1::true_type) +{ + for (size_t i(0); i < sz; i++) { + dest[i] = val; + } +} + +template <typename T> +Array<T>::Array(const Array & rhs) + : _array(rhs._array.create(rhs.size() * sizeof(T))), + _sz(rhs.size()) +{ + construct(array(0), rhs.array(0), _sz, std::tr1::has_trivial_destructor<T>()); +} + +template <typename T> +Array<T> & Array<T>::operator =(const Array & rhs) +{ + if (&rhs != this) { + Array t(rhs); + swap(t); + } + return *this; +} + +template <typename T> +Array<T> & Array<T>::operator =(Array && rhs) { + if (&rhs != this) { + Array t(std::move(rhs)); + swap(t); + } + return *this; +} + +template <typename T> +void Array<T>::assign(const_iterator begin_, const_iterator end_) { + Array tmp(begin_, end_); + swap(tmp); +} +template <typename T> +void Array<T>::reserve(size_t n) { + if (capacity() < n) { + increase(n); + } +} + +template <typename T> +bool Array<T>::operator ==(const Array & rhs) const +{ + bool retval(size() == rhs.size()); + for (size_t i(0); retval && (i < _sz); i++) { + if ( ! (*array(i) == rhs[i]) ) { + retval = false; + } + } + return retval; +} + +template <typename T> +void Array<T>::resize(size_t n) +{ + if (n > capacity()) { + reserve(n); + } + if (n > _sz) { + construct(array(_sz), n-_sz, std::tr1::has_trivial_destructor<T>()); + } else if (n < _sz) { + std::_Destroy(array(n), array(_sz)); + } + _sz = n; +} + +template <typename T> +void move(T * dest, T * source, size_t sz, std::tr1::false_type) +{ + for (size_t i(0); i < sz; i++) { + std::_Construct(dest + i, std::move(*(source + i))); + std::_Destroy(source + i); + } +} + +template <typename T> +void move(T * dest, const T * source, size_t sz, std::tr1::true_type) +{ + memcpy(dest, source, sz*sizeof(T)); +} + +template <typename T> +void Array<T>::increase(size_t n) +{ + Alloc newArray(_array.create(sizeof(T)*n)); + if (capacity() > 0) { + move(static_cast<T *>(newArray.get()), array(0), _sz, std::tr1::has_trivial_destructor<T>()); + } + _array.swap(newArray); +} + +template <typename T> +Array<T>::Array(const Alloc & initial) + : _array(initial.create(0)), + _sz(0) +{ } + +template <typename T> +Array<T>::Array(Alloc && buf, size_t sz) : + _array(std::move(buf)), + _sz(sz) +{ +} + + +template <typename T> +Array<T>::Array(Array &&rhs) + : _array(std::move(rhs._array)), + _sz(rhs._sz) +{ + rhs._sz = 0; +} + +template <typename T> +Array<T>::Array(size_t sz, const Alloc & initial) : + _array(initial.create(sz * sizeof(T))), + _sz(sz) +{ + construct(array(0), _sz, std::tr1::has_trivial_destructor<T>()); +} + +template <typename T> +Array<T>::Array(size_t sz, T value, const Alloc & initial) : + _array(initial.create(sz * sizeof(T))), + _sz(sz) +{ + construct(array(0), _sz, value, std::tr1::has_trivial_destructor<T>()); +} + +template <typename T> +Array<T>::Array(const_iterator begin_, const_iterator end_, const Alloc & initial) : + _array(initial.create(begin_ != end_ ? sizeof(T) * (end_-begin_) : 0)), + _sz(end_-begin_) +{ + construct(array(0), begin_, _sz, std::tr1::has_trivial_destructor<T>()); +} + +template <typename T> +Array<T>::~Array() +{ + cleanup(); +} + +template <typename T> +void Array<T>::cleanup() +{ + std::_Destroy(array(0), array(_sz)); + _sz = 0; + Alloc().swap(_array); +} + +} + diff --git a/vespalib/src/vespa/vespalib/util/arrayref.h b/vespalib/src/vespa/vespalib/util/arrayref.h new file mode 100644 index 00000000000..1e93fa36576 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/arrayref.h @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "array.h" + +namespace vespalib { + +/** + * This is a simple wrapper class for a typed array with no memory ownership. + * It is similar to vespalib::stringref + **/ +template <typename T> +class ArrayRef { +public: + ArrayRef(T * v, size_t sz) : _v(v), _sz(sz) { } + ArrayRef(std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { } + ArrayRef(Array<T> &v) : _v(&v[0]), _sz(v.size()) { } + T & operator [] (size_t i) { return _v[i]; } + const T & operator [] (size_t i) const { return _v[i]; } + size_t size() const { return _sz; } + T *begin() { return _v; } + T *end() { return _v + _sz; } +private: + T * _v; + size_t _sz; +}; + +template <typename T> +class ConstArrayRef { +public: + ConstArrayRef(const T *v, size_t sz) : _v(v), _sz(sz) { } + ConstArrayRef(const std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { } + ConstArrayRef(const ArrayRef<T> & v) : _v(&v[0]), _sz(v.size()) { } + ConstArrayRef(const Array<T> &v) : _v(&v[0]), _sz(v.size()) { } + ConstArrayRef() : _v(nullptr), _sz(0) {} + const T & operator [] (size_t i) const { return _v[i]; } + size_t size() const { return _sz; } + const T *cbegin() const { return _v; } + const T *cend() const { return _v + _sz; } + const T *begin() const { return _v; } + const T *end() const { return _v + _sz; } +private: + const T *_v; + size_t _sz; +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/generationholder.h b/vespalib/src/vespa/vespalib/util/generationholder.h index d178fc7f4e5..557104e2687 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.h +++ b/vespalib/src/vespa/vespalib/util/generationholder.h @@ -3,6 +3,7 @@ #pragma once #include <vector> +#include <memory> #include "generationhandler.h" namespace vespalib { @@ -22,8 +23,7 @@ public: GenerationHeldBase(size_t size) : _generation(0u), _size(size) - { - } + { } virtual ~GenerationHeldBase(void); size_t getSize(void) const { return _size; } diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h index 1ec0f7b0056..4daccc22d0b 100644 --- a/vespalib/src/vespa/vespalib/util/stash.h +++ b/vespalib/src/vespa/vespalib/util/stash.h @@ -2,9 +2,8 @@ #pragma once -#include <vespa/fastos/fastos.h> #include "traits.h" -#include "array.h" +#include "arrayref.h" namespace vespalib { namespace stash { diff --git a/vsm/src/vespa/vsm/common/docsum.cpp b/vsm/src/vespa/vsm/common/docsum.cpp index 0d07ab6d236..3d32f30aa87 100644 --- a/vsm/src/vespa/vsm/common/docsum.cpp +++ b/vsm/src/vespa/vsm/common/docsum.cpp @@ -1,17 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vsm/common/docsum.h> + +#include "docsum.h" #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #define DEBUGMASK 0x00 using search::DocumentIdT; -namespace vsm -{ +namespace vsm { using document::FieldValue; using document::StringFieldValue; @@ -20,12 +20,9 @@ IMPLEMENT_DUPLICATE(DocSumCache); DocSumCache::DocSumCache() : _list() -{ -} +{ } -DocSumCache::~DocSumCache() -{ -} +DocSumCache::~DocSumCache() { } const Document & DocSumCache::getDocSum(const DocumentIdT & docId) const { diff --git a/vsm/src/vespa/vsm/common/document.cpp b/vsm/src/vespa/vsm/common/document.cpp index b9936188cef..cc7414b8107 100644 --- a/vsm/src/vespa/vsm/common/document.cpp +++ b/vsm/src/vespa/vsm/common/document.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> using search::DocumentIdT; using search::TimeT; @@ -67,17 +68,16 @@ size_t StringFieldIdTMap::highestFieldNo() const Document::Document(const DocumentIdT & doc, size_t maxField) : _docId(doc), _fieldCount(maxField) -{ -} +{ } Document::Document() : _docId(0), _fieldCount(0) -{ -} +{ } -Document::~Document() -{ -} +Document::~Document() { } } + +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vsm::FieldIdTList); +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vsm::IndexFieldMapT);
\ No newline at end of file diff --git a/vsm/src/vespa/vsm/common/documenttypemapping.cpp b/vsm/src/vespa/vsm/common/documenttypemapping.cpp index ea74f2421e2..389c6608d70 100644 --- a/vsm/src/vespa/vsm/common/documenttypemapping.cpp +++ b/vsm/src/vespa/vsm/common/documenttypemapping.cpp @@ -1,26 +1,23 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "documenttypemapping.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + #include <vespa/log/log.h> LOG_SETUP(".vsm.common.documenttypemapping"); -#include <vespa/document/datatype/arraydatatype.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/vsm/common/documenttypemapping.h> -#include <stdexcept> - #define DEBUGMASK 0x00 -namespace vsm -{ +namespace vsm { DocumentTypeMapping::DocumentTypeMapping() : _fieldMap(), _defaultDocumentTypeName(), _defaultDocumentType(), _documentTypeFreq() -{ -} +{ } + +DocumentTypeMapping::~DocumentTypeMapping() { } namespace { diff --git a/vsm/src/vespa/vsm/common/documenttypemapping.h b/vsm/src/vespa/vsm/common/documenttypemapping.h index 9c94469c58b..5c869f61476 100644 --- a/vsm/src/vespa/vsm/common/documenttypemapping.h +++ b/vsm/src/vespa/vsm/common/documenttypemapping.h @@ -12,6 +12,7 @@ class DocumentTypeMapping { public: DocumentTypeMapping(); + ~DocumentTypeMapping(); /** * Prepares the given document by sharing the field info map @@ -41,10 +42,8 @@ private: void buildFieldMap(const document::DocumentType *docType, const StringFieldIdTMapT & fieldList, const vespalib::string & typeId); - typedef vespalib::hash_map<vespalib::string, FieldPathMapT> - FieldPathMapMapT; - typedef std::multimap<size_t, const document::DocumentType *> - DocumentTypeUsage; + typedef vespalib::hash_map<vespalib::string, FieldPathMapT> FieldPathMapMapT; + typedef std::multimap<size_t, const document::DocumentType *> DocumentTypeUsage; FieldPathMapMapT _fieldMap; vespalib::string _defaultDocumentTypeName; const document::DocumentType *_defaultDocumentType; diff --git a/vsm/src/vespa/vsm/common/fieldmodifier.cpp b/vsm/src/vespa/vsm/common/fieldmodifier.cpp index 21d5cd4b58a..6a619bfc533 100644 --- a/vsm/src/vespa/vsm/common/fieldmodifier.cpp +++ b/vsm/src/vespa/vsm/common/fieldmodifier.cpp @@ -1,13 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vsm/common/fieldmodifier.h> -namespace vsm -{ +#include "fieldmodifier.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + +namespace vsm { + FieldModifierMap::FieldModifierMap() : _map() -{ -} +{ } + +FieldModifierMap::~FieldModifierMap() { } FieldModifier * FieldModifierMap::getModifier(FieldIdT fId) const @@ -19,6 +21,4 @@ FieldModifierMap::getModifier(FieldIdT fId) const return itr->second.get(); } - } - diff --git a/vsm/src/vespa/vsm/common/fieldmodifier.h b/vsm/src/vespa/vsm/common/fieldmodifier.h index f6b8d949e55..e38166bd680 100644 --- a/vsm/src/vespa/vsm/common/fieldmodifier.h +++ b/vsm/src/vespa/vsm/common/fieldmodifier.h @@ -5,8 +5,7 @@ #include <vespa/vespalib/util/linkedptr.h> #include <vespa/vsm/common/document.h> -namespace vsm -{ +namespace vsm { /** * Interface for classes that want to modify a field value. @@ -43,6 +42,7 @@ private: public: FieldModifierMap(); + ~FieldModifierMap(); FieldModifierMapT & map() { return _map; } const FieldModifierMapT & map() const { return _map; } diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp index f024cbfb0bc..e2913e01967 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp +++ b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vsm/vsm/fieldsearchspec.h> + +#include "fieldsearchspec.h" #include <vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h> #include <vespa/vsm/searcher/utf8strchrfieldsearcher.h> #include <vespa/vsm/searcher/utf8substringsearcher.h> @@ -151,8 +151,7 @@ FieldSearchSpecMap::FieldSearchSpecMap() : _specMap(), _documentTypeMap(), _nameIdMap() -{ -} +{ } namespace { const vespalib::string _G_empty(""); diff --git a/vsm/src/vespa/vsm/vsm/snippetmodifier.cpp b/vsm/src/vespa/vsm/vsm/snippetmodifier.cpp index bd1904b48ad..ea3e1dc8e91 100644 --- a/vsm/src/vespa/vsm/vsm/snippetmodifier.cpp +++ b/vsm/src/vespa/vsm/vsm/snippetmodifier.cpp @@ -1,10 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "snippetmodifier.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + #include <vespa/log/log.h> LOG_SETUP(".vsm.snippetmodifier"); -#include <vespa/vsm/vsm/snippetmodifier.h> - using namespace document; using search::QueryTerm; using search::QueryTermList; |