diff options
254 files changed, 3512 insertions, 3429 deletions
diff --git a/config/src/vespa/config/common/configmanager.cpp b/config/src/vespa/config/common/configmanager.cpp index a73e8caf206..f15880fea01 100644 --- a/config/src/vespa/config/common/configmanager.cpp +++ b/config/src/vespa/config/common/configmanager.cpp @@ -3,9 +3,8 @@ #include "exceptions.h" #include "configholder.h" #include <vespa/vespalib/util/atomic.h> -#include <memory> #include <thread> -#include <chrono> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".config.common.configmanager"); diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp index ccc56fd51b9..b089af71782 100644 --- a/config/src/vespa/config/frt/protocol.cpp +++ b/config/src/vespa/config/frt/protocol.cpp @@ -3,6 +3,7 @@ #include <lz4.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".config.frt.protocol"); diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp index c62452a6415..75dcd3d3f65 100644 --- a/config/src/vespa/config/set/configsetsource.cpp +++ b/config/src/vespa/config/set/configsetsource.cpp @@ -2,9 +2,8 @@ #include "configsetsource.h" #include <vespa/config/print/asciiconfigwriter.h> -#include <vespa/config/common/misc.h> #include <vespa/config/common/exceptions.h> -#include <vespa/vespalib/stllike/asciistream.h> + #include <vespa/log/log.h> LOG_SETUP(".config.set.configsetsource"); @@ -48,7 +47,7 @@ ConfigSetSource::getConfig() void ConfigSetSource::reload(int64_t generation) { - LOG(debug, "Running update with generation(%" PRId64 ")", generation); + LOG(debug, "Running update with generation(%zd)", generation); _generation = generation; } diff --git a/document/src/tests/base/documentid_test.cpp b/document/src/tests/base/documentid_test.cpp index adda311e01a..7f17685f84e 100644 --- a/document/src/tests/base/documentid_test.cpp +++ b/document/src/tests/base/documentid_test.cpp @@ -2,6 +2,7 @@ // Unit tests for documentid. #include <vespa/document/base/documentid.h> +#include <vespa/document/base/idstringexception.h> #include <vespa/vespalib/testkit/testapp.h> using namespace document; diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 6957634cbde..7fb7703c04a 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -6,6 +6,7 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/select/parser.h> #include <vespa/document/select/visitor.h> diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index 3bbd5178055..a9e7b1897b2 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -5,6 +5,7 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/document/datatype/annotationreferencedatatype.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> @@ -19,6 +20,8 @@ using namespace document::config_builder; namespace document { +using namespace fieldvalue; + struct DocumentTest : public CppUnit::TestFixture { void testTraversing(); void testFieldPath(); @@ -105,7 +108,7 @@ void DocumentTest::testFieldPath() } } -class Handler : public FieldValue::IteratorHandler { +class Handler : public fieldvalue::IteratorHandler { public: Handler(); ~Handler(); @@ -185,7 +188,7 @@ void DocumentTest::testTraversing() std::string("<P<P<PP[PPP][<PP><PP>]>>>")); } -class VariableIteratorHandler : public FieldValue::IteratorHandler { +class VariableIteratorHandler : public IteratorHandler { public: VariableIteratorHandler(); ~VariableIteratorHandler(); @@ -268,7 +271,7 @@ DocumentTest::testVariables() } -class ModifyIteratorHandler : public FieldValue::IteratorHandler { +class ModifyIteratorHandler : public IteratorHandler { public: ModificationStatus doModify(FieldValue& fv) override { StringFieldValue* sfv = dynamic_cast<StringFieldValue*>(&fv); diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index baa777ceca1..e3cab489aab 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -24,6 +24,7 @@ #include <vespa/eval/tensor/tensor_factory.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/exception.h> using namespace document::config_builder; using vespalib::tensor::Tensor; diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 4a23f828921..4311e1059c2 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocman.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/select/node.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vdstestlib/cppunit/macros.h> @@ -21,6 +22,8 @@ using namespace document::config_builder; namespace document { +using namespace fieldvalue; + struct FieldPathUpdateTestCase : public CppUnit::TestFixture { DocumentTypeRepo::SP _repo; DocumentType _foobar_type; @@ -259,7 +262,7 @@ void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { struct TestFieldPathUpdate : FieldPathUpdate { - struct TestIteratorHandler : FieldValue::IteratorHandler + struct TestIteratorHandler : fieldvalue::IteratorHandler { TestIteratorHandler(std::string& str) : _str(str) {} @@ -288,9 +291,9 @@ struct TestFieldPathUpdate : FieldPathUpdate TestFieldPathUpdate(const TestFieldPathUpdate& other); - std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override + std::unique_ptr<IteratorHandler> getIteratorHandler(Document&) const override { - return std::unique_ptr<FieldValue::IteratorHandler>( + return std::unique_ptr<IteratorHandler>( new TestIteratorHandler(_str)); } diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp index d837f512637..4bdecf38996 100644 --- a/document/src/vespa/document/base/idstring.cpp +++ b/document/src/vespa/document/base/idstring.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "idstring.h" +#include "idstringexception.h" #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/md5.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/document/src/vespa/document/base/idstring.h b/document/src/vespa/document/base/idstring.h index f1a96974cc9..5e09b9060b8 100644 --- a/document/src/vespa/document/base/idstring.h +++ b/document/src/vespa/document/base/idstring.h @@ -1,29 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file idstring.h - * - * Contains the various URI schemes accepted in document identifiers. - */ #pragma once -#include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/objects/cloneable.h> #include <vespa/vespalib/stllike/string.h> +#include <cstdint> namespace document { /** - * \class document::IdParseException - * \ingroup base - * - * \brief Exception used to indicate failure to parse a %document identifier - * URI. - */ -VESPA_DEFINE_EXCEPTION(IdParseException, vespalib::Exception); - -/** * \class document::IdString * \ingroup base * diff --git a/document/src/vespa/document/base/idstringexception.h b/document/src/vespa/document/base/idstringexception.h new file mode 100644 index 00000000000..922876a8d6e --- /dev/null +++ b/document/src/vespa/document/base/idstringexception.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/exception.h> + +namespace document { + +/** + * \class document::IdParseException + * \ingroup base + * + * \brief Exception used to indicate failure to parse a %document identifier + * URI. + */ + +VESPA_DEFINE_EXCEPTION(IdParseException, vespalib::Exception); + +} diff --git a/document/src/vespa/document/bucket/CMakeLists.txt b/document/src/vespa/document/bucket/CMakeLists.txt index 757884f0050..7dfeaa1920f 100644 --- a/document/src/vespa/document/bucket/CMakeLists.txt +++ b/document/src/vespa/document/bucket/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(document_bucket OBJECT bucketdistribution.cpp bucketid.cpp bucketidfactory.cpp + bucketidlist.cpp bucketselector.cpp bucketspace.cpp DEPENDS diff --git a/document/src/vespa/document/bucket/bucketdistribution.h b/document/src/vespa/document/bucket/bucketdistribution.h index cc7232c81e6..aabc615bdda 100644 --- a/document/src/vespa/document/bucket/bucketdistribution.h +++ b/document/src/vespa/document/bucket/bucketdistribution.h @@ -12,8 +12,8 @@ */ #pragma once +#include "bucketid.h" #include <vector> -#include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/sync.h> namespace document { diff --git a/document/src/vespa/document/bucket/bucketid.h b/document/src/vespa/document/bucket/bucketid.h index 63b513dd5a2..79da6c2d11b 100644 --- a/document/src/vespa/document/bucket/bucketid.h +++ b/document/src/vespa/document/bucket/bucketid.h @@ -22,7 +22,6 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/array.h> namespace vespalib { class nbostream; @@ -33,6 +32,8 @@ namespace document { extern const unsigned char reverseBitTable[256]; +namespace bucket { class BucketIdList; } + class BucketId { public: @@ -47,7 +48,7 @@ public: * typedef when needed we can alter this later with less code changes. */ using Type = uint64_t; - using List = vespalib::Array<BucketId>; + using List = bucket::BucketIdList; /** Create an initially unset bucket id. */ BucketId() : _id(0) {} /** Create a bucket id with the given raw unchecked content. */ diff --git a/document/src/vespa/document/bucket/bucketidlist.cpp b/document/src/vespa/document/bucket/bucketidlist.cpp new file mode 100644 index 00000000000..6c9f7250b8e --- /dev/null +++ b/document/src/vespa/document/bucket/bucketidlist.cpp @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bucketidlist.h" + +namespace document::bucket { + +BucketIdList::BucketIdList() { } +BucketIdList::BucketIdList(const BucketIdList & rhs) = default; +BucketIdList & BucketIdList::operator = (const BucketIdList &) = default; +BucketIdList::~BucketIdList() { } + +} diff --git a/document/src/vespa/document/bucket/bucketidlist.h b/document/src/vespa/document/bucket/bucketidlist.h new file mode 100644 index 00000000000..f53301d1b44 --- /dev/null +++ b/document/src/vespa/document/bucket/bucketidlist.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "bucketid.h" +#include <vespa/vespalib/util/array.h> + +namespace document::bucket { + +using BucketIdListT = vespalib::Array<BucketId>; + +class BucketIdList : public BucketIdListT { +public: + BucketIdList(); + BucketIdList(BucketIdList && rhs) = default; + BucketIdList & operator = (BucketIdList &&) = default; + BucketIdList(const BucketIdList & rhs); + BucketIdList & operator = (const BucketIdList &); + ~BucketIdList(); +}; + +} diff --git a/document/src/vespa/document/bucket/bucketselector.cpp b/document/src/vespa/document/bucket/bucketselector.cpp index b41b1c284a6..7eb3821e283 100644 --- a/document/src/vespa/document/bucket/bucketselector.cpp +++ b/document/src/vespa/document/bucket/bucketselector.cpp @@ -4,7 +4,7 @@ #include "bucketidfactory.h" #include <vespa/document/base/documentid.h> #include <vespa/document/select/node.h> -#include <vespa/document/select/valuenode.h> +#include <vespa/document/select/valuenodes.h> #include <vespa/document/select/visitor.h> #include <vespa/document/select/branch.h> #include <vespa/document/select/compare.h> diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp index f6bd1fc5fa6..954bd9c742f 100644 --- a/document/src/vespa/document/datatype/structdatatype.cpp +++ b/document/src/vespa/document/datatype/structdatatype.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <iomanip> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".document.datatype.struct"); diff --git a/document/src/vespa/document/fieldvalue/CMakeLists.txt b/document/src/vespa/document/fieldvalue/CMakeLists.txt index 8c76db54d5f..18ea4fc9368 100644 --- a/document/src/vespa/document/fieldvalue/CMakeLists.txt +++ b/document/src/vespa/document/fieldvalue/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_add_library(document_fieldvalues OBJECT fieldvalue.cpp floatfieldvalue.cpp intfieldvalue.cpp + iteratorhandler.cpp literalfieldvalue.cpp longfieldvalue.cpp mapfieldvalue.cpp @@ -22,6 +23,7 @@ vespa_add_library(document_fieldvalues OBJECT structfieldvalue.cpp structuredfieldvalue.cpp tensorfieldvalue.cpp + variablemap.cpp weightedsetfieldvalue.cpp referencefieldvalue.cpp DEPENDS diff --git a/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp index 9863ee1c18a..cd6d43d2f4d 100644 --- a/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "annotationreferencefieldvalue.h" +#include <vespa/vespalib/util/xmlstream.h> #include <ostream> using std::ostream; @@ -23,8 +24,7 @@ int AnnotationReferenceFieldValue::compare(const FieldValue &other) const { return (getDataType()->getId() - other.getDataType()->getId()); } -void AnnotationReferenceFieldValue::print(ostream &out, bool, - const string &) const { +void AnnotationReferenceFieldValue::print(ostream &out, bool, const string &) const { out << "AnnotationReferenceFieldValue(n)"; } diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp index 8c05c25e651..01df81db69b 100644 --- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp @@ -3,15 +3,23 @@ #include "intfieldvalue.h" #include "stringfieldvalue.h" #include "predicatefieldvalue.h" +#include "iteratorhandler.h" #include <vespa/document/util/serializableexceptions.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/polymorphicarrays.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> LOG_SETUP(".document.fieldvalue.array"); +using namespace vespalib::xml; + namespace document { using vespalib::IllegalArgumentException; +using fieldvalue::IndexValue; +using fieldvalue::ModificationStatus; +using fieldvalue::IteratorHandler; +using fieldvalue::VariableMap; IMPLEMENT_IDENTIFIABLE_ABSTRACT(ArrayFieldValue, CollectionFieldValue); @@ -24,7 +32,7 @@ ArrayFieldValue::ArrayFieldValue(const DataType &type) "Cannot generate an array value with non-array type " + type.toString() + ".", VESPA_STRLOC); } - _array = createArray(getNestedType()); + _array.reset(static_cast<IArray *>(createArray(getNestedType()).release())); } ArrayFieldValue::ArrayFieldValue(const ArrayFieldValue& other) @@ -176,14 +184,13 @@ ArrayFieldValue::hasChanged() const return false; } -FieldValue::IteratorHandler::ModificationStatus +fieldvalue::ModificationStatus ArrayFieldValue::iterateSubset(int startPos, int endPos, const vespalib::stringref & variable, PathRange nested, - IteratorHandler& handler) const + fieldvalue::IteratorHandler& handler) const { - FieldValue::IteratorHandler::ModificationStatus - retVal = FieldValue::IteratorHandler::NOT_MODIFIED; + fieldvalue::ModificationStatus retVal = ModificationStatus::NOT_MODIFIED; LOG(spam, "iterateSubset(start=%d, end=%d, variable='%s')", startPos, endPos, variable.c_str()); @@ -192,16 +199,15 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos, for (int i = startPos; i <= endPos && i < static_cast<int>(_array->size()); ++i) { if (!variable.empty()) { - handler.getVariables()[variable] = IteratorHandler::IndexValue(i); + handler.getVariables()[variable] = IndexValue(i); } - FieldValue::IteratorHandler::ModificationStatus - status = array()[i].iterateNested(nested, handler); + ModificationStatus status = array()[i].iterateNested(nested, handler); - if (status == FieldValue::IteratorHandler::REMOVED) { + if (status == ModificationStatus::REMOVED) { indicesToRemove.push_back(i); - retVal = FieldValue::IteratorHandler::MODIFIED; - } else if (status == FieldValue::IteratorHandler::MODIFIED) { + retVal = ModificationStatus::MODIFIED; + } else if (status == ModificationStatus::MODIFIED) { retVal = status; } } @@ -219,7 +225,7 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos, return retVal; } -FieldValue::IteratorHandler::ModificationStatus +fieldvalue::ModificationStatus ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { IteratorHandler::CollectionScope autoScope(handler, *this); @@ -235,7 +241,7 @@ ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) co case FieldPathEntry::VARIABLE: { LOG(spam, "VARIABLE"); - IteratorHandler::VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName()); + VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName()); if (iter != handler.getVariables().end()) { int idx = iter->second.index; @@ -259,17 +265,17 @@ ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) co } return iterateSubset(0, static_cast<int>(_array->size()) - 1, "", nested, handler); } else { - IteratorHandler::ModificationStatus status = handler.modify(const_cast<ArrayFieldValue&>(*this)); + fieldvalue::ModificationStatus status = handler.modify(const_cast<ArrayFieldValue&>(*this)); - if (status == FieldValue::IteratorHandler::REMOVED) { + if (status == fieldvalue::REMOVED) { return status; } if (handler.handleComplex(*this)) { - if (iterateSubset(0, static_cast<int>(_array->size()) - 1, "", - nested, handler) != FieldValue::IteratorHandler::NOT_MODIFIED) + if (iterateSubset(0, static_cast<int>(_array->size()) - 1, "", nested, handler) + != ModificationStatus::NOT_MODIFIED) { - status = FieldValue::IteratorHandler::MODIFIED; + status = ModificationStatus::MODIFIED; } } diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h index 33e07e18588..d61acb8605c 100644 --- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h @@ -14,25 +14,27 @@ #include "collectionfieldvalue.h" #include <vespa/document/datatype/arraydatatype.h> +#include <vespa/vespalib/util/polymorphicarray.h> namespace document { class ArrayFieldValue : public CollectionFieldValue { private: - IArray::UP _array; + using IArray = vespalib::IArrayT<FieldValue>; + std::unique_ptr<IArray> _array; bool addValue(const FieldValue&) override; bool containsValue(const FieldValue& val) const override; bool removeValue(const FieldValue& val) override; - IteratorHandler::ModificationStatus iterateSubset( + fieldvalue::ModificationStatus iterateSubset( int startPos, int endPos, const vespalib::stringref & variable, PathRange nested, - IteratorHandler& handler) const; - IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const override; + fieldvalue::IteratorHandler& handler) const; + fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override; public: - typedef IArray::const_iterator const_iterator; - typedef IArray::iterator iterator; - typedef std::unique_ptr<ArrayFieldValue> UP; + using const_iterator = IArray::const_iterator; + using iterator = IArray::iterator; + using UP = std::unique_ptr<ArrayFieldValue>; /** * @param arrayType Type of the array. Must be an ArrayDataType, but does diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index 29f487df989..a8336519e16 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -10,13 +10,14 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/util/bytebuffer.h> - +#include <vespa/vespalib/util/xmlstream.h> #include <sstream> using vespalib::nbostream; using vespalib::make_string; using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; +using namespace vespalib::xml; namespace document { namespace { diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp index 3279eb77a64..c137811514c 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp @@ -10,17 +10,22 @@ #include "doublefieldvalue.h" #include "bytefieldvalue.h" #include "predicatefieldvalue.h" +#include "iteratorhandler.h" #include <vespa/document/util/bytebuffer.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/polymorphicarrays.h> +#include <vespa/vespalib/util/xmlstream.h> #include <sstream> using vespalib::FieldBase; using vespalib::nbostream; - +using namespace vespalib::xml; namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldValue, vespalib::Identifiable); void FieldValue::serialize(nbostream &stream) const { @@ -183,13 +188,13 @@ FieldValue::onGetNestedFieldValue(PathRange nested) const return FieldValue::UP(); } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus FieldValue::iterateNested(PathRange nested, IteratorHandler & handler) const { return onIterateNested(nested, handler); } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus FieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { if (nested.atEnd()) { @@ -200,78 +205,6 @@ FieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const } } -FieldValue::IteratorHandler::~IteratorHandler() { } - -bool -FieldValue::IteratorHandler::IndexValue::operator==(const FieldValue::IteratorHandler::IndexValue& other) const { - if (key.get() != NULL) { - if (other.key.get() != NULL && *key == *other.key) { - return true; - } - return false; - } - - return index == other.index; -} - -FieldValue::IteratorHandler::IndexValue::IndexValue(const FieldValue& key_) - : index(-1), - key(FieldValue::CP(key_.clone())) -{ } - -FieldValue::IteratorHandler::IndexValue::~IndexValue() { } - -vespalib::string -FieldValue::IteratorHandler::IndexValue::toString() const { - if (key.get() != NULL) { - return key->toString(); - } else { - return vespalib::make_string("%d", index); - } -} - -void -FieldValue::IteratorHandler::handlePrimitive(uint32_t fid, const FieldValue & fv) { - onPrimitive(fid, Content(fv, getWeight())); -} -bool -FieldValue::IteratorHandler::handleComplex(const FieldValue & fv) { - return onComplex(Content(fv, getWeight())); -} -void -FieldValue::IteratorHandler::handleCollectionStart(const FieldValue & fv) { - onCollectionStart(Content(fv, getWeight())); -} -void -FieldValue::IteratorHandler::handleCollectionEnd(const FieldValue & fv) { - onCollectionEnd(Content(fv, getWeight())); -} -void -FieldValue::IteratorHandler::handleStructStart(const FieldValue & fv) { - onStructStart(Content(fv, getWeight())); -} -void -FieldValue::IteratorHandler::handleStructEnd(const FieldValue & fv) { - onStructEnd(Content(fv, getWeight())); -} - -void -FieldValue::IteratorHandler::onPrimitive(uint32_t fid, const Content & fv) { - (void) fid; - (void) fv; -} - -std::string -FieldValue::IteratorHandler::toString(const VariableMap& vars) { - std::ostringstream out; - out << "[ "; - for (const auto & entry : vars) { - out << entry.first << "=" << entry.second.toString() << " "; - } - out << "]"; - return out.str(); -} - std::string FieldValue::toString(bool verbose, const std::string& indent) const { @@ -295,26 +228,26 @@ private: }; } -FieldValue::IArray::UP +std::unique_ptr<vespalib::IArrayBase> FieldValue::createArray(const DataType & baseType) { switch(baseType.getId()) { case DataType::T_INT: - return IArray::UP(new PrimitiveArrayT<IntFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<IntFieldValue, FieldValue>>(); case DataType::T_FLOAT: - return IArray::UP(new PrimitiveArrayT<FloatFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<FloatFieldValue, FieldValue>>(); case DataType::T_STRING: - return IArray::UP(new PrimitiveArrayT<StringFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<StringFieldValue, FieldValue>>(); case DataType::T_RAW: - return IArray::UP(new PrimitiveArrayT<RawFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<RawFieldValue, FieldValue>>(); case DataType::T_LONG: - return IArray::UP(new PrimitiveArrayT<LongFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<LongFieldValue, FieldValue>>(); case DataType::T_DOUBLE: - return IArray::UP(new PrimitiveArrayT<DoubleFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<DoubleFieldValue, FieldValue>>(); case DataType::T_BYTE: - return IArray::UP(new PrimitiveArrayT<ByteFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<ByteFieldValue, FieldValue>>(); default: - return IArray::UP(new ComplexArrayT<FieldValue>(FieldValueFactory::UP(new FieldValueFactory(DataType::UP(baseType.clone()))))); + return std::make_unique<ComplexArrayT<FieldValue>>(std::make_unique<FieldValueFactory>(DataType::UP(baseType.clone()))); } } diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h index 81e207bc31a..a96628514c3 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.h +++ b/document/src/vespa/document/fieldvalue/fieldvalue.h @@ -12,11 +12,12 @@ #pragma once #include "fieldvaluevisitor.h" -#include <vespa/document/datatype/datatype.h> +#include "modificationstatus.h" #include <vespa/document/util/xmlserializable.h> -#include <vespa/vespalib/util/polymorphicarrays.h> +#include <vespa/document/base/fieldpath.h> #include <vespa/vespalib/objects/cloneable.h> -#include <map> +#include <vespa/vespalib/objects/identifiable.h> +#include <vespa/vespalib/util/polymorphicarraybase.h> namespace vespalib { class nbostream; @@ -24,15 +25,19 @@ namespace vespalib { namespace document { +namespace fieldvalue { + class IteratorHandler; +} + class ByteBuffer; +class DataType; class FieldValue : public vespalib::Identifiable { protected: FieldValue(const FieldValue&) = default; FieldValue& operator=(const FieldValue&) = default; - using IArray = vespalib::IArrayT<FieldValue>; - static IArray::UP createArray(const DataType & baseType); + static std::unique_ptr<vespalib::IArrayBase> createArray(const DataType & baseType); public: using PathRange = FieldPath::Range<FieldPath::const_iterator>; @@ -40,110 +45,6 @@ public: using SP = std::shared_ptr<FieldValue>; using CP = vespalib::CloneablePtr<FieldValue>; - class IteratorHandler { - public: - class CollectionScope { - public: - CollectionScope(IteratorHandler& handler, const FieldValue& value) - : _handler(handler), _value(value) - { - _handler.handleCollectionStart(_value); - } - - ~CollectionScope() { - _handler.handleCollectionEnd(_value); - } - private: - IteratorHandler& _handler; - const FieldValue& _value; - }; - - class StructScope { - public: - StructScope(IteratorHandler& handler, const FieldValue& value) - : _handler(handler), _value(value) - { - _handler.handleStructStart(_value); - } - - ~StructScope() { - _handler.handleStructEnd(_value); - } - private: - IteratorHandler& _handler; - const FieldValue& _value; - }; - - class IndexValue { - public: - IndexValue() : index(-1), key() {} - IndexValue(int index_) : index(index_), key() {} - IndexValue(const FieldValue& key_); - ~IndexValue(); - - vespalib::string toString() const; - - bool operator==(const IndexValue& other) const; - - int index; // For array - FieldValue::CP key; // For map/wset - }; - - enum ModificationStatus { - MODIFIED, - REMOVED, - NOT_MODIFIED - }; - - typedef std::map<vespalib::string, IndexValue> VariableMap; - protected: - class Content { - public: - Content(const FieldValue & fv, int weight=1) : _fieldValue(fv), _weight(weight) { } - int getWeight() const { return _weight; } - const FieldValue & getValue() const { return _fieldValue; } - private: - const FieldValue & _fieldValue; - int _weight; - }; - IteratorHandler() : _weight(1) { } - public: - virtual ~IteratorHandler(); - - void handlePrimitive(uint32_t fid, const FieldValue & fv); - - /** - Handles a complex type (struct/array/map etc) that is at the end of the - field path. - @return Return true if you want to recurse into the members. - */ - bool handleComplex(const FieldValue& fv); - void handleCollectionStart(const FieldValue & fv); - void handleCollectionEnd(const FieldValue & fv); - void handleStructStart(const FieldValue & fv); - void handleStructEnd(const FieldValue & fv); - void setWeight(int weight) { _weight = weight; } - ModificationStatus modify(FieldValue& fv) { return doModify(fv); } - - VariableMap& getVariables() { return _variables; } - void setVariables(const VariableMap& vars) { _variables = vars; } - static std::string toString(const VariableMap& vars); - virtual bool createMissingPath() const { return false; } - private: - virtual bool onComplex(const Content& fv) { (void) fv; return true; } - virtual void onPrimitive(uint32_t fid, const Content & fv); - virtual void onCollectionStart(const Content & fv) { (void) fv; } - virtual void onCollectionEnd(const Content & fv) { (void) fv; } - virtual void onStructStart(const Content & fv) { (void) fv; } - virtual void onStructEnd(const Content & fv) { (void) fv; } - virtual ModificationStatus doModify(FieldValue&) { return NOT_MODIFIED; }; - - // Scratchpad to store pass on weight. - int getWeight() const { return _weight; } - int _weight; - VariableMap _variables; - }; - DECLARE_IDENTIFIABLE_ABSTRACT(FieldValue); FieldValue() {} @@ -268,9 +169,9 @@ public: * invocations of the before mentioned methods and the additional * onPrimitive. */ - IteratorHandler::ModificationStatus iterateNested(PathRange nested, IteratorHandler & handler) const; + fieldvalue::ModificationStatus iterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const; - IteratorHandler::ModificationStatus iterateNested(const FieldPath& fieldPath, IteratorHandler& handler) const { + fieldvalue::ModificationStatus iterateNested(const FieldPath& fieldPath, fieldvalue::IteratorHandler& handler) const { return iterateNested(fieldPath.begin(), fieldPath.end(), handler); } @@ -284,12 +185,12 @@ public: virtual void printXml(XmlOutputStream& out) const = 0; private: - IteratorHandler::ModificationStatus - iterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, IteratorHandler & handler) const { + fieldvalue::ModificationStatus + iterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, fieldvalue::IteratorHandler & handler) const { return iterateNested(PathRange(start, end), handler); } virtual FieldValue::UP onGetNestedFieldValue(PathRange nested) const; - virtual IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const; + virtual fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const; }; std::ostream& operator<<(std::ostream& out, const FieldValue & p); diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.cpp b/document/src/vespa/document/fieldvalue/iteratorhandler.cpp new file mode 100644 index 00000000000..394979f31cb --- /dev/null +++ b/document/src/vespa/document/fieldvalue/iteratorhandler.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include "iteratorhandler.h" + +namespace document::fieldvalue { + +IteratorHandler::~IteratorHandler() { } + + +void +IteratorHandler::handlePrimitive(uint32_t fid, const FieldValue & fv) { + onPrimitive(fid, Content(fv, getWeight())); +} +bool +IteratorHandler::handleComplex(const FieldValue & fv) { + return onComplex(Content(fv, getWeight())); +} +void +IteratorHandler::handleCollectionStart(const FieldValue & fv) { + onCollectionStart(Content(fv, getWeight())); +} +void +IteratorHandler::handleCollectionEnd(const FieldValue & fv) { + onCollectionEnd(Content(fv, getWeight())); +} +void +IteratorHandler::handleStructStart(const FieldValue & fv) { + onStructStart(Content(fv, getWeight())); +} +void +IteratorHandler::handleStructEnd(const FieldValue & fv) { + onStructEnd(Content(fv, getWeight())); +} + +void +IteratorHandler::onPrimitive(uint32_t fid, const Content & fv) { + (void) fid; + (void) fv; +} + +} + diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.h b/document/src/vespa/document/fieldvalue/iteratorhandler.h new file mode 100644 index 00000000000..8dc98b2107d --- /dev/null +++ b/document/src/vespa/document/fieldvalue/iteratorhandler.h @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "variablemap.h" +#include "modificationstatus.h" + +namespace document::fieldvalue { + +class IteratorHandler { +public: + class CollectionScope { + public: + CollectionScope(IteratorHandler &handler, const FieldValue &value) + : _handler(handler), _value(value) { + _handler.handleCollectionStart(_value); + } + + ~CollectionScope() { + _handler.handleCollectionEnd(_value); + } + + private: + IteratorHandler &_handler; + const FieldValue &_value; + }; + + class StructScope { + public: + StructScope(IteratorHandler &handler, const FieldValue &value) + : _handler(handler), _value(value) { + _handler.handleStructStart(_value); + } + + ~StructScope() { + _handler.handleStructEnd(_value); + } + + private: + IteratorHandler &_handler; + const FieldValue &_value; + }; + +protected: + class Content { + public: + Content(const FieldValue &fv, int weight = 1) : _fieldValue(fv), _weight(weight) {} + + int getWeight() const { return _weight; } + + const FieldValue &getValue() const { return _fieldValue; } + + private: + const FieldValue &_fieldValue; + int _weight; + }; + + IteratorHandler() : _weight(1) {} + +public: + virtual ~IteratorHandler(); + void handlePrimitive(uint32_t fid, const FieldValue &fv); + + /** + Handles a complex type (struct/array/map etc) that is at the end of the + field path. + @return Return true if you want to recurse into the members. + */ + bool handleComplex(const FieldValue &fv); + void handleCollectionStart(const FieldValue &fv); + void handleCollectionEnd(const FieldValue &fv); + void handleStructStart(const FieldValue &fv); + void handleStructEnd(const FieldValue &fv); + void setWeight(int weight) { _weight = weight; } + ModificationStatus modify(FieldValue &fv) { return doModify(fv); } + fieldvalue::VariableMap &getVariables() { return _variables; } + void setVariables(const fieldvalue::VariableMap &vars) { _variables = vars; } + virtual bool createMissingPath() const { return false; } +private: + virtual bool onComplex(const Content &fv) { + (void) fv; + return true; + } + + virtual void onPrimitive(uint32_t fid, const Content &fv); + virtual void onCollectionStart(const Content &fv) { (void) fv; } + virtual void onCollectionEnd(const Content &fv) { (void) fv; } + virtual void onStructStart(const Content &fv) { (void) fv; } + virtual void onStructEnd(const Content &fv) { (void) fv; } + virtual ModificationStatus doModify(FieldValue &) { return NOT_MODIFIED; }; + + // Scratchpad to store pass on weight. + int getWeight() const { return _weight; } + + int _weight; + fieldvalue::VariableMap _variables; +}; + +} diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp index 4d044219d4c..50c6c901808 100644 --- a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp @@ -1,9 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/fieldvalue/literalfieldvalue.h> +#include "literalfieldvalue.h" #include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/util/xmlstream.h> #include <sstream> +using namespace vespalib::xml; + namespace document { IMPLEMENT_IDENTIFIABLE_ABSTRACT(LiteralFieldValueB, FieldValue); diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp index 2186ff28f60..2b37c43be55 100644 --- a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp @@ -2,12 +2,16 @@ #include "mapfieldvalue.h" #include "weightedsetfieldvalue.h" +#include "iteratorhandler.h" #include <vespa/document/base/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.fieldvalue.map"); using vespalib::Identifiable; +using namespace vespalib::xml; /// \todo TODO (was warning): // Find a way to search through internal map without @@ -15,6 +19,8 @@ using vespalib::Identifiable; namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(MapFieldValue, FieldValue); namespace { @@ -31,8 +37,8 @@ const MapDataType *verifyMapType(const DataType& type) { MapFieldValue::MapFieldValue(const DataType &mapType) : FieldValue(), _type(verifyMapType(mapType)), - _keys(createArray(getMapType().getKeyType())), - _values(createArray(getMapType().getValueType())), + _keys(static_cast<IArray *>(createArray(getMapType().getKeyType()).release())), + _values(static_cast<IArray *>(createArray(getMapType().getValueType()).release())), _altered(true) { } @@ -203,8 +209,7 @@ MapFieldValue::compare(const FieldValue& other) const } void -MapFieldValue::print(std::ostream& out, bool verbose, - const std::string& indent) const +MapFieldValue::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "Map("; @@ -268,22 +273,22 @@ MapFieldValue::find(const FieldValue& key) } bool MapFieldValue::checkAndRemove(const FieldValue& key, - FieldValue::IteratorHandler::ModificationStatus status, + ModificationStatus status, bool wasModified, std::vector<const FieldValue*>& keysToRemove) const { - if (status == FieldValue::IteratorHandler::REMOVED) { + if (status == ModificationStatus::REMOVED) { LOG(spam, "will remove: %s", key.toString().c_str()); keysToRemove.push_back(&key); return true; - } else if (status == FieldValue::IteratorHandler::MODIFIED) { + } else if (status == ModificationStatus::MODIFIED) { return true; } return wasModified; } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus MapFieldValue::iterateNestedImpl(PathRange nested, IteratorHandler & handler, const FieldValue& complexFieldValue) const @@ -307,10 +312,9 @@ MapFieldValue::iterateNestedImpl(PathRange nested, wasModified, keysToRemove); } else if (handler.createMissingPath()) { LOG(spam, "creating missing path"); - FieldValue::UP val = - getMapType().getValueType().createFieldValue(); - IteratorHandler::ModificationStatus status = val->iterateNested(nested.next(), handler); - if (status == IteratorHandler::MODIFIED) { + FieldValue::UP val = getMapType().getValueType().createFieldValue(); + ModificationStatus status = val->iterateNested(nested.next(), handler); + if (status == ModificationStatus::MODIFIED) { const_cast<MapFieldValue&>(*this).put(FieldValue::UP(fpe.getLookupKey()->clone()), std::move(val)); return status; } @@ -339,8 +343,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested, case FieldPathEntry::VARIABLE: { LOG(spam, "VARIABLE"); - IteratorHandler::VariableMap::iterator - iter = handler.getVariables().find(fpe.getVariableName()); + VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName()); if (iter != handler.getVariables().end()) { LOG(spam, "variable key = %s", iter->second.key->toString().c_str()); const_iterator found = find(*iter->second.key); @@ -353,12 +356,9 @@ MapFieldValue::iterateNestedImpl(PathRange nested, PathRange next = nested.next(); for (const_iterator it(begin()), mt(end()); it != mt; it++) { LOG(spam, "key is '%s'", it->first->toString().c_str()); - handler.getVariables()[fpe.getVariableName()] - = IteratorHandler::IndexValue(*it->first); - LOG(spam, "vars at this time = %s", - FieldValue::IteratorHandler::toString(handler.getVariables()).c_str()); - wasModified = checkAndRemove(*it->first, - it->second->iterateNested(next, handler), + handler.getVariables()[fpe.getVariableName()] = IndexValue(*it->first); + LOG(spam, "vars at this time = %s", handler.getVariables().toString().c_str()); + wasModified = checkAndRemove(*it->first, it->second->iterateNested(next, handler), wasModified, keysToRemove); } handler.getVariables().erase(fpe.getVariableName()); @@ -371,8 +371,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested, if (isWSet) { handler.setWeight(static_cast<const IntFieldValue &>(*it->second).getValue()); } - wasModified = checkAndRemove(*it->first, - it->first->iterateNested(nested, handler), + wasModified = checkAndRemove(*it->first, it->first->iterateNested(nested, handler), wasModified, keysToRemove); // Don't iterate over values /*wasModified = checkAndRemove(*it->second, @@ -383,13 +382,12 @@ MapFieldValue::iterateNestedImpl(PathRange nested, } } else { LOG(spam, "at end of field path"); - IteratorHandler::ModificationStatus - status = handler.modify(const_cast<FieldValue&>(complexFieldValue)); + ModificationStatus status = handler.modify(const_cast<FieldValue&>(complexFieldValue)); - if (status == IteratorHandler::REMOVED) { + if (status == ModificationStatus::REMOVED) { LOG(spam, "status = REMOVED"); return status; - } else if (status == IteratorHandler::MODIFIED) { + } else if (status == ModificationStatus::MODIFIED) { LOG(spam, "status = MODIFIED"); wasModified = true; } @@ -419,10 +417,10 @@ MapFieldValue::iterateNestedImpl(PathRange nested, LOG(spam, "erasing map entry with key %s", (*i)->toString().c_str()); const_cast<MapFieldValue&>(*this).erase(**i); } - return wasModified ? IteratorHandler::MODIFIED : IteratorHandler::NOT_MODIFIED; + return wasModified ? ModificationStatus::MODIFIED : ModificationStatus::NOT_MODIFIED; } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus MapFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { LOG(spam, "iterating over MapFieldValue"); diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.h b/document/src/vespa/document/fieldvalue/mapfieldvalue.h index 8b480cbb7f9..edc20fea777 100644 --- a/document/src/vespa/document/fieldvalue/mapfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.h @@ -26,10 +26,10 @@ private: virtual bool containsValue(const FieldValue& fv) const { return contains(fv); } virtual bool removeValue(const FieldValue& fv) { return erase(fv); } bool checkAndRemove(const FieldValue& key, - FieldValue::IteratorHandler::ModificationStatus status, + fieldvalue::ModificationStatus status, bool wasModified, std::vector<const FieldValue*>& keysToRemove) const; - IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const override; + fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override; // Utility method to avoid constant explicit casting const MapDataType& getMapType() const { return *_type; } @@ -118,8 +118,8 @@ public: void reserve(size_t sz) { _keys->reserve(sz); _values->reserve(sz); } void resize(size_t sz) { _keys->resize(sz); _values->resize(sz); } - IteratorHandler::ModificationStatus iterateNestedImpl(PathRange nested, IteratorHandler & handler, - const FieldValue& complexFieldValue) const; + fieldvalue::ModificationStatus iterateNestedImpl(PathRange nested, fieldvalue::IteratorHandler & handler, + const FieldValue& complexFieldValue) const; // FieldValue implementation FieldValue& assign(const FieldValue&) override; diff --git a/document/src/vespa/document/fieldvalue/modificationstatus.h b/document/src/vespa/document/fieldvalue/modificationstatus.h new file mode 100644 index 00000000000..344a89338bb --- /dev/null +++ b/document/src/vespa/document/fieldvalue/modificationstatus.h @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace document::fieldvalue { + +enum ModificationStatus { + MODIFIED, + REMOVED, + NOT_MODIFIED +}; + +} diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp b/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp index 3d39544b3d0..b897a8db930 100644 --- a/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "numericfieldvalue.h" +#include <vespa/vespalib/util/xmlstream.h> namespace document { @@ -9,7 +10,7 @@ IMPLEMENT_IDENTIFIABLE_ABSTRACT(NumericFieldValueBase, FieldValue); void NumericFieldValueBase::printXml(XmlOutputStream& out) const { - out << XmlContent(getAsString()); + out << vespalib::xml::XmlContent(getAsString()); } diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp b/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp index 8e98ebc4c4e..a3394d8fcf4 100644 --- a/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp @@ -1,13 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "predicatefieldvalue.h" +#include <vespa/document/datatype/datatype.h> #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_printer.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::Slime; using vespalib::slime::SlimeInserter; +using namespace vespalib::xml; namespace document { @@ -67,4 +70,19 @@ void PredicateFieldValue::print(std::ostream& out, bool, const std::string&) con out << PredicatePrinter::print(*_slime) << "\n"; } +const DataType * +PredicateFieldValue::getDataType() const { + return DataType::PREDICATE; +} + +bool +PredicateFieldValue::hasChanged() const { + return _altered; +} + +FieldValue * +PredicateFieldValue::clone() const { + return new PredicateFieldValue(*this); +} + } // namespace document diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h index b38f9e5dcc1..74dc0d11322 100644 --- a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h +++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h @@ -2,7 +2,6 @@ #pragma once -#include <memory> #include "fieldvalue.h" namespace vespalib { @@ -25,20 +24,20 @@ public: void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); } void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); } - virtual FieldValue *clone() const override { return new PredicateFieldValue(*this); } - virtual int compare(const FieldValue &rhs) const override; + FieldValue *clone() const override; + int compare(const FieldValue &rhs) const override; - virtual void printXml(XmlOutputStream &out) const override; - virtual void print(std::ostream &out, bool verbose, const std::string &indent) const override; + void printXml(XmlOutputStream &out) const override; + void print(std::ostream &out, bool verbose, const std::string &indent) const override; - virtual const DataType *getDataType() const override { return DataType::PREDICATE; } - virtual bool hasChanged() const override { return _altered; } + const DataType *getDataType() const override; + bool hasChanged() const override; const vespalib::Slime &getSlime() const { return *_slime; } - virtual FieldValue &assign(const FieldValue &rhs) override; + FieldValue &assign(const FieldValue &rhs) override; -DECLARE_IDENTIFIABLE(PredicateFieldValue); + DECLARE_IDENTIFIABLE(PredicateFieldValue); }; } // namespace document diff --git a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp index 96701832b5e..c9950036435 100644 --- a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp @@ -3,6 +3,9 @@ #include "rawfieldvalue.h" #include "literalfieldvalue.hpp" #include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/util/xmlstream.h> + +using namespace vespalib::xml; namespace document { @@ -18,8 +21,7 @@ RawFieldValue::printXml(XmlOutputStream& out) const void RawFieldValue::print(std::ostream& out, bool, const std::string&) const { - StringUtil::printAsHex(out, _value.c_str(), - _value.size()); + StringUtil::printAsHex(out, _value.c_str(), _value.size()); } } // document diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index 74805ce3f10..cdfba106c49 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -11,6 +11,7 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> LOG_SETUP(".document.structfieldvalue"); @@ -19,6 +20,7 @@ using std::vector; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::make_string; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp index 6d77d101ff2..beef80ed2b0 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp @@ -1,13 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "structuredfieldvalue.hpp" -#include "fieldvalues.h" +#include "iteratorhandler.h" +#include "weightedsetfieldvalue.h" +#include "arrayfieldvalue.h" #include <vespa/log/log.h> LOG_SETUP(".document.fieldvalue.structured"); +using vespalib::IllegalArgumentException; + namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(StructuredFieldValue, FieldValue); StructuredFieldValue::Iterator::Iterator() @@ -59,7 +65,7 @@ void StructuredFieldValue::setFieldValue(const Field & field, const FieldValue & if (!field.getDataType().isValueType(value) && !value.getDataType()->isA(field.getDataType())) { - throw vespalib::IllegalArgumentException( + throw IllegalArgumentException( "Cannot assign value of type " + value.getDataType()->toString() + "with value : '" + value.toString() + "' to field " + field.getName().c_str() + " of type " @@ -81,7 +87,7 @@ StructuredFieldValue::onGetNestedFieldValue(PathRange nested) const return fv; } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { IteratorHandler::StructScope autoScope(handler, *this); @@ -93,38 +99,36 @@ StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handle LOG(spam, "fieldRef = %s", fpe.getFieldRef().toString().c_str()); LOG(spam, "fieldValueToSet = %s", fpe.getFieldValueToSet().toString().c_str()); if (exists) { - IteratorHandler::ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler); - if (status == IteratorHandler::REMOVED) { + ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler); + if (status == ModificationStatus::REMOVED) { LOG(spam, "field exists, status = REMOVED"); const_cast<StructuredFieldValue&>(*this).remove(fpe.getFieldRef()); - return IteratorHandler::MODIFIED; - } else if (status == IteratorHandler::MODIFIED) { + return ModificationStatus::MODIFIED; + } else if (status == ModificationStatus::MODIFIED) { LOG(spam, "field exists, status = MODIFIED"); const_cast<StructuredFieldValue&>(*this).setFieldValue(fpe.getFieldRef(), fpe.getFieldValueToSet()); - return IteratorHandler::MODIFIED; + return ModificationStatus::MODIFIED; } else { LOG(spam, "field exists, status = %u", status); return status; } } else if (handler.createMissingPath()) { LOG(spam, "createMissingPath is true"); - IteratorHandler::ModificationStatus status - = fpe.getFieldValueToSet().iterateNested(nested.next(), handler); - if (status == IteratorHandler::MODIFIED) { + ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler); + if (status == ModificationStatus::MODIFIED) { LOG(spam, "field did not exist, status = MODIFIED"); const_cast<StructuredFieldValue&>(*this).setFieldValue(fpe.getFieldRef(), fpe.getFieldValueToSet()); return status; } } LOG(spam, "field did not exist, returning NOT_MODIFIED"); - return IteratorHandler::NOT_MODIFIED; + return ModificationStatus ::NOT_MODIFIED; } else { - throw vespalib::IllegalArgumentException("Illegal field path for struct value"); + throw IllegalArgumentException("Illegal field path for struct value"); } } else { - IteratorHandler::ModificationStatus - status = handler.modify(const_cast<StructuredFieldValue&>(*this)); - if (status == IteratorHandler::REMOVED) { + ModificationStatus status = handler.modify(const_cast<StructuredFieldValue&>(*this)); + if (status == ModificationStatus::REMOVED) { LOG(spam, "field REMOVED"); return status; } @@ -133,11 +137,11 @@ StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handle LOG(spam, "handleComplex"); std::vector<const Field*> fieldsToRemove; for (const_iterator it(begin()), mt(end()); it != mt; ++it) { - IteratorHandler::ModificationStatus currStatus = getValue(it.field())->iterateNested(nested, handler); - if (currStatus == IteratorHandler::REMOVED) { + ModificationStatus currStatus = getValue(it.field())->iterateNested(nested, handler); + if (currStatus == ModificationStatus::REMOVED) { fieldsToRemove.push_back(&it.field()); - status = IteratorHandler::MODIFIED; - } else if (currStatus == IteratorHandler::MODIFIED) { + status = ModificationStatus::MODIFIED; + } else if (currStatus == ModificationStatus::MODIFIED) { status = currStatus; } } diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h index 4cccbf05de0..eb5912734c5 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h @@ -92,8 +92,8 @@ protected: virtual void setFieldValue(const Field&, FieldValue::UP value) = 0; void setFieldValue(const Field & field, const FieldValue & value); - IteratorHandler::ModificationStatus - onIterateNested(PathRange nested, IteratorHandler & handler) const override; + fieldvalue::ModificationStatus + onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override; public: DECLARE_IDENTIFIABLE_ABSTRACT(StructuredFieldValue); diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp index 03b7d50c1c1..d7d7721e833 100644 --- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp @@ -1,11 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tensorfieldvalue.h" +#include <vespa/document/datatype/datatype.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/eval/tensor/tensor.h> #include <ostream> #include <cassert> using vespalib::tensor::Tensor; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/fieldvalue/variablemap.cpp b/document/src/vespa/document/fieldvalue/variablemap.cpp new file mode 100644 index 00000000000..b38b5baeec2 --- /dev/null +++ b/document/src/vespa/document/fieldvalue/variablemap.cpp @@ -0,0 +1,59 @@ +#include "variablemap.h" +#include "fieldvalue.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace document::fieldvalue { + +IndexValue::IndexValue() : index(-1), key() {} +IndexValue::IndexValue(int index_) : index(index_), key() {} + +bool +IndexValue::operator==(const IndexValue& other) const { + if (key.get() != NULL) { + if (other.key.get() != NULL && *key == *other.key) { + return true; + } + return false; + } + + return index == other.index; +} + +IndexValue::IndexValue(const FieldValue& key_) + : index(-1), + key(FieldValue::CP(key_.clone())) +{ } + +IndexValue::IndexValue(const IndexValue & rhs) = default; +IndexValue & IndexValue::operator = (const IndexValue & rhs) = default; + +IndexValue::~IndexValue() { } + +vespalib::string +IndexValue::toString() const { + if (key.get() != NULL) { + return key->toString(); + } else { + return vespalib::make_string("%d", index); + } +} + +VariableMap::VariableMap() {} +VariableMap::~VariableMap() {} +VariableMap::VariableMap(const VariableMap & rhs) = default; +VariableMap & VariableMap::operator = (const VariableMap & rhs) = default; + +vespalib::string +VariableMap::toString() const { + vespalib::asciistream out; + out << "[ "; + for (const auto & entry : *this) { + out << entry.first << "=" << entry.second.toString() << " "; + } + out << "]"; + return out.str(); +} + +} + diff --git a/document/src/vespa/document/fieldvalue/variablemap.h b/document/src/vespa/document/fieldvalue/variablemap.h new file mode 100644 index 00000000000..06679682e0c --- /dev/null +++ b/document/src/vespa/document/fieldvalue/variablemap.h @@ -0,0 +1,47 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/memory.h> +#include <vespa/vespalib/stllike/string.h> +#include <map> + +namespace document { + class FieldValue; +} + +namespace document::fieldvalue { + +class IndexValue { +public: + IndexValue(); + IndexValue(int index_); + IndexValue(const FieldValue& key_); + IndexValue(IndexValue && rhs) = default; + IndexValue & operator = (IndexValue && rhs) = default; + IndexValue(const IndexValue & rhs); + IndexValue & operator = (const IndexValue & rhs); + + ~IndexValue(); + + vespalib::string toString() const; + bool operator==(const IndexValue& other) const; + + int index; // For array + vespalib::CloneablePtr<FieldValue> key; // For map/wset +}; + +using VariableMapT = std::map<vespalib::string, IndexValue>; + +class VariableMap : public VariableMapT { +public: + VariableMap(); + VariableMap(VariableMap && rhs) = default; + VariableMap & operator = (VariableMap && rhs) = default; + VariableMap(const VariableMap & rhs); + VariableMap & operator = (const VariableMap & rhs); + ~VariableMap(); + vespalib::string toString() const; +}; + +}
\ No newline at end of file diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp index 7bf5f5350d3..67e0c8327e6 100644 --- a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp @@ -2,14 +2,20 @@ #include "weightedsetfieldvalue.h" #include <vespa/document/base/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> #include <ostream> using vespalib::Identifiable; +using vespalib::IllegalArgumentException; +using vespalib::IllegalStateException; +using namespace vespalib::xml; /// \todo TODO (was warning): Find a way to search through internal map without duplicating keys to create shared pointers. namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(WeightedSetFieldValue, CollectionFieldValue); namespace { @@ -17,9 +23,8 @@ const DataType &getKeyType(const DataType &type) { const WeightedSetDataType *wtype = Identifiable::cast<const WeightedSetDataType *>(&type); if (!wtype) { - throw vespalib::IllegalArgumentException( - "Cannot generate a weighted set value with non-weighted set " - "type " + type.toString() + ".", VESPA_STRLOC); + throw IllegalArgumentException("Cannot generate a weighted set value with non-weighted set " + "type " + type.toString() + ".", VESPA_STRLOC); } return wtype->getNestedType(); } @@ -39,8 +44,7 @@ WeightedSetFieldValue::~WeightedSetFieldValue() { } void WeightedSetFieldValue::verifyKey(const FieldValue & v) { if (!getNestedType().isValueType(v)) { - throw InvalidDataTypeException(*v.getDataType(), getNestedType(), - VESPA_STRLOC); + throw InvalidDataTypeException(*v.getDataType(), getNestedType(), VESPA_STRLOC); } } @@ -92,9 +96,7 @@ WeightedSetFieldValue::increment(const FieldValue& key, int val) } } else { if (it == _map.end()) { - throw vespalib::IllegalStateException("Cannot modify non-existing " - "entry in weightedset without createIfNonExistent set", - VESPA_STRLOC); + throw IllegalStateException("Cannot modify non-existing entry in weightedset without createIfNonExistent set", VESPA_STRLOC); } IntFieldValue& fv = static_cast<IntFieldValue&>(*it->second); fv.setValue(fv.getValue() + val); @@ -203,7 +205,7 @@ WeightedSetFieldValue::find(const FieldValue& key) return _map.find(key); } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus WeightedSetFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { return _map.iterateNestedImpl(nested, handler, *this); diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h index 4d6be1fd35a..5de0bf91614 100644 --- a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h @@ -7,9 +7,9 @@ */ #pragma once -#include <vespa/document/fieldvalue/collectionfieldvalue.h> -#include <vespa/document/fieldvalue/mapfieldvalue.h> -#include <vespa/document/fieldvalue/intfieldvalue.h> +#include "collectionfieldvalue.h" +#include "mapfieldvalue.h" +#include "intfieldvalue.h" #include <vespa/document/datatype/weightedsetdatatype.h> #include <map> @@ -33,7 +33,7 @@ private: bool addValue(const FieldValue& fval) override { return add(fval, 1); } bool containsValue(const FieldValue& val) const override; bool removeValue(const FieldValue& val) override; - IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler& handler) const override; + fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler& handler) const override; public: typedef std::unique_ptr<WeightedSetFieldValue> UP; diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt index 8521c525171..5077c9f0494 100644 --- a/document/src/vespa/document/select/CMakeLists.txt +++ b/document/src/vespa/document/select/CMakeLists.txt @@ -1,26 +1,27 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(document_select OBJECT SOURCES + bodyfielddetector.cpp branch.cpp + cloningvisitor.cpp compare.cpp constant.cpp + context.cpp doctype.cpp + gid_filter.cpp + invalidconstant.cpp operator.cpp - parser.cpp - result.cpp - value.cpp - valuenode.cpp orderingselector.cpp orderingspecification.cpp + parser.cpp + result.cpp + resultset.cpp resultlist.cpp - invalidconstant.cpp - bodyfielddetector.cpp simpleparser.cpp - resultset.cpp traversingvisitor.cpp - cloningvisitor.cpp - context.cpp - gid_filter.cpp + value.cpp + valuenode.cpp + valuenodes.cpp AFTER document_documentconfig ) diff --git a/document/src/vespa/document/select/bodyfielddetector.cpp b/document/src/vespa/document/select/bodyfielddetector.cpp index d0ddd0043ef..2b4c9890e45 100644 --- a/document/src/vespa/document/select/bodyfielddetector.cpp +++ b/document/src/vespa/document/select/bodyfielddetector.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bodyfielddetector.h" +#include "valuenodes.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/vespalib/util/closure.h> -#include "valuenode.h" -namespace document { - -namespace select { +namespace document::select { void BodyFieldDetector::detectFieldType(const FieldValueNode *expr, @@ -35,10 +32,8 @@ BodyFieldDetector::detectFieldType(const FieldValueNode *expr, void BodyFieldDetector::visitFieldValueNode(const FieldValueNode& expr) { - _repo.forEachDocumentType( - *makeClosure(this, &BodyFieldDetector::detectFieldType, &expr)); + _repo.forEachDocumentType(*makeClosure(this, &BodyFieldDetector::detectFieldType, &expr)); } -} // namespace select -} // namespace document +} diff --git a/document/src/vespa/document/select/cloningvisitor.cpp b/document/src/vespa/document/select/cloningvisitor.cpp index 84188788416..c0a45b758d5 100644 --- a/document/src/vespa/document/select/cloningvisitor.cpp +++ b/document/src/vespa/document/select/cloningvisitor.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "cloningvisitor.h" -#include "valuenode.h" +#include "valuenodes.h" #include "branch.h" #include "compare.h" #include "constant.h" diff --git a/document/src/vespa/document/select/gid_filter.cpp b/document/src/vespa/document/select/gid_filter.cpp index cb194452ead..7630053c16f 100644 --- a/document/src/vespa/document/select/gid_filter.cpp +++ b/document/src/vespa/document/select/gid_filter.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "gid_filter.h" #include "node.h" -#include "parser.h" #include "visitor.h" -#include "valuenode.h" +#include "valuenodes.h" #include "compare.h" #include "branch.h" #include <vespa/document/base/idstring.h> -namespace document { -namespace select { +namespace document::select { namespace { @@ -156,5 +154,4 @@ GidFilter::GidFilter(const Node& ast_root) { } -} // select -} // document +} diff --git a/document/src/vespa/document/select/node.h b/document/src/vespa/document/select/node.h index c56203da76c..6ccc1fa5dec 100644 --- a/document/src/vespa/document/select/node.h +++ b/document/src/vespa/document/select/node.h @@ -12,7 +12,6 @@ #pragma once -#include <string> #include "resultlist.h" #include "context.h" @@ -33,15 +32,11 @@ public: typedef std::shared_ptr<Node> SP; Node(const vespalib::stringref & name) : _name(name), _parentheses(false) {} - virtual ~Node() {} + ~Node() override {} void setParentheses() { _parentheses = true; } - void - clearParentheses() - { - _parentheses = false; - } + void clearParentheses() { _parentheses = false; } bool hadParentheses() const { return _parentheses; } diff --git a/document/src/vespa/document/select/orderingselector.cpp b/document/src/vespa/document/select/orderingselector.cpp index 32a1f0a8adf..4195a6c28a2 100644 --- a/document/src/vespa/document/select/orderingselector.cpp +++ b/document/src/vespa/document/select/orderingselector.cpp @@ -2,18 +2,16 @@ #include "orderingselector.h" #include "node.h" -#include "valuenode.h" +#include "valuenodes.h" #include "visitor.h" #include "compare.h" #include "branch.h" -#include <algorithm> #include <vespa/document/base/documentid.h> -#include <vespa/document/base/idstring.h> namespace document { -using namespace document::select; +using namespace select; namespace { /** diff --git a/document/src/vespa/document/select/parser.h b/document/src/vespa/document/select/parser.h index 61cc6e05876..b25ec951968 100644 --- a/document/src/vespa/document/select/parser.h +++ b/document/src/vespa/document/select/parser.h @@ -2,10 +2,8 @@ #pragma once -#include <memory> -#include <string> -#include <vespa/document/bucket/bucketidfactory.h> #include "node.h" +#include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/sync.h> diff --git a/document/src/vespa/document/select/result.h b/document/src/vespa/document/select/result.h index ae7b0fdcabf..ace3f5f717e 100644 --- a/document/src/vespa/document/select/result.h +++ b/document/src/vespa/document/select/result.h @@ -19,11 +19,9 @@ #pragma once -#include <string> #include <vespa/document/util/printable.h> -namespace document { -namespace select { +namespace document::select { class Result : public Printable { public: @@ -41,16 +39,9 @@ public: const Result& operator!() const; static const Result& get(bool b) { return (b ? True : False); } + static uint32_t enumRange() { return 3u; } - static uint32_t - enumRange(void) - { - return 3u; - } - - uint32_t - toEnum(void) const - { + uint32_t toEnum() const { if (this == &Result::Invalid) return 0u; if (this == &Result::False) @@ -60,9 +51,7 @@ public: abort(); } - static const Result & - fromEnum(uint32_t val) - { + static const Result &fromEnum(uint32_t val) { if (val == 0u) return Result::Invalid; if (val == 1u) @@ -74,12 +63,9 @@ public: private: Result(); - - // Singletons are not copyable - Result(const Result&); - Result& operator=(const Result&); + // Singletons are not copyable + Result(const Result&) = delete; + Result& operator=(const Result&) = delete; }; -} // select -} // document - +} diff --git a/document/src/vespa/document/select/resultlist.cpp b/document/src/vespa/document/select/resultlist.cpp index f8224b7a158..2cde06fa2f2 100644 --- a/document/src/vespa/document/select/resultlist.cpp +++ b/document/src/vespa/document/select/resultlist.cpp @@ -15,8 +15,7 @@ ResultList::ResultList(const Result& result) { } void -ResultList::add(const FieldValue::IteratorHandler::VariableMap& variables, - const Result& result) +ResultList::add(const fieldvalue::VariableMap& variables, const Result& result) { _results.push_back(ResultPair(variables, &result)); } @@ -27,8 +26,7 @@ ResultList::print(std::ostream& out, bool, const std::string&) const out << "ResultList("; for (uint32_t i = 0; i < _results.size(); i++) { if (!_results[i].first.empty()) { - out << FieldValue::IteratorHandler::toString(_results[i].first) - << " => "; + out << _results[i].first.toString() << " => "; } out << _results[i].second->toString() << " "; } @@ -66,15 +64,12 @@ ResultList::combineResults() const { bool ResultList::combineVariables( - FieldValue::IteratorHandler::VariableMap& output, - const FieldValue::IteratorHandler::VariableMap& input) const + fieldvalue::VariableMap& output, + const fieldvalue::VariableMap& input) const { // First, verify that all variables are overlapping - for (FieldValue::IteratorHandler::VariableMap::const_iterator iter - = output.begin(); iter != output.end(); iter++) - { - FieldValue::IteratorHandler::VariableMap::const_iterator found( - input.find(iter->first)); + for (fieldvalue::VariableMap::const_iterator iter = output.begin(); iter != output.end(); iter++) { + fieldvalue::VariableMap::const_iterator found(input.find(iter->first)); if (found != input.end()) { if (!(found->second == iter->second)) { @@ -83,11 +78,8 @@ ResultList::combineVariables( } } - for (FieldValue::IteratorHandler::VariableMap::const_iterator iter - = input.begin(); iter != input.end(); iter++) - { - FieldValue::IteratorHandler::VariableMap::const_iterator found( - output.find(iter->first)); + for (fieldvalue::VariableMap::const_iterator iter = input.begin(); iter != input.end(); iter++) { + fieldvalue::VariableMap::const_iterator found(output.find(iter->first)); if (found != output.end()) { if (!(found->second == iter->second)) { return false; @@ -95,9 +87,7 @@ ResultList::combineVariables( } } // Ok, variables are overlapping. Add all variables from input to output. - for (FieldValue::IteratorHandler::VariableMap::const_iterator iter - = input.begin(); iter != input.end(); iter++) - { + for (fieldvalue::VariableMap::const_iterator iter = input.begin(); iter != input.end(); iter++) { output[iter->first] = iter->second; } @@ -112,7 +102,7 @@ ResultList::operator&&(const ResultList& other) const // TODO: optimize for (const auto & it : _results) { for (const auto & it2 : other._results) { - FieldValue::IteratorHandler::VariableMap vars = it.first; + fieldvalue::VariableMap vars = it.first; if (combineVariables(vars, it2.first)) { result.add(vars, *it.second && *it2.second); @@ -131,7 +121,7 @@ ResultList::operator||(const ResultList& other) const // TODO: optimize for (const auto & it : _results) { for (const auto & it2 : other._results) { - FieldValue::IteratorHandler::VariableMap vars = it.first; + fieldvalue::VariableMap vars = it.first; if (combineVariables(vars, it2.first)) { result.add(vars, *it.second || *it2.second); diff --git a/document/src/vespa/document/select/resultlist.h b/document/src/vespa/document/select/resultlist.h index 6a3cc1f78d1..a00419d9196 100644 --- a/document/src/vespa/document/select/resultlist.h +++ b/document/src/vespa/document/select/resultlist.h @@ -1,16 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> -#include <vespa/document/fieldvalue/fieldvalue.h> #include "result.h" +#include <vector> +#include <vespa/document/fieldvalue/variablemap.h> -namespace document { -namespace select { +namespace document::select { class ResultList : public Printable { public: - typedef FieldValue::IteratorHandler::VariableMap VariableMap; + using VariableMap = fieldvalue::VariableMap; typedef std::pair<VariableMap, const Result*> ResultPair; typedef std::vector<ResultPair> Results; typedef Results::iterator iterator; @@ -49,7 +48,7 @@ public: private: Results _results; - bool combineVariables(VariableMap& output, const FieldValue::IteratorHandler::VariableMap& input) const; + bool combineVariables(VariableMap& output, const VariableMap& input) const; }; inline bool operator==(const ResultList& list, const Result& other) { @@ -61,5 +60,4 @@ inline bool operator!=(const ResultList& list, const Result& other) { } } -} diff --git a/document/src/vespa/document/select/simpleparser.h b/document/src/vespa/document/select/simpleparser.h index e6457c3dec7..c52aed0f367 100644 --- a/document/src/vespa/document/select/simpleparser.h +++ b/document/src/vespa/document/select/simpleparser.h @@ -2,17 +2,11 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> #include "node.h" -#include "valuenode.h" +#include "valuenodes.h" #include "operator.h" -#include <vespa/document/bucket/bucketidfactory.h> -namespace document { - -namespace select { - -namespace simple { +namespace document::select::simple { class Parser { public: @@ -89,9 +83,4 @@ private: const BucketIdFactory & _bucketIdFactory; }; - - -} // simple -} // select -} // parser - +} diff --git a/document/src/vespa/document/select/traversingvisitor.cpp b/document/src/vespa/document/select/traversingvisitor.cpp index 7f85d8b7ebb..2f2468e5afc 100644 --- a/document/src/vespa/document/select/traversingvisitor.cpp +++ b/document/src/vespa/document/select/traversingvisitor.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "traversingvisitor.h" -#include "valuenode.h" +#include "valuenodes.h" #include "branch.h" #include "compare.h" -namespace document { - -namespace select { +namespace document::select { void TraversingVisitor::visitAndBranch(const And &expr) @@ -127,5 +125,3 @@ TraversingVisitor::visitInvalidValueNode(const InvalidValueNode &) } } - -} diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp index 7a453fe5cf3..3540935f8c7 100644 --- a/document/src/vespa/document/select/value.cpp +++ b/document/src/vespa/document/select/value.cpp @@ -2,6 +2,7 @@ #include "value.h" #include "operator.h" +#include <vespa/document/fieldvalue/fieldvalue.h> namespace document { namespace select { diff --git a/document/src/vespa/document/select/value.h b/document/src/vespa/document/select/value.h index 15856648985..b26e959ea1d 100644 --- a/document/src/vespa/document/select/value.h +++ b/document/src/vespa/document/select/value.h @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + /** * @class document::select::Value * @ingroup select @@ -190,7 +191,7 @@ inline ResultList FloatValue::operator==(const FloatValue& value) const class ArrayValue : public Value { public: - typedef std::pair<FieldValue::IteratorHandler::VariableMap, Value::SP> VariableValue; + using VariableValue = std::pair<fieldvalue::VariableMap, Value::SP>; ArrayValue(const std::vector<VariableValue>& values); diff --git a/document/src/vespa/document/select/valuenode.cpp b/document/src/vespa/document/select/valuenode.cpp index 96bc46bd54e..79b795ca76f 100644 --- a/document/src/vespa/document/select/valuenode.cpp +++ b/document/src/vespa/document/select/valuenode.cpp @@ -1,39 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "valuenode.h" -#include "visitor.h" -#include "parser.h" -#include <vespa/document/base/exceptions.h> -#include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/vespalib/util/md5.h> -#include <vespa/document/util/stringutil.h> -#include <vespa/vespalib/text/lowercase.h> -#include <regex> -#include <iomanip> +#include <ostream> +namespace document::select { -#include <vespa/log/log.h> -LOG_SETUP(".document.select.valuenode"); - -namespace document { -namespace select { - -namespace { - static const std::regex FIELD_NAME_REGEX("^([_A-Za-z][_A-Za-z0-9]*).*"); -} - -namespace { - bool documentTypeEqualsName(const DocumentType& type, const vespalib::stringref& name) - { - if (type.getName() == name) return true; - for (std::vector<const DocumentType *>::const_iterator it - = type.getInheritedTypes().begin(); - it != type.getInheritedTypes().end(); ++it) - { - if (documentTypeEqualsName(**it, name)) return true; - } - return false; - } +std::unique_ptr<Value> +ValueNode::traceValue(const Context &context, std::ostream &out) const { + return defaultTrace(getValue(context), out); } std::unique_ptr<Value> @@ -43,1128 +16,5 @@ ValueNode::defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const return std::move(val); } -InvalidValueNode::InvalidValueNode(const vespalib::stringref & name) - : _name(name) -{ } - - -void -InvalidValueNode::visit(Visitor &visitor) const -{ - visitor.visitInvalidValueNode(*this); -} - - -void -InvalidValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _name; - if (hadParentheses()) out << ')'; -} - -NullValueNode::NullValueNode(const vespalib::stringref & name) - : _name(name) -{ } - - -void -NullValueNode::visit(Visitor &visitor) const -{ - visitor.visitNullValueNode(*this); -} - - -void -NullValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _name; - if (hadParentheses()) out << ')'; -} - -StringValueNode::StringValueNode(const vespalib::stringref & val) - : _value(val) -{ -} - - -void -StringValueNode::visit(Visitor &visitor) const -{ - visitor.visitStringValueNode(*this); -} - - -void -StringValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << "\"" << StringUtil::escape(_value) << "\""; - if (hadParentheses()) out << ')'; -} - - -void -IntegerValueNode::visit(Visitor &visitor) const -{ - visitor.visitIntegerValueNode(*this); -} - - -void -IntegerValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _value; - if (hadParentheses()) out << ')'; -} - -int64_t -CurrentTimeValueNode::getValue() const -{ - struct timeval mytime; - gettimeofday(&mytime, 0); - return mytime.tv_sec; -} - - -void -CurrentTimeValueNode::visit(Visitor &visitor) const -{ - visitor.visitCurrentTimeValueNode(*this); -} - - -void -CurrentTimeValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "now()"; -} - -std::unique_ptr<Value> -VariableValueNode::getValue(const Context& context) const { - return context.getValue(_value); -} - - -void -VariableValueNode::visit(Visitor &visitor) const -{ - visitor.visitVariableValueNode(*this); -} - - -void -VariableValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << "$" << _value; - if (hadParentheses()) out << ')'; -} - - -void -FloatValueNode::visit(Visitor &visitor) const -{ - visitor.visitFloatValueNode(*this); -} - - -void -FloatValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _value; - if (hadParentheses()) out << ')'; -} - -FieldValueNode::FieldValueNode(const vespalib::string& doctype, - const vespalib::string& fieldExpression) - : _doctype(doctype), - _fieldExpression(fieldExpression), - _fieldName(extractFieldName(fieldExpression)) -{ -} - -FieldValueNode::FieldValueNode(const FieldValueNode &) = default; -FieldValueNode & FieldValueNode::operator = (const FieldValueNode &) = default; - -FieldValueNode::~FieldValueNode() {} -vespalib::string -FieldValueNode::extractFieldName(const std::string & fieldExpression) { - std::smatch match; - - if (std::regex_match(fieldExpression, match, FIELD_NAME_REGEX) && match[1].matched) { - return vespalib::string(match[1].first, match[1].second); - } - - throw ParsingFailedException("Fatal: could not extract field name from field expression '" + fieldExpression + "'"); -} - -void -FieldValueNode::initFieldPath(const DocumentType& type) const { - if (_fieldPath.size() == 0) { - FieldPath::UP path(type.buildFieldPath(_fieldExpression)); - if (!path.get()) { - throw FieldNotFoundException( - vespalib::make_string("Could not create field path for doc type: '%s' field: '%s'", - type.toString().c_str(), _fieldExpression.c_str()), - VESPA_STRLOC); - } - _fieldPath = *path; - } -} - -std::unique_ptr<Value> -FieldValueNode::getValue(const Context& context) const -{ - if (context._doc == NULL) { - return std::unique_ptr<Value>(new InvalidValue()); - } - - const Document& doc = *context._doc; - - if (!documentTypeEqualsName(doc.getType(), _doctype)) { - return std::unique_ptr<Value>(new InvalidValue()); - } - try{ - initFieldPath(doc.getType()); - - IteratorHandler handler; - doc.iterateNested(_fieldPath.getFullRange(), handler); - - if (handler.hasSingleValue()) { - return handler.getSingleValue(); - } else { - const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); - - if (values.size() == 0) { - return std::unique_ptr<Value>(new NullValue()); - } else { - return std::unique_ptr<Value>(new ArrayValue(handler.getValues())); - } - } - } catch (vespalib::IllegalArgumentException& e) { - LOG(warning, "Caught exception while fetching field from document: %s", e.what()); - return std::unique_ptr<Value>(new InvalidValue()); - } catch (FieldNotFoundException& e) { - LOG(warning, "Tried to compare to field %s, not found in document type", _fieldExpression.c_str()); - return std::unique_ptr<Value>(new InvalidValue()); - } -} - -FieldValueNode::IteratorHandler::IteratorHandler() { } -FieldValueNode::IteratorHandler::~IteratorHandler() { } - -bool -FieldValueNode::IteratorHandler::hasSingleValue() const { - return _firstValue.get() && (_values.size() == 0); -} - -std::unique_ptr<Value> -FieldValueNode::IteratorHandler::getSingleValue() { - return std::move(_firstValue); -} - -const std::vector<ArrayValue::VariableValue>& -FieldValueNode::IteratorHandler::getValues() { - if (_firstValue.get()) { - _values.insert(_values.begin(), ArrayValue::VariableValue(FieldValue::IteratorHandler::VariableMap(), Value::SP(_firstValue.release()))); - } - - return _values; -} - -void -FieldValueNode::IteratorHandler::onPrimitive(uint32_t fid, const Content& fv) { - (void) fid; - if (!_firstValue && getVariables().empty()) { - _firstValue = getInternalValue(fv.getValue()); - } else { - _values.emplace_back(getVariables(), Value::SP(getInternalValue(fv.getValue()).release())); - } -} - -std::unique_ptr<Value> -FieldValueNode::IteratorHandler::getInternalValue(const FieldValue& fval) const -{ - switch(fval.getClass().id()) { - case document::IntFieldValue::classId: - { - const IntFieldValue& val(dynamic_cast<const IntFieldValue&>(fval)); - return std::make_unique<IntegerValue>(val.getAsInt(), false); - } - case document::ByteFieldValue::classId: - { - const ByteFieldValue& val(dynamic_cast<const ByteFieldValue&>(fval)); - return std::make_unique<IntegerValue>(val.getAsByte(), false); - } - case LongFieldValue::classId: - { - const LongFieldValue& val(dynamic_cast<const LongFieldValue&>(fval)); - return std::make_unique<IntegerValue>(val.getAsLong(), false); - } - case FloatFieldValue::classId: - { - const FloatFieldValue& val(dynamic_cast<const FloatFieldValue&>(fval)); - return std::make_unique<FloatValue>(val.getAsFloat()); - } - case DoubleFieldValue::classId: - { - const DoubleFieldValue& val(dynamic_cast<const DoubleFieldValue&>(fval)); - return std::make_unique<FloatValue>(val.getAsDouble()); - } - case StringFieldValue::classId: - { - const StringFieldValue& val(dynamic_cast<const StringFieldValue&>(fval)); - return std::make_unique<StringValue>(val.getAsString()); - } - case ArrayFieldValue::classId: - { - const ArrayFieldValue& val(dynamic_cast<const ArrayFieldValue&>(fval)); - if (val.size() == 0) { - return std::make_unique<NullValue>(); - } else { - std::vector<ArrayValue::VariableValue> values; - // TODO: Array comparison. - return std::make_unique<ArrayValue>(values); - } - } - case StructFieldValue::classId: - { - const StructFieldValue& val(dynamic_cast<const StructFieldValue&>(fval)); - if (val.empty()) { - return std::make_unique<NullValue>(); - } else { - StructValue::ValueMap values; - for (StructFieldValue::const_iterator it(val.begin()); it != val.end(); ++it) { - FieldValue::UP fv(val.getValue(it.field())); - values[it.field().getName()] = Value::SP(getInternalValue(*fv).release()); - } - return std::make_unique<StructValue>(values); - } - } - case MapFieldValue::classId: - { - const MapFieldValue& val(static_cast<const MapFieldValue&>(fval)); - if (val.isEmpty()) { - return std::make_unique<NullValue>(); - } else { - std::vector<ArrayValue::VariableValue> values; - // TODO: Map comparison - return std::make_unique<ArrayValue>(values); - } - } - } - LOG(warning, "Tried to use unsupported datatype %s in field comparison", - fval.getDataType()->toString().c_str()); - return std::make_unique<InvalidValue>(); -} - - -void -FieldValueNode::visit(Visitor &visitor) const -{ - visitor.visitFieldValueNode(*this); -} - - -void -FieldValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _doctype << "." << _fieldExpression; - if (hadParentheses()) out << ')'; -} - - -std::unique_ptr<Value> -FieldValueNode::traceValue(const Context &context, std::ostream& out) const -{ - if (context._doc == NULL) { - return defaultTrace(getValue(context), out); - } - const Document &doc(*context._doc); - if (!documentTypeEqualsName(doc.getType(), _doctype)) { - out << "Document is of type " << doc.getType() << " which isn't a " - << _doctype << " document, thus resolving invalid.\n"; - return std::unique_ptr<Value>(new InvalidValue()); - } - try{ - initFieldPath(doc.getType()); - - IteratorHandler handler; - doc.iterateNested(_fieldPath.getFullRange(), handler); - - if (handler.hasSingleValue()) { - return handler.getSingleValue(); - } else { - const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); - - if (values.size() == 0) { - return std::unique_ptr<Value>(new NullValue()); - } else { - return std::unique_ptr<Value>(new ArrayValue(handler.getValues())); - } - } - } catch (FieldNotFoundException& e) { - LOG(warning, "Tried to compare to field %s, not found in document type", - _fieldExpression.c_str()); - out << "Field not found in document type " << doc.getType() - << ". Returning invalid.\n"; - return std::unique_ptr<Value>(new InvalidValue()); - } -} - -IdValueNode::IdValueNode(const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, const vespalib::stringref & type, - int widthBits, int divisionBits) - : _bucketIdFactory(bucketIdFactory), - _id(name), - _typestring(type), - _type(ALL), - _widthBits(widthBits), - _divisionBits(divisionBits) -{ - if (type.length() > 2) switch (type[0]) { - case 'b': _type = BUCKET; - break; - case 'n': _type = NS; - break; - case 'g': - if (type[1] == 'r') { - _type = GROUP; - } else if (type[1] == 'i') { - _type = GID; - } - break; - case 's': { - if (type[1] == 'c') { _type = SCHEME; } else { _type = SPEC; } - break; - } - case 't': - _type = TYPE; - break; - case 'u': - _type = USER; - break; - case 'o': - _type = ORDER; - break; - } -} - - -std::unique_ptr<Value> -IdValueNode::getValue(const Context& context) const -{ - if (context._doc != NULL) { - return getValue(context._doc->getId()); - } else if (context._docId != NULL) { - return getValue(*context._docId); - } else { - return getValue(context._docUpdate->getId()); - } -} - - -std::unique_ptr<Value> -IdValueNode::getValue(const DocumentId& id) const -{ - vespalib::string value; - switch (_type) { - case BUCKET: - return std::unique_ptr<Value>( - new IntegerValue( - _bucketIdFactory.getBucketId(id).getId(), true)); - case NS: - value = id.getScheme().getNamespace(); break; - case SCHEME: - value = id.getScheme().getTypeName(id.getScheme().getType()); - break; - case TYPE: - if (id.getScheme().hasDocType()) { - value = id.getScheme().getDocType(); - } else { - return std::unique_ptr<Value>(new InvalidValue); - } - break; - case SPEC: - value = id.getScheme().getNamespaceSpecific(); - break; - case ALL: - value = id.getScheme().toString(); - break; - case GROUP: - if (id.getScheme().hasGroup()) { - value = id.getScheme().getGroup(); - } else { - fprintf(stderr, "***** Returning invalid value for %s\n", - id.toString().c_str()); - return std::unique_ptr<Value>(new InvalidValue); - } - break; - case GID: - value = id.getGlobalId().toString(); - break; - case ORDER: - if (id.getScheme().getType() == IdString::ORDERDOC) { - const OrderDocIdString& ods( - static_cast<const OrderDocIdString&>(id.getScheme())); - if (ods.getWidthBits() == _widthBits - && ods.getDivisionBits() == _divisionBits) - { - return std::unique_ptr<Value>(new IntegerValue( - static_cast<const OrderDocIdString&>(id.getScheme()) - .getOrdering(), false)); - } - } - return std::unique_ptr<Value>(new InvalidValue()); - case USER: - if (id.getScheme().hasNumber()) { - return std::unique_ptr<Value>( - new IntegerValue(id.getScheme().getNumber(), false)); - } else { - return std::unique_ptr<Value>(new InvalidValue); - } - } - - return std::unique_ptr<Value>(new StringValue(value)); -} - - -std::unique_ptr<Value> -IdValueNode::traceValue(const Context& context, - std::ostream &out) const -{ - if (context._doc != NULL) { - return traceValue(context._doc->getId(), out); - } else if (context._docId != NULL) { - return traceValue(*context._docId, out); - } else { - return traceValue(context._docUpdate->getId(), out); - } -} - - -std::unique_ptr<Value> -IdValueNode::traceValue(const DocumentId& id, std::ostream& out) const -{ - vespalib::string value; - switch (_type) { - case BUCKET: - { - document::BucketId bucket(_bucketIdFactory.getBucketId(id)); - std::unique_ptr<Value> result( - new IntegerValue(bucket.getId(), true)); - out << "Found id.bucket specification. Resolved to " - << bucket.toString() << ".\n"; - return result; - } - case NS: - value = id.getScheme().getNamespace(); - out << "Resolved id.namespace to value\"" << value << "\".\n"; - break; - case SCHEME: - value = id.getScheme().getTypeName(id.getScheme().getType()); - out << "Resolved id.scheme to value\"" << value << "\".\n"; - break; - case TYPE: - if (id.getScheme().hasDocType()) { - value = id.getScheme().getDocType(); - out << "Resolved id.type to value\"" << value << "\".\n"; - } else { - out << "Could not resolve type of doc " << id << ".\n"; - return std::unique_ptr<Value>(new InvalidValue); - } - break; - case SPEC: - value = id.getScheme().getNamespaceSpecific(); - out << "Resolved id.specific to value\"" << value << "\".\n"; - break; - case ALL: - value = id.getScheme().toString(); - out << "Resolved id to \"" << value << "\".\n"; - break; - case GROUP: - if (id.getScheme().hasGroup()) { - value = id.getScheme().getGroup(); - out << "Resolved group of doc (type " << id.getScheme().getType() - << ") to \"" << value << "\".\n"; - } else { - out << "Can't resolve group of doc \"" << id << "\".\n"; - return std::unique_ptr<Value>(new InvalidValue); - } - break; - case GID: - value = id.getGlobalId().toString(); - out << "Resolved gid to \"" << value << "\".\n"; - break; - case ORDER: - if (id.getScheme().getType() == IdString::ORDERDOC) { - const OrderDocIdString& ods( - static_cast<const OrderDocIdString&>(id.getScheme())); - if (ods.getWidthBits() == _widthBits - && ods.getDivisionBits() == _divisionBits) - { - std::unique_ptr<Value> result(new IntegerValue( - static_cast<const OrderDocIdString&>(id.getScheme()) - .getOrdering(), false)); - out << "Resolved id.order to int " << *result << "\n"; - return result; - } - } - out << "Could not resolve id.order(" << _widthBits << ", " - << _divisionBits << ") of doc " << id << ".\n"; - return std::unique_ptr<Value>(new InvalidValue()); - case USER: - if (id.getScheme().hasNumber()) { - std::unique_ptr<Value> result( - new IntegerValue(id.getScheme().getNumber(), false)); - out << "Resolved user of doc type " << id.getScheme().getType() - << " to " << *result << ".\n"; - return result; - } else { - out << "Could not resolve user of doc " << id << ".\n"; - return std::unique_ptr<Value>(new InvalidValue); - } - } - - return std::unique_ptr<Value>(new StringValue(value)); -} - - -void -IdValueNode::visit(Visitor &visitor) const -{ - visitor.visitIdValueNode(*this); -} - - -void -IdValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _id; - if (_type != ALL) { - out << '.' << _typestring; - } - if (_type == ORDER) { - out << "(" << _widthBits << "," << _divisionBits << ")"; - } - if (hadParentheses()) out << ')'; -} - -SearchColumnValueNode::SearchColumnValueNode( - const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, int numColumns) - : _bucketIdFactory(bucketIdFactory), - _id(name), - _numColumns(numColumns), - _distribution(_numColumns, 16) -{ -} - -int64_t -SearchColumnValueNode::getValue(const BucketId& id) const -{ - return _distribution.getColumn(id); -} - - -std::unique_ptr<Value> -SearchColumnValueNode::getValue(const Context& context) const -{ - if (context._doc != NULL) { - return getValue(context._doc->getId()); - } else if (context._docId != NULL) { - return getValue(*context._docId); - } else { - return getValue(context._docUpdate->getId()); - } -} - - -std::unique_ptr<Value> -SearchColumnValueNode::getValue(const DocumentId& id) const -{ - return std::unique_ptr<Value>(new IntegerValue( - getValue(_bucketIdFactory.getBucketId(id)), false)); -} - - -std::unique_ptr<Value> -SearchColumnValueNode::traceValue(const Context& context, - std::ostream &out) const -{ - if (context._doc != NULL) { - return traceValue(context._doc->getId(), out); - } else if (context._docId != NULL) { - return traceValue(*context._docId, out); - } else { - return traceValue(context._docUpdate->getId(), out); - } -} - - -std::unique_ptr<Value> -SearchColumnValueNode::traceValue(const DocumentId& id, - std::ostream& out) const -{ - std::unique_ptr<Value> result(new IntegerValue( - getValue(_bucketIdFactory.getBucketId(id)), false)); - out << "Resolved search column of doc \"" << id << "\" to " << *result - << "\n"; - return result; -} - - -void -SearchColumnValueNode::visit(Visitor &visitor) const -{ - visitor.visitSearchColumnValueNode(*this); -} - - -void -SearchColumnValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _id; - out << '.' << _numColumns; - if (hadParentheses()) out << ')'; -} - -namespace { - union HashUnion { - unsigned char _key[16]; - int64_t _hash[2]; - }; - int64_t hash(const void* data, uint32_t len) { - HashUnion hash; - fastc_md5sum((const unsigned char*) data, len, hash._key); - return hash._hash[0]; - } -} - -FunctionValueNode::FunctionValueNode(const vespalib::stringref & name, - std::unique_ptr<ValueNode> src) - : _function(), - _funcname(name), - _source(std::move(src)) -{ - if (name == "lowercase") { - _function = LOWERCASE; - } else if (name == "hash") { - _function = HASH; - } else if (name == "abs") { - _function = ABS; - } else { - throw ParsingFailedException("No function '"+name+"' exist.", - VESPA_STRLOC); - } -} - -std::unique_ptr<Value> -FunctionValueNode::getValue(std::unique_ptr<Value> val) const -{ - switch (val->getType()) { - case Value::String: - { - StringValue& sval(static_cast<StringValue&>(*val)); - if (_function == LOWERCASE) { - return std::unique_ptr<Value>(new StringValue( - vespalib::LowerCase::convert(sval.getValue()))); - } else if (_function == HASH) { - return std::unique_ptr<Value>(new IntegerValue( - hash(sval.getValue().c_str(), sval.getValue().size()), - false)); - } - break; - } - case Value::Float: - { - FloatValue& fval(static_cast<FloatValue&>(*val)); - if (_function == HASH) { - FloatValue::ValueType ffval = fval.getValue(); - return std::unique_ptr<Value>(new IntegerValue( - hash(&ffval, sizeof(ffval)), false)); - } else if (_function == ABS) { - FloatValue::ValueType ffval = fval.getValue(); - if (ffval < 0) ffval *= -1; - return std::unique_ptr<Value>(new FloatValue(ffval)); - } - break; - } - case Value::Integer: - { - IntegerValue& ival(static_cast<IntegerValue&>(*val)); - if (_function == HASH) { - IntegerValue::ValueType iival = ival.getValue(); - return std::unique_ptr<Value>(new IntegerValue( - hash(&iival, sizeof(iival)), false)); - } else if (_function == ABS) { - IntegerValue::ValueType iival = ival.getValue(); - if (iival < 0) iival *= -1; - return std::unique_ptr<Value>(new IntegerValue(iival, false)); - } - break; - } - case Value::Bucket: - { - throw ParsingFailedException( - "No functioncalls are allowed on value of type bucket", - VESPA_STRLOC); - break; - } - - case Value::Array: break; - case Value::Struct: break; - case Value::Invalid: break; - case Value::Null: break; - } - return std::unique_ptr<Value>(new InvalidValue); -} - -std::unique_ptr<Value> -FunctionValueNode::traceValue(std::unique_ptr<Value> val, - std::ostream& out) const -{ - switch (val->getType()) { - case Value::String: - { - StringValue& sval(static_cast<StringValue&>(*val)); - if (_function == LOWERCASE) { - std::unique_ptr<Value> result(new StringValue( - vespalib::LowerCase::convert(sval.getValue()))); - out << "Performed lowercase function on '" << sval - << "' => '" << *result << "'.\n"; - return result; - } else if (_function == HASH) { - std::unique_ptr<Value> result(new IntegerValue( - hash(sval.getValue().c_str(), sval.getValue().size()), - false)); - out << "Performed hash on string '" << sval << "' -> " - << *result << "\n"; - return result; - } - break; - } - case Value::Float: - { - FloatValue& fval(static_cast<FloatValue&>(*val)); - if (_function == HASH) { - FloatValue::ValueType ffval = fval.getValue(); - std::unique_ptr<Value> result(new IntegerValue( - hash(&ffval, sizeof(ffval)), false)); - out << "Performed hash on float " << ffval << " -> " << *result - << "\n"; - return result; - } else if (_function == ABS) { - FloatValue::ValueType ffval = fval.getValue(); - if (ffval < 0) ffval *= -1; - out << "Performed abs on float " << fval.getValue() << " -> " - << ffval << "\n"; - return std::unique_ptr<Value>(new FloatValue(ffval)); - } - break; - } - case Value::Integer: - { - IntegerValue& ival(static_cast<IntegerValue&>(*val)); - if (_function == HASH) { - IntegerValue::ValueType iival = ival.getValue(); - std::unique_ptr<Value> result(new IntegerValue( - hash(&iival, sizeof(iival)), false)); - out << "Performed hash on float " << iival << " -> " << *result - << "\n"; - return result; - } else if (_function == ABS) { - IntegerValue::ValueType iival = ival.getValue(); - if (iival < 0) iival *= -1; - out << "Performed abs on integer " << ival.getValue() << " -> " - << iival << "\n"; - return std::unique_ptr<Value>(new IntegerValue(iival, false)); - } - break; - } - case Value::Bucket: break; - case Value::Array: break; - case Value::Struct: break; - case Value::Invalid: break; - case Value::Null: break; - } - out << "Cannot use function " << _function << " on a value of type " - << val->getType() << ". Resolving invalid.\n"; - return std::unique_ptr<Value>(new InvalidValue); -} - - -void -FunctionValueNode::visit(Visitor &visitor) const -{ - visitor.visitFunctionValueNode(*this); -} - - -void -FunctionValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - if (hadParentheses()) out << '('; - _source->print(out, verbose, indent); - out << '.' << _funcname << "()"; - if (hadParentheses()) out << ')'; -} - -ArithmeticValueNode::ArithmeticValueNode( - std::unique_ptr<ValueNode> left, const vespalib::stringref & op, - std::unique_ptr<ValueNode> right) - : _operator(), - _left(std::move(left)), - _right(std::move(right)) -{ - if (op.size() == 1) switch (op[0]) { - case '+': _operator = ADD; return; - case '-': _operator = SUB; return; - case '*': _operator = MUL; return; - case '/': _operator = DIV; return; - case '%': _operator = MOD; return; - } - throw ParsingFailedException( - "Arithmetic operator '"+op+"' does not exist.", VESPA_STRLOC); -} - -const char* -ArithmeticValueNode::getOperatorName() const -{ - switch (_operator) { - case ADD: return "+"; - case SUB: return "-"; - case MUL: return "*"; - case DIV: return "/"; - case MOD: return "%"; - } - return "UNKNOWN"; -} - - - -std::unique_ptr<Value> -ArithmeticValueNode::getValue(std::unique_ptr<Value> lval, - std::unique_ptr<Value> rval) const -{ - switch (_operator) { - case ADD: - { - if (lval->getType() == Value::String && - rval->getType() == Value::String) - { - StringValue& slval(static_cast<StringValue&>(*lval)); - StringValue& srval(static_cast<StringValue&>(*rval)); - return std::unique_ptr<Value>(new StringValue( - slval.getValue() + srval.getValue())); - } - } - case SUB: - case MUL: - case DIV: - { - if (lval->getType() == Value::Integer && - rval->getType() == Value::Integer) - { - IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); - IntegerValue& irval(static_cast<IntegerValue&>(*rval)); - IntegerValue::ValueType res = 0; - switch (_operator) { - case ADD: res = ilval.getValue() + irval.getValue(); break; - case SUB: res = ilval.getValue() - irval.getValue(); break; - case MUL: res = ilval.getValue() * irval.getValue(); break; - case DIV: - if (irval.getValue() != 0) { - res = ilval.getValue() / irval.getValue(); - } else { - throw vespalib::IllegalArgumentException("Division by zero"); - } - break; - case MOD: assert(0); - } - return std::unique_ptr<Value>(new IntegerValue(res, false)); - } - NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get())); - NumberValue* nrval(dynamic_cast<NumberValue*>(rval.get())); - if (nlval != 0 && nrval != 0) { - NumberValue::CommonValueType res = 0; - switch (_operator) { - case ADD: res = nlval->getCommonValue() - + nrval->getCommonValue(); break; - case SUB: res = nlval->getCommonValue() - - nrval->getCommonValue(); break; - case MUL: res = nlval->getCommonValue() - * nrval->getCommonValue(); break; - case DIV: - if (nrval->getCommonValue() != 0) { - res = nlval->getCommonValue() - / nrval->getCommonValue(); - } else { - throw vespalib::IllegalArgumentException("Division by zero"); - } - break; - case MOD: assert(0); - } - return std::unique_ptr<Value>(new FloatValue(res)); - } - } - case MOD: - { - if (lval->getType() == Value::Integer && - rval->getType() == Value::Integer) - { - IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); - IntegerValue& irval(static_cast<IntegerValue&>(*rval)); - if (irval.getValue() != 0) { - return std::unique_ptr<Value>(new IntegerValue(ilval.getValue() % irval.getValue(), false)); - } else { - throw vespalib::IllegalArgumentException("Division by zero"); - } - } - } - } - return std::unique_ptr<Value>(new InvalidValue); -} - -std::unique_ptr<Value> -ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval, - std::unique_ptr<Value> rval, - std::ostream& out) const -{ - switch (_operator) { - case ADD: - { - if (lval->getType() == Value::String && - rval->getType() == Value::String) - { - StringValue& slval(static_cast<StringValue&>(*lval)); - StringValue& srval(static_cast<StringValue&>(*rval)); - std::unique_ptr<Value> result(new StringValue( - slval.getValue() + srval.getValue())); - out << "Appended strings '" << slval << "' + '" << srval - << "' -> '" << *result << "'.\n"; - return result; - } - } - case SUB: - case MUL: - case DIV: - { - if (lval->getType() == Value::Integer && - rval->getType() == Value::Integer) - { - IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); - IntegerValue& irval(static_cast<IntegerValue&>(*rval)); - IntegerValue::ValueType res = 0; - switch (_operator) { - case ADD: res = ilval.getValue() + irval.getValue(); break; - case SUB: res = ilval.getValue() - irval.getValue(); break; - case MUL: res = ilval.getValue() * irval.getValue(); break; - case DIV: res = ilval.getValue() / irval.getValue(); break; - case MOD: assert(0); - } - std::unique_ptr<Value> result(new IntegerValue(res, false)); - out << "Performed integer operation " << ilval << " " - << getOperatorName() << " " << irval << " = " << *result - << "\n"; - return result; - } - NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get())); - NumberValue* nrval(dynamic_cast<NumberValue*>(lval.get())); - if (nlval != 0 && nrval != 0) { - NumberValue::CommonValueType res = 0; - switch (_operator) { - case ADD: res = nlval->getCommonValue() - + nrval->getCommonValue(); break; - case SUB: res = nlval->getCommonValue() - - nrval->getCommonValue(); break; - case MUL: res = nlval->getCommonValue() - * nrval->getCommonValue(); break; - case DIV: res = nlval->getCommonValue() - / nrval->getCommonValue(); break; - case MOD: assert(0); - } - std::unique_ptr<Value> result(new FloatValue(res)); - out << "Performed float operation " << nlval << " " - << getOperatorName() << " " << nrval << " = " << *result - << "\n"; - return result; - } - } - case MOD: - { - if (lval->getType() == Value::Integer && - rval->getType() == Value::Integer) - { - IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); - IntegerValue& irval(static_cast<IntegerValue&>(*rval)); - std::unique_ptr<Value> result(new IntegerValue( - ilval.getValue() % irval.getValue(), false)); - out << "Performed integer operation " << ilval << " " - << getOperatorName() << " " << irval << " = " << *result - << "\n"; - return result; - } - } - } - out << "Failed to do operation " << getOperatorName() - << " on values of type " << lval->getType() << " and " - << rval->getType() << ". Resolving invalid.\n"; - return std::unique_ptr<Value>(new InvalidValue); -} - - -void -ArithmeticValueNode::visit(Visitor &visitor) const -{ - visitor.visitArithmeticValueNode(*this); -} - - -void -ArithmeticValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - if (hadParentheses()) out << '('; - _left->print(out, verbose, indent); - switch (_operator) { - case ADD: out << " + "; break; - case SUB: out << " - "; break; - case MUL: out << " * "; break; - case DIV: out << " / "; break; - case MOD: out << " % "; break; - } - _right->print(out, verbose, indent); - if (hadParentheses()) out << ')'; } -} // select -} // document diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h index d56b1468508..f36085f769d 100644 --- a/document/src/vespa/document/select/valuenode.h +++ b/document/src/vespa/document/select/valuenode.h @@ -13,18 +13,11 @@ #pragma once #include "value.h" -#include "context.h" -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> -#include <vespa/document/bucket/bucketdistribution.h> -#include <vespa/document/bucket/bucketidfactory.h> -namespace document { +namespace document::select { -class Document; -class Field; - -namespace select { +class Context; +class Visitor; class ValueNode : public Printable { @@ -38,17 +31,10 @@ public: void clearParentheses() { _parentheses = false; } bool hadParentheses() const { return _parentheses; } - virtual std::unique_ptr<Value> - getValue(const Context& context) const = 0; - - virtual std::unique_ptr<Value> - traceValue(const Context &context, std::ostream &out) const { - return defaultTrace(getValue(context), out); - } - - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override = 0; + virtual std::unique_ptr<Value> getValue(const Context& context) const = 0; virtual void visit(Visitor&) const = 0; virtual ValueNode::UP clone() const = 0; + virtual std::unique_ptr<Value> traceValue(const Context &context, std::ostream &out) const; private: bool _parentheses; // Set to true if parentheses was used around this part // Set such that we can recreate original query in print. @@ -64,341 +50,5 @@ protected: std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const; }; -class InvalidValueNode : public ValueNode -{ - vespalib::string _name; -public: - InvalidValueNode(const vespalib::stringref & name); - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new InvalidValue()); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new InvalidValueNode(_name)); - } -}; - -class NullValueNode : public ValueNode -{ - vespalib::string _name; -public: - NullValueNode(const vespalib::stringref & name); - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new NullValue()); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new NullValueNode(_name)); - } -}; - -class StringValueNode : public ValueNode -{ - vespalib::string _value; -public: - StringValueNode(const vespalib::stringref & val); - - const vespalib::string& getValue() const { return _value; } - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new StringValue(_value)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new StringValueNode(_value)); - } -}; - -class IntegerValueNode : public ValueNode -{ - int64_t _value; - bool _isBucketValue; -public: - IntegerValueNode(int64_t val, bool isBucketValue) - : _value(val), _isBucketValue(isBucketValue) {} - - int64_t getValue() const { return _value; } - - virtual std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new IntegerValue(_value, _isBucketValue)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new IntegerValueNode(_value, _isBucketValue)); - } -}; - -class CurrentTimeValueNode : public ValueNode -{ -public: - int64_t getValue() const; - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new IntegerValue(getValue(), false)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new CurrentTimeValueNode); - } -}; - -class VariableValueNode : public ValueNode -{ - vespalib::string _value; -public: - VariableValueNode(const vespalib::string & variableName) : _value(variableName) {} - - const vespalib::string& getVariableName() const { return _value; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new VariableValueNode(_value)); - } -}; - -class FloatValueNode : public ValueNode -{ - double _value; -public: - FloatValueNode(double val) : _value(val) {} - - double getValue() const { return _value; } - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new FloatValue(_value)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new FloatValueNode(_value)); - } -}; - -class FieldValueNode : public ValueNode -{ - vespalib::string _doctype; - vespalib::string _fieldExpression; - vespalib::string _fieldName; - mutable FieldPath _fieldPath; - -public: - FieldValueNode(const vespalib::string& doctype, const vespalib::string& fieldExpression); - FieldValueNode(const FieldValueNode &); - FieldValueNode & operator = (const FieldValueNode &); - FieldValueNode(FieldValueNode &&) = default; - FieldValueNode & operator = (FieldValueNode &&) = default; - ~FieldValueNode(); - - const vespalib::string& getDocType() const { return _doctype; } - const vespalib::string& getRealFieldName() const { return _fieldName; } - const vespalib::string& getFieldName() const { return _fieldExpression; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new FieldValueNode(_doctype, _fieldExpression)); - } - - static vespalib::string extractFieldName(const std::string & fieldExpression); - -private: - class IteratorHandler : public FieldValue::IteratorHandler - { - public: - IteratorHandler(); - ~IteratorHandler(); - bool hasSingleValue() const; - - std::unique_ptr<Value> getSingleValue(); - const std::vector<ArrayValue::VariableValue>& getValues(); - - private: - std::unique_ptr<Value> _firstValue; - std::vector<ArrayValue::VariableValue> _values; - - void onPrimitive(uint32_t fid, const Content & fv) override; - std::unique_ptr<Value> getInternalValue(const FieldValue& fval) const; - }; - - void initFieldPath(const DocumentType&) const; -}; - -class IdValueNode : public ValueNode -{ -public: - enum Type { SCHEME, NS, TYPE, USER, GROUP, GID, SPEC, BUCKET, ORDER, ALL }; - - IdValueNode(const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, const vespalib::stringref & type, - int widthBits = -1, int divisionBits = -1); - - Type getType() const { return _type; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - - std::unique_ptr<Value> getValue(const DocumentId& id) const; - - std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; - - std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new IdValueNode(_bucketIdFactory, _id, _typestring, _widthBits, _divisionBits)); - } - - int getWidthBits() const { return _widthBits; } - int getDivisionBits() const { return _divisionBits; } - -private: - const BucketIdFactory& _bucketIdFactory; - vespalib::string _id; - vespalib::string _typestring; - Type _type; - int _widthBits; - int _divisionBits; -}; - -class SearchColumnValueNode : public ValueNode -{ -public: - SearchColumnValueNode(const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, - int numColumns); - - int getColumns() { return _numColumns; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - std::unique_ptr<Value> getValue(const DocumentId& id) const; - std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; - std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; - - int64_t getValue(const BucketId& bucketId) const; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new SearchColumnValueNode(_bucketIdFactory, _id, _numColumns)); } -private: - const BucketIdFactory& _bucketIdFactory; - vespalib::string _id; - int _numColumns; - BucketDistribution _distribution; -}; - -class FunctionValueNode : public ValueNode -{ -public: - enum Function { LOWERCASE, HASH, ABS }; - - FunctionValueNode(const vespalib::stringref & name, std::unique_ptr<ValueNode> src); - - Function getFunction() const { return _function; } - const vespalib::string &getFunctionName(void) const { return _funcname; } - - std::unique_ptr<Value> getValue(const Context& context) const override { - return getValue(_source->getValue(context)); - } - - std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override { - return traceValue(_source->getValue(context), out); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new FunctionValueNode(_funcname, _source->clone())); - } - - const ValueNode& getChild() const { return *_source; } - -private: - Function _function; - vespalib::string _funcname; - std::unique_ptr<ValueNode> _source; - - virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> val) const; - virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> val, - std::ostream& out) const; -}; - -class ArithmeticValueNode : public ValueNode -{ -public: - enum Operator { ADD, SUB, MUL, DIV, MOD }; - - ArithmeticValueNode(std::unique_ptr<ValueNode> left, - const vespalib::stringref & op, - std::unique_ptr<ValueNode> right); - - Operator getOperator() const { return _operator; } - const char* getOperatorName() const; - - std::unique_ptr<Value> - getValue(const Context& context) const override { - return getValue(_left->getValue(context), _right->getValue(context)); - } - - std::unique_ptr<Value> - traceValue(const Context &context, std::ostream& out) const override { - return traceValue(_left->getValue(context), _right->getValue(context), out); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new ArithmeticValueNode(_left->clone(), - getOperatorName(), - _right->clone())); - } - - const ValueNode& getLeft() const { return *_left; } - const ValueNode& getRight() const { return *_right; } - -private: - Operator _operator; - std::unique_ptr<ValueNode> _left; - std::unique_ptr<ValueNode> _right; - - virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> lval, - std::unique_ptr<Value> rval) const; - virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> lval, - std::unique_ptr<Value> rval, - std::ostream&) const; -}; - -} // select -} // document diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp new file mode 100644 index 00000000000..3a4edbef093 --- /dev/null +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -0,0 +1,1183 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "valuenodes.h" +#include "visitor.h" +#include "parser.h" +#include <vespa/document/bucket/bucketdistribution.h> +#include <vespa/document/base/exceptions.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/vespalib/util/md5.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/text/lowercase.h> +#include <regex> +#include <iomanip> + + +#include <vespa/log/log.h> +LOG_SETUP(".document.select.valuenode"); + +namespace document::select { + +namespace { + static const std::regex FIELD_NAME_REGEX("^([_A-Za-z][_A-Za-z0-9]*).*"); +} + +namespace { + bool documentTypeEqualsName(const DocumentType& type, const vespalib::stringref& name) + { + if (type.getName() == name) return true; + for (std::vector<const DocumentType *>::const_iterator it + = type.getInheritedTypes().begin(); + it != type.getInheritedTypes().end(); ++it) + { + if (documentTypeEqualsName(**it, name)) return true; + } + return false; + } +} + +InvalidValueNode::InvalidValueNode(const vespalib::stringref & name) + : _name(name) +{ } + + +void +InvalidValueNode::visit(Visitor &visitor) const +{ + visitor.visitInvalidValueNode(*this); +} + + +void +InvalidValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _name; + if (hadParentheses()) out << ')'; +} + +NullValueNode::NullValueNode(const vespalib::stringref & name) + : _name(name) +{ } + + +void +NullValueNode::visit(Visitor &visitor) const +{ + visitor.visitNullValueNode(*this); +} + + +void +NullValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _name; + if (hadParentheses()) out << ')'; +} + +StringValueNode::StringValueNode(const vespalib::stringref & val) + : _value(val) +{ +} + + +void +StringValueNode::visit(Visitor &visitor) const +{ + visitor.visitStringValueNode(*this); +} + + +void +StringValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << "\"" << StringUtil::escape(_value) << "\""; + if (hadParentheses()) out << ')'; +} + + +void +IntegerValueNode::visit(Visitor &visitor) const +{ + visitor.visitIntegerValueNode(*this); +} + + +void +IntegerValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _value; + if (hadParentheses()) out << ')'; +} + +int64_t +CurrentTimeValueNode::getValue() const +{ + struct timeval mytime; + gettimeofday(&mytime, 0); + return mytime.tv_sec; +} + + +void +CurrentTimeValueNode::visit(Visitor &visitor) const +{ + visitor.visitCurrentTimeValueNode(*this); +} + + +void +CurrentTimeValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + out << "now()"; +} + +std::unique_ptr<Value> +VariableValueNode::getValue(const Context& context) const { + return context.getValue(_value); +} + +void +VariableValueNode::visit(Visitor &visitor) const +{ + visitor.visitVariableValueNode(*this); +} + + +void +VariableValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << "$" << _value; + if (hadParentheses()) out << ')'; +} + + +void +FloatValueNode::visit(Visitor &visitor) const +{ + visitor.visitFloatValueNode(*this); +} + + +void +FloatValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _value; + if (hadParentheses()) out << ')'; +} + +FieldValueNode::FieldValueNode(const vespalib::string& doctype, + const vespalib::string& fieldExpression) + : _doctype(doctype), + _fieldExpression(fieldExpression), + _fieldName(extractFieldName(fieldExpression)) +{ +} + +FieldValueNode::FieldValueNode(const FieldValueNode &) = default; +FieldValueNode & FieldValueNode::operator = (const FieldValueNode &) = default; + +FieldValueNode::~FieldValueNode() {} +vespalib::string +FieldValueNode::extractFieldName(const std::string & fieldExpression) { + std::smatch match; + + if (std::regex_match(fieldExpression, match, FIELD_NAME_REGEX) && match[1].matched) { + return vespalib::string(match[1].first, match[1].second); + } + + throw ParsingFailedException("Fatal: could not extract field name from field expression '" + fieldExpression + "'"); +} + +namespace { + +class IteratorHandler : public fieldvalue::IteratorHandler { +public: + IteratorHandler(); + ~IteratorHandler(); + bool hasSingleValue() const; + std::unique_ptr<Value> getSingleValue(); + const std::vector<ArrayValue::VariableValue> &getValues(); + +private: + std::unique_ptr<Value> _firstValue; + std::vector<ArrayValue::VariableValue> _values; + + void onPrimitive(uint32_t fid, const Content &fv) override; + std::unique_ptr<Value> getInternalValue(const FieldValue &fval) const; +}; + +IteratorHandler::IteratorHandler() { } +IteratorHandler::~IteratorHandler() { } + +bool +IteratorHandler::hasSingleValue() const { + return _firstValue.get() && (_values.size() == 0); +} + +std::unique_ptr<Value> +IteratorHandler::getSingleValue() { + return std::move(_firstValue); +} + +const std::vector<ArrayValue::VariableValue>& +IteratorHandler::getValues() { + if (_firstValue.get()) { + _values.insert(_values.begin(), ArrayValue::VariableValue(fieldvalue::VariableMap(), Value::SP(_firstValue.release()))); + } + + return _values; +} + +void +IteratorHandler::onPrimitive(uint32_t fid, const Content& fv) { + (void) fid; + if (!_firstValue && getVariables().empty()) { + _firstValue = getInternalValue(fv.getValue()); + } else { + _values.emplace_back(getVariables(), Value::SP(getInternalValue(fv.getValue()).release())); + } +} + +std::unique_ptr<Value> +IteratorHandler::getInternalValue(const FieldValue& fval) const +{ + switch(fval.getClass().id()) { + case document::IntFieldValue::classId: + { + const IntFieldValue& val(dynamic_cast<const IntFieldValue&>(fval)); + return std::make_unique<IntegerValue>(val.getAsInt(), false); + } + case document::ByteFieldValue::classId: + { + const ByteFieldValue& val(dynamic_cast<const ByteFieldValue&>(fval)); + return std::make_unique<IntegerValue>(val.getAsByte(), false); + } + case LongFieldValue::classId: + { + const LongFieldValue& val(dynamic_cast<const LongFieldValue&>(fval)); + return std::make_unique<IntegerValue>(val.getAsLong(), false); + } + case FloatFieldValue::classId: + { + const FloatFieldValue& val(dynamic_cast<const FloatFieldValue&>(fval)); + return std::make_unique<FloatValue>(val.getAsFloat()); + } + case DoubleFieldValue::classId: + { + const DoubleFieldValue& val(dynamic_cast<const DoubleFieldValue&>(fval)); + return std::make_unique<FloatValue>(val.getAsDouble()); + } + case StringFieldValue::classId: + { + const StringFieldValue& val(dynamic_cast<const StringFieldValue&>(fval)); + return std::make_unique<StringValue>(val.getAsString()); + } + case ArrayFieldValue::classId: + { + const ArrayFieldValue& val(dynamic_cast<const ArrayFieldValue&>(fval)); + if (val.size() == 0) { + return std::make_unique<NullValue>(); + } else { + std::vector<ArrayValue::VariableValue> values; + // TODO: Array comparison. + return std::make_unique<ArrayValue>(values); + } + } + case StructFieldValue::classId: + { + const StructFieldValue& val(dynamic_cast<const StructFieldValue&>(fval)); + if (val.empty()) { + return std::make_unique<NullValue>(); + } else { + StructValue::ValueMap values; + for (StructFieldValue::const_iterator it(val.begin()); it != val.end(); ++it) { + FieldValue::UP fv(val.getValue(it.field())); + values[it.field().getName()] = Value::SP(getInternalValue(*fv).release()); + } + return std::make_unique<StructValue>(values); + } + } + case MapFieldValue::classId: + { + const MapFieldValue& val(static_cast<const MapFieldValue&>(fval)); + if (val.isEmpty()) { + return std::make_unique<NullValue>(); + } else { + std::vector<ArrayValue::VariableValue> values; + // TODO: Map comparison + return std::make_unique<ArrayValue>(values); + } + } + } + LOG(warning, "Tried to use unsupported datatype %s in field comparison", + fval.getDataType()->toString().c_str()); + return std::make_unique<InvalidValue>(); +} + +} + +void +FieldValueNode::initFieldPath(const DocumentType& type) const { + if (_fieldPath.size() == 0) { + FieldPath::UP path(type.buildFieldPath(_fieldExpression)); + if (!path.get()) { + throw FieldNotFoundException( + vespalib::make_string("Could not create field path for doc type: '%s' field: '%s'", + type.toString().c_str(), _fieldExpression.c_str()), + VESPA_STRLOC); + } + _fieldPath = *path; + } +} + +std::unique_ptr<Value> +FieldValueNode::getValue(const Context& context) const +{ + if (context._doc == NULL) { + return std::unique_ptr<Value>(new InvalidValue()); + } + + const Document& doc = *context._doc; + + if (!documentTypeEqualsName(doc.getType(), _doctype)) { + return std::unique_ptr<Value>(new InvalidValue()); + } + try{ + initFieldPath(doc.getType()); + + IteratorHandler handler; + doc.iterateNested(_fieldPath.getFullRange(), handler); + + if (handler.hasSingleValue()) { + return handler.getSingleValue(); + } else { + const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); + + if (values.size() == 0) { + return std::unique_ptr<Value>(new NullValue()); + } else { + return std::unique_ptr<Value>(new ArrayValue(handler.getValues())); + } + } + } catch (vespalib::IllegalArgumentException& e) { + LOG(warning, "Caught exception while fetching field from document: %s", e.what()); + return std::unique_ptr<Value>(new InvalidValue()); + } catch (FieldNotFoundException& e) { + LOG(warning, "Tried to compare to field %s, not found in document type", _fieldExpression.c_str()); + return std::unique_ptr<Value>(new InvalidValue()); + } +} + +void +FieldValueNode::visit(Visitor &visitor) const +{ + visitor.visitFieldValueNode(*this); +} + + +void +FieldValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _doctype << "." << _fieldExpression; + if (hadParentheses()) out << ')'; +} + + +std::unique_ptr<Value> +FieldValueNode::traceValue(const Context &context, std::ostream& out) const +{ + if (context._doc == NULL) { + return defaultTrace(getValue(context), out); + } + const Document &doc(*context._doc); + if (!documentTypeEqualsName(doc.getType(), _doctype)) { + out << "Document is of type " << doc.getType() << " which isn't a " + << _doctype << " document, thus resolving invalid.\n"; + return std::unique_ptr<Value>(new InvalidValue()); + } + try{ + initFieldPath(doc.getType()); + + IteratorHandler handler; + doc.iterateNested(_fieldPath.getFullRange(), handler); + + if (handler.hasSingleValue()) { + return handler.getSingleValue(); + } else { + const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); + + if (values.size() == 0) { + return std::unique_ptr<Value>(new NullValue()); + } else { + return std::unique_ptr<Value>(new ArrayValue(handler.getValues())); + } + } + } catch (FieldNotFoundException& e) { + LOG(warning, "Tried to compare to field %s, not found in document type", + _fieldExpression.c_str()); + out << "Field not found in document type " << doc.getType() + << ". Returning invalid.\n"; + return std::unique_ptr<Value>(new InvalidValue()); + } +} + +IdValueNode::IdValueNode(const BucketIdFactory& bucketIdFactory, + const vespalib::stringref & name, const vespalib::stringref & type, + int widthBits, int divisionBits) + : _bucketIdFactory(bucketIdFactory), + _id(name), + _typestring(type), + _type(ALL), + _widthBits(widthBits), + _divisionBits(divisionBits) +{ + if (type.length() > 2) switch (type[0]) { + case 'b': _type = BUCKET; + break; + case 'n': _type = NS; + break; + case 'g': + if (type[1] == 'r') { + _type = GROUP; + } else if (type[1] == 'i') { + _type = GID; + } + break; + case 's': { + if (type[1] == 'c') { _type = SCHEME; } else { _type = SPEC; } + break; + } + case 't': + _type = TYPE; + break; + case 'u': + _type = USER; + break; + case 'o': + _type = ORDER; + break; + } +} + + +std::unique_ptr<Value> +IdValueNode::getValue(const Context& context) const +{ + if (context._doc != NULL) { + return getValue(context._doc->getId()); + } else if (context._docId != NULL) { + return getValue(*context._docId); + } else { + return getValue(context._docUpdate->getId()); + } +} + + +std::unique_ptr<Value> +IdValueNode::getValue(const DocumentId& id) const +{ + vespalib::string value; + switch (_type) { + case BUCKET: + return std::unique_ptr<Value>( + new IntegerValue( + _bucketIdFactory.getBucketId(id).getId(), true)); + case NS: + value = id.getScheme().getNamespace(); break; + case SCHEME: + value = id.getScheme().getTypeName(id.getScheme().getType()); + break; + case TYPE: + if (id.getScheme().hasDocType()) { + value = id.getScheme().getDocType(); + } else { + return std::unique_ptr<Value>(new InvalidValue); + } + break; + case SPEC: + value = id.getScheme().getNamespaceSpecific(); + break; + case ALL: + value = id.getScheme().toString(); + break; + case GROUP: + if (id.getScheme().hasGroup()) { + value = id.getScheme().getGroup(); + } else { + fprintf(stderr, "***** Returning invalid value for %s\n", + id.toString().c_str()); + return std::unique_ptr<Value>(new InvalidValue); + } + break; + case GID: + value = id.getGlobalId().toString(); + break; + case ORDER: + if (id.getScheme().getType() == IdString::ORDERDOC) { + const OrderDocIdString& ods( + static_cast<const OrderDocIdString&>(id.getScheme())); + if (ods.getWidthBits() == _widthBits + && ods.getDivisionBits() == _divisionBits) + { + return std::unique_ptr<Value>(new IntegerValue( + static_cast<const OrderDocIdString&>(id.getScheme()) + .getOrdering(), false)); + } + } + return std::unique_ptr<Value>(new InvalidValue()); + case USER: + if (id.getScheme().hasNumber()) { + return std::unique_ptr<Value>( + new IntegerValue(id.getScheme().getNumber(), false)); + } else { + return std::unique_ptr<Value>(new InvalidValue); + } + } + + return std::unique_ptr<Value>(new StringValue(value)); +} + + +std::unique_ptr<Value> +IdValueNode::traceValue(const Context& context, + std::ostream &out) const +{ + if (context._doc != NULL) { + return traceValue(context._doc->getId(), out); + } else if (context._docId != NULL) { + return traceValue(*context._docId, out); + } else { + return traceValue(context._docUpdate->getId(), out); + } +} + + +std::unique_ptr<Value> +IdValueNode::traceValue(const DocumentId& id, std::ostream& out) const +{ + vespalib::string value; + switch (_type) { + case BUCKET: + { + document::BucketId bucket(_bucketIdFactory.getBucketId(id)); + std::unique_ptr<Value> result( + new IntegerValue(bucket.getId(), true)); + out << "Found id.bucket specification. Resolved to " + << bucket.toString() << ".\n"; + return result; + } + case NS: + value = id.getScheme().getNamespace(); + out << "Resolved id.namespace to value\"" << value << "\".\n"; + break; + case SCHEME: + value = id.getScheme().getTypeName(id.getScheme().getType()); + out << "Resolved id.scheme to value\"" << value << "\".\n"; + break; + case TYPE: + if (id.getScheme().hasDocType()) { + value = id.getScheme().getDocType(); + out << "Resolved id.type to value\"" << value << "\".\n"; + } else { + out << "Could not resolve type of doc " << id << ".\n"; + return std::unique_ptr<Value>(new InvalidValue); + } + break; + case SPEC: + value = id.getScheme().getNamespaceSpecific(); + out << "Resolved id.specific to value\"" << value << "\".\n"; + break; + case ALL: + value = id.getScheme().toString(); + out << "Resolved id to \"" << value << "\".\n"; + break; + case GROUP: + if (id.getScheme().hasGroup()) { + value = id.getScheme().getGroup(); + out << "Resolved group of doc (type " << id.getScheme().getType() + << ") to \"" << value << "\".\n"; + } else { + out << "Can't resolve group of doc \"" << id << "\".\n"; + return std::unique_ptr<Value>(new InvalidValue); + } + break; + case GID: + value = id.getGlobalId().toString(); + out << "Resolved gid to \"" << value << "\".\n"; + break; + case ORDER: + if (id.getScheme().getType() == IdString::ORDERDOC) { + const OrderDocIdString& ods( + static_cast<const OrderDocIdString&>(id.getScheme())); + if (ods.getWidthBits() == _widthBits + && ods.getDivisionBits() == _divisionBits) + { + std::unique_ptr<Value> result(new IntegerValue( + static_cast<const OrderDocIdString&>(id.getScheme()) + .getOrdering(), false)); + out << "Resolved id.order to int " << *result << "\n"; + return result; + } + } + out << "Could not resolve id.order(" << _widthBits << ", " + << _divisionBits << ") of doc " << id << ".\n"; + return std::unique_ptr<Value>(new InvalidValue()); + case USER: + if (id.getScheme().hasNumber()) { + std::unique_ptr<Value> result( + new IntegerValue(id.getScheme().getNumber(), false)); + out << "Resolved user of doc type " << id.getScheme().getType() + << " to " << *result << ".\n"; + return result; + } else { + out << "Could not resolve user of doc " << id << ".\n"; + return std::unique_ptr<Value>(new InvalidValue); + } + } + + return std::unique_ptr<Value>(new StringValue(value)); +} + + +void +IdValueNode::visit(Visitor &visitor) const +{ + visitor.visitIdValueNode(*this); +} + + +void +IdValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _id; + if (_type != ALL) { + out << '.' << _typestring; + } + if (_type == ORDER) { + out << "(" << _widthBits << "," << _divisionBits << ")"; + } + if (hadParentheses()) out << ')'; +} + +SearchColumnValueNode::SearchColumnValueNode( + const BucketIdFactory& bucketIdFactory, + const vespalib::stringref & name, int numColumns) + : _bucketIdFactory(bucketIdFactory), + _id(name), + _numColumns(numColumns), + _distribution(std::make_unique<BucketDistribution>(_numColumns, 16)) +{ +} + +int64_t +SearchColumnValueNode::getValue(const BucketId& id) const +{ + return _distribution->getColumn(id); +} + + +std::unique_ptr<Value> +SearchColumnValueNode::getValue(const Context& context) const +{ + if (context._doc != NULL) { + return getValue(context._doc->getId()); + } else if (context._docId != NULL) { + return getValue(*context._docId); + } else { + return getValue(context._docUpdate->getId()); + } +} + + +std::unique_ptr<Value> +SearchColumnValueNode::getValue(const DocumentId& id) const +{ + return std::unique_ptr<Value>(new IntegerValue( + getValue(_bucketIdFactory.getBucketId(id)), false)); +} + + +std::unique_ptr<Value> +SearchColumnValueNode::traceValue(const Context& context, + std::ostream &out) const +{ + if (context._doc != NULL) { + return traceValue(context._doc->getId(), out); + } else if (context._docId != NULL) { + return traceValue(*context._docId, out); + } else { + return traceValue(context._docUpdate->getId(), out); + } +} + + +std::unique_ptr<Value> +SearchColumnValueNode::traceValue(const DocumentId& id, + std::ostream& out) const +{ + std::unique_ptr<Value> result(new IntegerValue( + getValue(_bucketIdFactory.getBucketId(id)), false)); + out << "Resolved search column of doc \"" << id << "\" to " << *result + << "\n"; + return result; +} + + +void +SearchColumnValueNode::visit(Visitor &visitor) const +{ + visitor.visitSearchColumnValueNode(*this); +} + + +void +SearchColumnValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _id; + out << '.' << _numColumns; + if (hadParentheses()) out << ')'; +} + +namespace { + union HashUnion { + unsigned char _key[16]; + int64_t _hash[2]; + }; + int64_t hash(const void* data, uint32_t len) { + HashUnion hash; + fastc_md5sum((const unsigned char*) data, len, hash._key); + return hash._hash[0]; + } +} + +FunctionValueNode::FunctionValueNode(const vespalib::stringref & name, + std::unique_ptr<ValueNode> src) + : _function(), + _funcname(name), + _source(std::move(src)) +{ + if (name == "lowercase") { + _function = LOWERCASE; + } else if (name == "hash") { + _function = HASH; + } else if (name == "abs") { + _function = ABS; + } else { + throw ParsingFailedException("No function '"+name+"' exist.", + VESPA_STRLOC); + } +} + +std::unique_ptr<Value> +FunctionValueNode::getValue(std::unique_ptr<Value> val) const +{ + switch (val->getType()) { + case Value::String: + { + StringValue& sval(static_cast<StringValue&>(*val)); + if (_function == LOWERCASE) { + return std::unique_ptr<Value>(new StringValue( + vespalib::LowerCase::convert(sval.getValue()))); + } else if (_function == HASH) { + return std::unique_ptr<Value>(new IntegerValue( + hash(sval.getValue().c_str(), sval.getValue().size()), + false)); + } + break; + } + case Value::Float: + { + FloatValue& fval(static_cast<FloatValue&>(*val)); + if (_function == HASH) { + FloatValue::ValueType ffval = fval.getValue(); + return std::unique_ptr<Value>(new IntegerValue( + hash(&ffval, sizeof(ffval)), false)); + } else if (_function == ABS) { + FloatValue::ValueType ffval = fval.getValue(); + if (ffval < 0) ffval *= -1; + return std::unique_ptr<Value>(new FloatValue(ffval)); + } + break; + } + case Value::Integer: + { + IntegerValue& ival(static_cast<IntegerValue&>(*val)); + if (_function == HASH) { + IntegerValue::ValueType iival = ival.getValue(); + return std::unique_ptr<Value>(new IntegerValue( + hash(&iival, sizeof(iival)), false)); + } else if (_function == ABS) { + IntegerValue::ValueType iival = ival.getValue(); + if (iival < 0) iival *= -1; + return std::unique_ptr<Value>(new IntegerValue(iival, false)); + } + break; + } + case Value::Bucket: + { + throw ParsingFailedException( + "No functioncalls are allowed on value of type bucket", + VESPA_STRLOC); + break; + } + + case Value::Array: break; + case Value::Struct: break; + case Value::Invalid: break; + case Value::Null: break; + } + return std::unique_ptr<Value>(new InvalidValue); +} + +std::unique_ptr<Value> +FunctionValueNode::traceValue(std::unique_ptr<Value> val, + std::ostream& out) const +{ + switch (val->getType()) { + case Value::String: + { + StringValue& sval(static_cast<StringValue&>(*val)); + if (_function == LOWERCASE) { + std::unique_ptr<Value> result(new StringValue( + vespalib::LowerCase::convert(sval.getValue()))); + out << "Performed lowercase function on '" << sval + << "' => '" << *result << "'.\n"; + return result; + } else if (_function == HASH) { + std::unique_ptr<Value> result(new IntegerValue( + hash(sval.getValue().c_str(), sval.getValue().size()), + false)); + out << "Performed hash on string '" << sval << "' -> " + << *result << "\n"; + return result; + } + break; + } + case Value::Float: + { + FloatValue& fval(static_cast<FloatValue&>(*val)); + if (_function == HASH) { + FloatValue::ValueType ffval = fval.getValue(); + std::unique_ptr<Value> result(new IntegerValue( + hash(&ffval, sizeof(ffval)), false)); + out << "Performed hash on float " << ffval << " -> " << *result + << "\n"; + return result; + } else if (_function == ABS) { + FloatValue::ValueType ffval = fval.getValue(); + if (ffval < 0) ffval *= -1; + out << "Performed abs on float " << fval.getValue() << " -> " + << ffval << "\n"; + return std::unique_ptr<Value>(new FloatValue(ffval)); + } + break; + } + case Value::Integer: + { + IntegerValue& ival(static_cast<IntegerValue&>(*val)); + if (_function == HASH) { + IntegerValue::ValueType iival = ival.getValue(); + std::unique_ptr<Value> result(new IntegerValue( + hash(&iival, sizeof(iival)), false)); + out << "Performed hash on float " << iival << " -> " << *result + << "\n"; + return result; + } else if (_function == ABS) { + IntegerValue::ValueType iival = ival.getValue(); + if (iival < 0) iival *= -1; + out << "Performed abs on integer " << ival.getValue() << " -> " + << iival << "\n"; + return std::unique_ptr<Value>(new IntegerValue(iival, false)); + } + break; + } + case Value::Bucket: break; + case Value::Array: break; + case Value::Struct: break; + case Value::Invalid: break; + case Value::Null: break; + } + out << "Cannot use function " << _function << " on a value of type " + << val->getType() << ". Resolving invalid.\n"; + return std::unique_ptr<Value>(new InvalidValue); +} + + +void +FunctionValueNode::visit(Visitor &visitor) const +{ + visitor.visitFunctionValueNode(*this); +} + + +void +FunctionValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + if (hadParentheses()) out << '('; + _source->print(out, verbose, indent); + out << '.' << _funcname << "()"; + if (hadParentheses()) out << ')'; +} + +ArithmeticValueNode::ArithmeticValueNode( + std::unique_ptr<ValueNode> left, const vespalib::stringref & op, + std::unique_ptr<ValueNode> right) + : _operator(), + _left(std::move(left)), + _right(std::move(right)) +{ + if (op.size() == 1) switch (op[0]) { + case '+': _operator = ADD; return; + case '-': _operator = SUB; return; + case '*': _operator = MUL; return; + case '/': _operator = DIV; return; + case '%': _operator = MOD; return; + } + throw ParsingFailedException( + "Arithmetic operator '"+op+"' does not exist.", VESPA_STRLOC); +} + +const char* +ArithmeticValueNode::getOperatorName() const +{ + switch (_operator) { + case ADD: return "+"; + case SUB: return "-"; + case MUL: return "*"; + case DIV: return "/"; + case MOD: return "%"; + } + return "UNKNOWN"; +} + + + +std::unique_ptr<Value> +ArithmeticValueNode::getValue(std::unique_ptr<Value> lval, + std::unique_ptr<Value> rval) const +{ + switch (_operator) { + case ADD: + { + if (lval->getType() == Value::String && + rval->getType() == Value::String) + { + StringValue& slval(static_cast<StringValue&>(*lval)); + StringValue& srval(static_cast<StringValue&>(*rval)); + return std::unique_ptr<Value>(new StringValue( + slval.getValue() + srval.getValue())); + } + } + case SUB: + case MUL: + case DIV: + { + if (lval->getType() == Value::Integer && + rval->getType() == Value::Integer) + { + IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); + IntegerValue& irval(static_cast<IntegerValue&>(*rval)); + IntegerValue::ValueType res = 0; + switch (_operator) { + case ADD: res = ilval.getValue() + irval.getValue(); break; + case SUB: res = ilval.getValue() - irval.getValue(); break; + case MUL: res = ilval.getValue() * irval.getValue(); break; + case DIV: + if (irval.getValue() != 0) { + res = ilval.getValue() / irval.getValue(); + } else { + throw vespalib::IllegalArgumentException("Division by zero"); + } + break; + case MOD: assert(0); + } + return std::unique_ptr<Value>(new IntegerValue(res, false)); + } + NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get())); + NumberValue* nrval(dynamic_cast<NumberValue*>(rval.get())); + if (nlval != 0 && nrval != 0) { + NumberValue::CommonValueType res = 0; + switch (_operator) { + case ADD: res = nlval->getCommonValue() + + nrval->getCommonValue(); break; + case SUB: res = nlval->getCommonValue() + - nrval->getCommonValue(); break; + case MUL: res = nlval->getCommonValue() + * nrval->getCommonValue(); break; + case DIV: + if (nrval->getCommonValue() != 0) { + res = nlval->getCommonValue() + / nrval->getCommonValue(); + } else { + throw vespalib::IllegalArgumentException("Division by zero"); + } + break; + case MOD: assert(0); + } + return std::unique_ptr<Value>(new FloatValue(res)); + } + } + case MOD: + { + if (lval->getType() == Value::Integer && + rval->getType() == Value::Integer) + { + IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); + IntegerValue& irval(static_cast<IntegerValue&>(*rval)); + if (irval.getValue() != 0) { + return std::unique_ptr<Value>(new IntegerValue(ilval.getValue() % irval.getValue(), false)); + } else { + throw vespalib::IllegalArgumentException("Division by zero"); + } + } + } + } + return std::unique_ptr<Value>(new InvalidValue); +} + +std::unique_ptr<Value> +ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval, + std::unique_ptr<Value> rval, + std::ostream& out) const +{ + switch (_operator) { + case ADD: + { + if (lval->getType() == Value::String && + rval->getType() == Value::String) + { + StringValue& slval(static_cast<StringValue&>(*lval)); + StringValue& srval(static_cast<StringValue&>(*rval)); + std::unique_ptr<Value> result(new StringValue( + slval.getValue() + srval.getValue())); + out << "Appended strings '" << slval << "' + '" << srval + << "' -> '" << *result << "'.\n"; + return result; + } + } + case SUB: + case MUL: + case DIV: + { + if (lval->getType() == Value::Integer && + rval->getType() == Value::Integer) + { + IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); + IntegerValue& irval(static_cast<IntegerValue&>(*rval)); + IntegerValue::ValueType res = 0; + switch (_operator) { + case ADD: res = ilval.getValue() + irval.getValue(); break; + case SUB: res = ilval.getValue() - irval.getValue(); break; + case MUL: res = ilval.getValue() * irval.getValue(); break; + case DIV: res = ilval.getValue() / irval.getValue(); break; + case MOD: assert(0); + } + std::unique_ptr<Value> result(new IntegerValue(res, false)); + out << "Performed integer operation " << ilval << " " + << getOperatorName() << " " << irval << " = " << *result + << "\n"; + return result; + } + NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get())); + NumberValue* nrval(dynamic_cast<NumberValue*>(lval.get())); + if (nlval != 0 && nrval != 0) { + NumberValue::CommonValueType res = 0; + switch (_operator) { + case ADD: res = nlval->getCommonValue() + + nrval->getCommonValue(); break; + case SUB: res = nlval->getCommonValue() + - nrval->getCommonValue(); break; + case MUL: res = nlval->getCommonValue() + * nrval->getCommonValue(); break; + case DIV: res = nlval->getCommonValue() + / nrval->getCommonValue(); break; + case MOD: assert(0); + } + std::unique_ptr<Value> result(new FloatValue(res)); + out << "Performed float operation " << nlval << " " + << getOperatorName() << " " << nrval << " = " << *result + << "\n"; + return result; + } + } + case MOD: + { + if (lval->getType() == Value::Integer && + rval->getType() == Value::Integer) + { + IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); + IntegerValue& irval(static_cast<IntegerValue&>(*rval)); + std::unique_ptr<Value> result(new IntegerValue( + ilval.getValue() % irval.getValue(), false)); + out << "Performed integer operation " << ilval << " " + << getOperatorName() << " " << irval << " = " << *result + << "\n"; + return result; + } + } + } + out << "Failed to do operation " << getOperatorName() + << " on values of type " << lval->getType() << " and " + << rval->getType() << ". Resolving invalid.\n"; + return std::unique_ptr<Value>(new InvalidValue); +} + + +void +ArithmeticValueNode::visit(Visitor &visitor) const +{ + visitor.visitArithmeticValueNode(*this); +} + + +void +ArithmeticValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + if (hadParentheses()) out << '('; + _left->print(out, verbose, indent); + switch (_operator) { + case ADD: out << " + "; break; + case SUB: out << " - "; break; + case MUL: out << " * "; break; + case DIV: out << " / "; break; + case MOD: out << " % "; break; + } + _right->print(out, verbose, indent); + if (hadParentheses()) out << ')'; +} + +} + diff --git a/document/src/vespa/document/select/valuenodes.h b/document/src/vespa/document/select/valuenodes.h new file mode 100644 index 00000000000..98f80e2904a --- /dev/null +++ b/document/src/vespa/document/select/valuenodes.h @@ -0,0 +1,348 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class document::select::ValueNode + * @ingroup select + * + * @brief Node representing a value in the tree + * + * @author H�kon Humberset + * @date 2007-04-20 + * @version $Id$ + */ + +#pragma once + +#include "valuenode.h" +#include <vespa/document/base/fieldpath.h> + +namespace document { + +class BucketDistribution; +class BucketIdFactory; +class DocumentId; +class BucketId; +class DocumentType; + +namespace select { + +class InvalidValueNode : public ValueNode +{ + vespalib::string _name; +public: + InvalidValueNode(const vespalib::stringref & name); + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new InvalidValue()); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new InvalidValueNode(_name)); + } +}; + +class NullValueNode : public ValueNode +{ + vespalib::string _name; +public: + NullValueNode(const vespalib::stringref & name); + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new NullValue()); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new NullValueNode(_name)); + } +}; + +class StringValueNode : public ValueNode +{ + vespalib::string _value; +public: + StringValueNode(const vespalib::stringref & val); + + const vespalib::string& getValue() const { return _value; } + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new StringValue(_value)); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new StringValueNode(_value)); + } +}; + +class IntegerValueNode : public ValueNode +{ + int64_t _value; + bool _isBucketValue; +public: + IntegerValueNode(int64_t val, bool isBucketValue) + : _value(val), _isBucketValue(isBucketValue) {} + + int64_t getValue() const { return _value; } + + virtual std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new IntegerValue(_value, _isBucketValue)); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new IntegerValueNode(_value, _isBucketValue)); + } +}; + +class CurrentTimeValueNode : public ValueNode +{ +public: + int64_t getValue() const; + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new IntegerValue(getValue(), false)); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new CurrentTimeValueNode); + } +}; + +class VariableValueNode : public ValueNode +{ + vespalib::string _value; +public: + VariableValueNode(const vespalib::string & variableName) : _value(variableName) {} + + const vespalib::string& getVariableName() const { return _value; } + + std::unique_ptr<Value> getValue(const Context& context) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new VariableValueNode(_value)); + } +}; + +class FloatValueNode : public ValueNode +{ + double _value; +public: + FloatValueNode(double val) : _value(val) {} + + double getValue() const { return _value; } + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new FloatValue(_value)); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new FloatValueNode(_value)); + } +}; + +class FieldValueNode : public ValueNode +{ + vespalib::string _doctype; + vespalib::string _fieldExpression; + vespalib::string _fieldName; + mutable FieldPath _fieldPath; + +public: + FieldValueNode(const vespalib::string& doctype, const vespalib::string& fieldExpression); + FieldValueNode(const FieldValueNode &); + FieldValueNode & operator = (const FieldValueNode &); + FieldValueNode(FieldValueNode &&) = default; + FieldValueNode & operator = (FieldValueNode &&) = default; + ~FieldValueNode(); + + const vespalib::string& getDocType() const { return _doctype; } + const vespalib::string& getRealFieldName() const { return _fieldName; } + const vespalib::string& getFieldName() const { return _fieldExpression; } + + std::unique_ptr<Value> getValue(const Context& context) const override; + std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new FieldValueNode(_doctype, _fieldExpression)); + } + + static vespalib::string extractFieldName(const std::string & fieldExpression); + +private: + + void initFieldPath(const DocumentType&) const; +}; + +class IdValueNode : public ValueNode +{ +public: + enum Type { SCHEME, NS, TYPE, USER, GROUP, GID, SPEC, BUCKET, ORDER, ALL }; + + IdValueNode(const BucketIdFactory& bucketIdFactory, + const vespalib::stringref & name, const vespalib::stringref & type, + int widthBits = -1, int divisionBits = -1); + + Type getType() const { return _type; } + + std::unique_ptr<Value> getValue(const Context& context) const override; + + std::unique_ptr<Value> getValue(const DocumentId& id) const; + + std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; + + std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new IdValueNode(_bucketIdFactory, _id, _typestring, _widthBits, _divisionBits)); + } + + int getWidthBits() const { return _widthBits; } + int getDivisionBits() const { return _divisionBits; } + +private: + const BucketIdFactory& _bucketIdFactory; + vespalib::string _id; + vespalib::string _typestring; + Type _type; + int _widthBits; + int _divisionBits; +}; + +class SearchColumnValueNode : public ValueNode +{ +public: + SearchColumnValueNode(const BucketIdFactory& bucketIdFactory, + const vespalib::stringref & name, + int numColumns); + + int getColumns() { return _numColumns; } + + std::unique_ptr<Value> getValue(const Context& context) const override; + std::unique_ptr<Value> getValue(const DocumentId& id) const; + std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; + std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; + + int64_t getValue(const BucketId& bucketId) const; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new SearchColumnValueNode(_bucketIdFactory, _id, _numColumns)); +} + +private: + const BucketIdFactory& _bucketIdFactory; + vespalib::string _id; + int _numColumns; + std::unique_ptr<BucketDistribution> _distribution; +}; + +class FunctionValueNode : public ValueNode +{ +public: + enum Function { LOWERCASE, HASH, ABS }; + + FunctionValueNode(const vespalib::stringref & name, std::unique_ptr<ValueNode> src); + + Function getFunction() const { return _function; } + const vespalib::string &getFunctionName(void) const { return _funcname; } + + std::unique_ptr<Value> getValue(const Context& context) const override { + return getValue(_source->getValue(context)); + } + + std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override { + return traceValue(_source->getValue(context), out); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new FunctionValueNode(_funcname, _source->clone())); + } + + const ValueNode& getChild() const { return *_source; } + +private: + Function _function; + vespalib::string _funcname; + std::unique_ptr<ValueNode> _source; + + virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> val) const; + virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> val, + std::ostream& out) const; +}; + +class ArithmeticValueNode : public ValueNode +{ +public: + enum Operator { ADD, SUB, MUL, DIV, MOD }; + + ArithmeticValueNode(std::unique_ptr<ValueNode> left, + const vespalib::stringref & op, + std::unique_ptr<ValueNode> right); + + Operator getOperator() const { return _operator; } + const char* getOperatorName() const; + + std::unique_ptr<Value> + getValue(const Context& context) const override { + return getValue(_left->getValue(context), _right->getValue(context)); + } + + std::unique_ptr<Value> + traceValue(const Context &context, std::ostream& out) const override { + return traceValue(_left->getValue(context), _right->getValue(context), out); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new ArithmeticValueNode(_left->clone(), + getOperatorName(), + _right->clone())); + } + + const ValueNode& getLeft() const { return *_left; } + const ValueNode& getRight() const { return *_right; } + +private: + Operator _operator; + std::unique_ptr<ValueNode> _left; + std::unique_ptr<ValueNode> _right; + + virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> lval, + std::unique_ptr<Value> rval) const; + virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> lval, + std::unique_ptr<Value> rval, + std::ostream&) const; +}; + +} // select +} // document diff --git a/document/src/vespa/document/update/addfieldpathupdate.cpp b/document/src/vespa/document/update/addfieldpathupdate.cpp index 03a31d882f3..0f8b730367e 100644 --- a/document/src/vespa/document/update/addfieldpathupdate.cpp +++ b/document/src/vespa/document/update/addfieldpathupdate.cpp @@ -1,27 +1,24 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "addfieldpathupdate.h" -#include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/select/parser.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> +#include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> +#include <ostream> using vespalib::nbostream; -LOG_SETUP(".document.update.fieldpathupdate"); - namespace document { +using namespace fieldvalue; +using vespalib::make_string; + IMPLEMENT_IDENTIFIABLE(AddFieldPathUpdate, FieldPathUpdate); -AddFieldPathUpdate::AddFieldPathUpdate( - const DocumentTypeRepo& repo, - const DataType& type, - stringref fieldPath, - stringref whereClause, - const ArrayFieldValue& values) +AddFieldPathUpdate::AddFieldPathUpdate(const DocumentTypeRepo& repo, const DataType& type, + stringref fieldPath, stringref whereClause, const ArrayFieldValue& values) : FieldPathUpdate(repo, type, fieldPath, whereClause), _values(vespalib::CloneablePtr<ArrayFieldValue>(values.clone())) { @@ -39,22 +36,33 @@ AddFieldPathUpdate::clone() const { return new AddFieldPathUpdate(*this); } -FieldValue::IteratorHandler::ModificationStatus -AddFieldPathUpdate::AddIteratorHandler::doModify(FieldValue& fv) -{ - LOG(spam, "Adding values to %s", fv.toString().c_str()); +namespace { + +class AddIteratorHandler : public fieldvalue::IteratorHandler { +public: + AddIteratorHandler(const ArrayFieldValue &values) : _values(values) {} + fieldvalue::ModificationStatus doModify(FieldValue &fv) override; + bool createMissingPath() const override { return true; } + bool onComplex(const fieldvalue::IteratorHandler::Content &) override { return false; } +private: + const ArrayFieldValue &_values; +}; + + +ModificationStatus +AddIteratorHandler::doModify(FieldValue &fv) { if (fv.inherits(CollectionFieldValue::classId)) { - CollectionFieldValue& cf = static_cast<CollectionFieldValue&>(fv); + CollectionFieldValue &cf = static_cast<CollectionFieldValue &>(fv); for (std::size_t i = 0; i < _values.size(); ++i) { cf.add(_values[i]); } } else { - vespalib::string err = vespalib::make_string( - "Unable to add a value to a \"%s\" field value.", - fv.getClass().name()); - throw vespalib::IllegalArgumentException(err, VESPA_STRLOC); + vespalib::string err = make_string("Unable to add a value to a \"%s\" field value.", fv.getClass().name()); + throw vespalib::IllegalArgumentException(err, VESPA_STRLOC); } - return MODIFIED; + return ModificationStatus::MODIFIED; +} + } bool @@ -68,8 +76,7 @@ AddFieldPathUpdate::operator==(const FieldPathUpdate& other) const } void -AddFieldPathUpdate::print(std::ostream& out, bool verbose, - const std::string& indent) const +AddFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "AddFieldPathUpdate(\n"; FieldPathUpdate::print(out, verbose, indent + " "); @@ -79,8 +86,7 @@ AddFieldPathUpdate::print(std::ostream& out, bool verbose, } void -AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, - const DataType& type, +AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) { FieldPathUpdate::deserialize(repo, type, buffer, version); @@ -95,4 +101,11 @@ AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, buffer.incPos(buffer.getRemaining() - stream.size()); } +std::unique_ptr<IteratorHandler> +AddFieldPathUpdate::getIteratorHandler(Document&) const { + return std::make_unique<AddIteratorHandler>(*_values); +} + + + } // ns document diff --git a/document/src/vespa/document/update/addfieldpathupdate.h b/document/src/vespa/document/update/addfieldpathupdate.h index 6d75ebc4209..94156156257 100644 --- a/document/src/vespa/document/update/addfieldpathupdate.h +++ b/document/src/vespa/document/update/addfieldpathupdate.h @@ -10,7 +10,6 @@ class AddFieldPathUpdate : public FieldPathUpdate public: /** For deserialization */ AddFieldPathUpdate(); - AddFieldPathUpdate(const DocumentTypeRepo& repo, const DataType& type, stringref fieldPath, stringref whereClause, const ArrayFieldValue& values); ~AddFieldPathUpdate(); @@ -18,7 +17,6 @@ public: FieldPathUpdate* clone() const override; bool operator==(const FieldPathUpdate& other) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - const ArrayFieldValue & getValues() const { return *_values; } DECLARE_IDENTIFIABLE(AddFieldPathUpdate); @@ -29,22 +27,7 @@ private: void deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) override; - class AddIteratorHandler : public FieldValue::IteratorHandler - { - public: - AddIteratorHandler(const ArrayFieldValue& values) : _values(values) { } - - ModificationStatus doModify(FieldValue& fv) override; - bool createMissingPath() const override { return true; } - bool onComplex(const Content&) override { return false; } - private: - const ArrayFieldValue& _values; - }; - - std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override { - return std::unique_ptr<FieldValue::IteratorHandler>( - new AddIteratorHandler(*_values)); - } + std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document&) const override; vespalib::CloneablePtr<ArrayFieldValue> _values; }; diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp index 910e3d7dd50..b1142836fc2 100644 --- a/document/src/vespa/document/update/addvalueupdate.cpp +++ b/document/src/vespa/document/update/addvalueupdate.cpp @@ -2,17 +2,17 @@ #include "addvalueupdate.h" #include <vespa/document/base/field.h> #include <vespa/document/datatype/arraydatatype.h> -#include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> -#include <vespa/document/util/serializable.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/xmlstream.h> + using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp index 29c24e7140f..a8fe541d0fe 100644 --- a/document/src/vespa/document/update/arithmeticvalueupdate.cpp +++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp @@ -3,9 +3,11 @@ #include <vespa/document/base/field.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp index 4b8ee510576..3d67a589ea5 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.cpp +++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp @@ -2,6 +2,7 @@ #include "assignfieldpathupdate.h" #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/select/parser.h> #include <vespa/document/select/variablemap.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> @@ -16,6 +17,8 @@ using vespalib::nbostream; namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE(AssignFieldPathUpdate, FieldPathUpdate); AssignFieldPathUpdate::AssignFieldPathUpdate() @@ -69,24 +72,56 @@ FieldPathUpdate* AssignFieldPathUpdate::clone() const { return new AssignFieldPathUpdate(*this); } +namespace { -std::unique_ptr<FieldValue::IteratorHandler> -AssignFieldPathUpdate::getIteratorHandler(Document& doc) const +class AssignValueIteratorHandler : public IteratorHandler { - if (!_expression.empty()) { - return std::unique_ptr<FieldValue::IteratorHandler>( - new AssignExpressionIteratorHandler( - *_repo, doc, _expression, _removeIfZero, _createMissingPath)); - } else { - return std::unique_ptr<FieldValue::IteratorHandler>( - new AssignValueIteratorHandler( - *_newValue, _removeIfZero, _createMissingPath)); - } -} - - -FieldValue::IteratorHandler::ModificationStatus -AssignFieldPathUpdate::AssignValueIteratorHandler::doModify(FieldValue& fv) { +public: + AssignValueIteratorHandler(const FieldValue& newValue, + bool removeIfZero, + bool createMissingPath_) + : _newValue(newValue), _removeIfZero(removeIfZero), + _createMissingPath(createMissingPath_) + {} + + ModificationStatus doModify(FieldValue& fv) override; + bool onComplex(const Content&) override { return false; } + bool createMissingPath() const override { return _createMissingPath; } + +private: + const FieldValue& _newValue; + bool _removeIfZero; + bool _createMissingPath; +}; + +class AssignExpressionIteratorHandler : public IteratorHandler +{ +public: + AssignExpressionIteratorHandler( + const DocumentTypeRepo& repo, + Document& doc, + const vespalib::string& expression, + bool removeIfZero, + bool createMissingPath_) + : _calc(repo, expression), + _doc(doc), + _removeIfZero(removeIfZero), + _createMissingPath(createMissingPath_) + {} + + ModificationStatus doModify(FieldValue& fv) override; + bool onComplex(const Content&) override { return false; } + bool createMissingPath() const override { return _createMissingPath; } + +private: + DocumentCalculator _calc; + Document& _doc; + bool _removeIfZero; + bool _createMissingPath; +}; + +ModificationStatus +AssignValueIteratorHandler::doModify(FieldValue& fv) { LOG(spam, "fv = %s", fv.toString().c_str()); if (!(*fv.getDataType() == *_newValue.getDataType())) { std::string err = vespalib::make_string( @@ -105,8 +140,8 @@ AssignFieldPathUpdate::AssignValueIteratorHandler::doModify(FieldValue& fv) { return MODIFIED; } -FieldValue::IteratorHandler::ModificationStatus -AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) { +ModificationStatus +AssignExpressionIteratorHandler::doModify(FieldValue& fv) { LOG(spam, "fv = %s", fv.toString().c_str()); if (fv.inherits(NumericFieldValueBase::classId)) { std::unique_ptr<select::VariableMap> varHolder = std::make_unique<select::VariableMap>(); @@ -146,6 +181,18 @@ AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) return MODIFIED; } +} + +std::unique_ptr<IteratorHandler> +AssignFieldPathUpdate::getIteratorHandler(Document& doc) const +{ + if (!_expression.empty()) { + return std::make_unique<AssignExpressionIteratorHandler>(*_repo, doc, _expression, _removeIfZero, _createMissingPath); + } else { + return std::make_unique<AssignValueIteratorHandler>(*_newValue, _removeIfZero, _createMissingPath); + } +} + bool AssignFieldPathUpdate::operator==(const FieldPathUpdate& other) const { diff --git a/document/src/vespa/document/update/assignfieldpathupdate.h b/document/src/vespa/document/update/assignfieldpathupdate.h index 81853246e54..3794cb331ce 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.h +++ b/document/src/vespa/document/update/assignfieldpathupdate.h @@ -58,53 +58,7 @@ private: void deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) override; - class AssignValueIteratorHandler : public FieldValue::IteratorHandler - { - public: - AssignValueIteratorHandler(const FieldValue& newValue, - bool removeIfZero, - bool createMissingPath_) - : _newValue(newValue), _removeIfZero(removeIfZero), - _createMissingPath(createMissingPath_) - {} - - ModificationStatus doModify(FieldValue& fv) override; - bool onComplex(const Content&) override { return false; } - bool createMissingPath() const override { return _createMissingPath; } - - private: - const FieldValue& _newValue; - bool _removeIfZero; - bool _createMissingPath; - }; - - class AssignExpressionIteratorHandler : public FieldValue::IteratorHandler - { - public: - AssignExpressionIteratorHandler( - const DocumentTypeRepo& repo, - Document& doc, - const vespalib::string& expression, - bool removeIfZero, - bool createMissingPath_) - : _calc(repo, expression), - _doc(doc), - _removeIfZero(removeIfZero), - _createMissingPath(createMissingPath_) - {} - - ModificationStatus doModify(FieldValue& fv) override; - bool onComplex(const Content&) override { return false; } - bool createMissingPath() const override { return _createMissingPath; } - - private: - DocumentCalculator _calc; - Document& _doc; - bool _removeIfZero; - bool _createMissingPath; - }; - - std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document& doc) const override; + std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc) const override; const DocumentTypeRepo *_repo; FieldValue::CP _newValue; diff --git a/document/src/vespa/document/update/assignvalueupdate.cpp b/document/src/vespa/document/update/assignvalueupdate.cpp index 0538bec10e5..63609ed18c1 100644 --- a/document/src/vespa/document/update/assignvalueupdate.cpp +++ b/document/src/vespa/document/update/assignvalueupdate.cpp @@ -3,14 +3,16 @@ #include "assignvalueupdate.h" #include <vespa/document/base/field.h> #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> + using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/update/clearvalueupdate.cpp b/document/src/vespa/document/update/clearvalueupdate.cpp index cb526852299..f34987ccfeb 100644 --- a/document/src/vespa/document/update/clearvalueupdate.cpp +++ b/document/src/vespa/document/update/clearvalueupdate.cpp @@ -4,10 +4,12 @@ #include <vespa/document/base/field.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> #include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index d91b33650d4..43fe6c28dc8 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -9,12 +9,14 @@ #include <vespa/document/util/bufferexceptions.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; using vespalib::make_string; using vespalib::string; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index a4145a9b58a..11ed5f08c93 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -1,14 +1,21 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/fieldvalue/fieldvalues.h> +#include "fieldpathupdates.h" +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/select/parser.h> -#include <vespa/document/update/fieldpathupdates.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/log/log.h> +#include <ostream> +#include <vespa/log/log.h> LOG_SETUP(".document.update.fieldpathupdate"); +using vespalib::make_string; +using vespalib::IllegalArgumentException; + namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldPathUpdate, Identifiable); namespace { @@ -43,9 +50,9 @@ FieldPathUpdate::FieldPathUpdate(const DocumentTypeRepo& repo, : std::unique_ptr<select::Node>()) { if (!_fieldPath.get()) { - throw vespalib::IllegalArgumentException( - vespalib::make_string("Could not create field path update for: path='%s', where='%s'", - fieldPath.c_str(), whereClause.c_str()), VESPA_STRLOC); + throw IllegalArgumentException( + make_string("Could not create field path update for: path='%s', where='%s'", + fieldPath.c_str(), whereClause.c_str()), VESPA_STRLOC); } } @@ -61,7 +68,7 @@ FieldPathUpdate::operator==(const FieldPathUpdate& other) const void FieldPathUpdate::applyTo(Document& doc) const { - std::unique_ptr<FieldValue::IteratorHandler> handler(getIteratorHandler(doc)); + std::unique_ptr<IteratorHandler> handler(getIteratorHandler(doc)); if (!_whereClause) { doc.iterateNested(*_fieldPath, *handler); @@ -70,7 +77,7 @@ FieldPathUpdate::applyTo(Document& doc) const for (select::ResultList::const_iterator i = results.begin(); i != results.end(); ++i) { - LOG(spam, "vars = %s", FieldValue::IteratorHandler::toString(handler->getVariables()).c_str()); + LOG(spam, "vars = %s", handler->getVariables().toString().c_str()); if (*i->second == select::Result::True) { handler->setVariables(i->first); doc.iterateNested(*_fieldPath, *handler); @@ -88,8 +95,7 @@ FieldPathUpdate::affectsDocumentBody() const } void -FieldPathUpdate::print(std::ostream& out, bool, - const std::string& indent) const +FieldPathUpdate::print(std::ostream& out, bool, const std::string& indent) const { out << indent << "fieldPath='" << _originalFieldPath << "',\n" << indent << "whereClause='" << _originalWhereClause << "'"; @@ -99,10 +105,10 @@ void FieldPathUpdate::checkCompatibility(const FieldValue& fv) const { if ( !getResultingDataType().isValueType(fv)) { - throw vespalib::IllegalArgumentException( - vespalib::make_string("Cannot update a '%s' field with a '%s' value", - getResultingDataType().toString().c_str(), - fv.getDataType()->toString().c_str()), + throw IllegalArgumentException( + make_string("Cannot update a '%s' field with a '%s' value", + getResultingDataType().toString().c_str(), + fv.getDataType()->toString().c_str()), VESPA_STRLOC); } } @@ -111,8 +117,7 @@ const DataType& FieldPathUpdate::getResultingDataType() const { if (_fieldPath->empty()) { - throw vespalib::IllegalStateException("Cannot get resulting data " - "type from an empty field path", VESPA_STRLOC); + throw vespalib::IllegalStateException("Cannot get resulting data type from an empty field path", VESPA_STRLOC); } return _fieldPath->rbegin()->getDataType(); } @@ -138,9 +143,7 @@ FieldPathUpdate::deserialize(const DocumentTypeRepo& repo, try { _fieldPath = type.buildFieldPath(_originalFieldPath).release(); if (!_fieldPath.get()) { - throw DeserializeException( - vespalib::make_string("Invalid field path: '%s'", _originalFieldPath.c_str()), - VESPA_STRLOC); + throw DeserializeException(make_string("Invalid field path: '%s'", _originalFieldPath.c_str()), VESPA_STRLOC); } _whereClause = !_originalWhereClause.empty() ? parseDocumentSelection(_originalWhereClause, repo) @@ -159,8 +162,7 @@ FieldPathUpdate::createInstance(const DocumentTypeRepo& repo, buffer.getByte(updateType); std::unique_ptr<FieldPathUpdate> update; - switch (updateType) - { + switch (updateType) { case 0: update.reset(new AssignFieldPathUpdate()); break; @@ -171,9 +173,7 @@ FieldPathUpdate::createInstance(const DocumentTypeRepo& repo, update.reset(new AddFieldPathUpdate()); break; default: - throw DeserializeException( - vespalib::make_string("Unknown fieldpath update type: %d", updateType), - VESPA_STRLOC); + throw DeserializeException(make_string("Unknown fieldpath update type: %d", updateType), VESPA_STRLOC); } update->deserialize(repo, type, buffer, serializationVersion); return update; diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 80a63d62281..0eaff9485bf 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -104,8 +104,7 @@ protected: enum SerializedMagic {AssignMagic=0, RemoveMagic=1, AddMagic=2}; private: // TODO: rename to createIteratorHandler? - virtual std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler( - Document& doc) const = 0; + virtual std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc) const = 0; vespalib::string _originalFieldPath; vespalib::string _originalWhereClause; diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp index 0a92fa7fb15..37d775bb204 100644 --- a/document/src/vespa/document/update/mapvalueupdate.cpp +++ b/document/src/vespa/document/update/mapvalueupdate.cpp @@ -5,13 +5,14 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using namespace vespalib::xml; -namespace document -{ +namespace document { IMPLEMENT_IDENTIFIABLE(MapValueUpdate, ValueUpdate); diff --git a/document/src/vespa/document/update/removefieldpathupdate.cpp b/document/src/vespa/document/update/removefieldpathupdate.cpp index 6c9d60abd2f..4af867995d9 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.cpp +++ b/document/src/vespa/document/update/removefieldpathupdate.cpp @@ -1,11 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "removefieldpathupdate.h" -#include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/select/parser.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> +#include <ostream> namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE(RemoveFieldPathUpdate, FieldPathUpdate); RemoveFieldPathUpdate::RemoveFieldPathUpdate() @@ -30,8 +32,7 @@ RemoveFieldPathUpdate::operator==(const FieldPathUpdate& other) const } void -RemoveFieldPathUpdate::print(std::ostream& out, bool verbose, - const std::string& indent) const +RemoveFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "RemoveFieldPathUpdate(\n"; FieldPathUpdate::print(out, verbose, indent + " "); @@ -39,11 +40,28 @@ RemoveFieldPathUpdate::print(std::ostream& out, bool verbose, } void -RemoveFieldPathUpdate::deserialize( - const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) +RemoveFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, + ByteBuffer& buffer, uint16_t version) { FieldPathUpdate::deserialize(repo, type, buffer, version); } +namespace { + +class RemoveIteratorHandler : public IteratorHandler { +public: + RemoveIteratorHandler() {} + + ModificationStatus doModify(FieldValue &) override { + return ModificationStatus::REMOVED; + } +}; + +} + +std::unique_ptr<IteratorHandler> +RemoveFieldPathUpdate::getIteratorHandler(Document&) const { + return std::make_unique<RemoveIteratorHandler>(); +} + } // ns document diff --git a/document/src/vespa/document/update/removefieldpathupdate.h b/document/src/vespa/document/update/removefieldpathupdate.h index 6de2a105cbf..1a3865fe761 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.h +++ b/document/src/vespa/document/update/removefieldpathupdate.h @@ -29,19 +29,7 @@ private: void deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) override; - class RemoveIteratorHandler : public FieldValue::IteratorHandler - { - public: - RemoveIteratorHandler() {} - - ModificationStatus doModify(FieldValue&) override { - return REMOVED; - } - }; - - std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override { - return std::make_unique<RemoveIteratorHandler>(); - } + std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document&) const override; }; diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp index e0575f600da..34af57edbe8 100644 --- a/document/src/vespa/document/update/removevalueupdate.cpp +++ b/document/src/vespa/document/update/removevalueupdate.cpp @@ -1,20 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "removevalueupdate.h" #include <vespa/document/base/field.h> #include <vespa/document/datatype/arraydatatype.h> #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> -#include <vespa/document/update/removevalueupdate.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using namespace vespalib::xml; -namespace document -{ +namespace document { IMPLEMENT_IDENTIFIABLE(RemoveValueUpdate, ValueUpdate); diff --git a/document/src/vespa/document/util/xmlserializable.h b/document/src/vespa/document/util/xmlserializable.h index 21e0b858dec..f74761a04f6 100644 --- a/document/src/vespa/document/util/xmlserializable.h +++ b/document/src/vespa/document/util/xmlserializable.h @@ -5,14 +5,6 @@ #include <vespa/vespalib/util/xmlserializable.h> namespace document { - typedef vespalib::xml::XmlOutputStream XmlOutputStream; - typedef vespalib::xml::XmlSerializable XmlSerializable; - typedef vespalib::xml::XmlTag XmlTag; - typedef vespalib::xml::XmlEndTag XmlEndTag; - typedef vespalib::xml::XmlAttribute XmlAttribute; - typedef vespalib::xml::XmlContent XmlContent; - typedef vespalib::xml::XmlEscapedContent XmlEscapedContent; - typedef vespalib::xml::XmlBase64Content XmlBase64Content; - typedef vespalib::xml::XmlContentWrapper XmlContentWrapper; + using XmlSerializable = vespalib::xml::XmlSerializable; + using XmlOutputStream = vespalib::xml::XmlOutputStream; } - diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index 3c9f068468f..834bea0e757 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "messages50test.h" -#include <vespa/document/datatype/datatype.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/fieldpathupdates.h> -#include <vespa/document/datatype/documenttype.h> #include <vespa/documentapi/documentapi.h> #include <vespa/vdslib/container/writabledocumentlist.h> +#include <vespa/document/update/fieldpathupdates.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/select/parser.h> using document::DataType; using document::DocumentTypeRepo; diff --git a/documentapi/src/tests/systemstate/systemstate.cpp b/documentapi/src/tests/systemstate/systemstate.cpp index e3163937a3f..d1ca470f670 100644 --- a/documentapi/src/tests/systemstate/systemstate.cpp +++ b/documentapi/src/tests/systemstate/systemstate.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/documentapi/messagebus/systemstate/systemstate.h> +#include <vespa/documentapi/messagebus/systemstate/nodestate.h> #include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index 0849a93acbc..6c63e33c745 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -4,16 +4,15 @@ #include "routablefactories51.h" #include "routablefactories52.h" #include "routingpolicyfactories.h" -#include <vespa/document/repo/documenttyperepo.h> +#include "routablerepository.h" +#include "routingpolicyrepository.h" +#include "replymerger.h" #include <vespa/document/util/stringutil.h> #include <vespa/documentapi/documentapi.h> -#include <vespa/documentapi/messagebus/replymerger.h> #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/routing/routingcontext.h> -#include <vespa/vespalib/component/versionspecification.h> #include <vespa/vespalib/util/exceptions.h> -#include "routablerepository.h" -#include "routingpolicyrepository.h" +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".documentprotocol"); diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp index 01e95e9f035..aa95cba1785 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp @@ -3,8 +3,7 @@ #include "batchdocumentupdatemessage.h" #include "batchdocumentupdatereply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/document/select/parser.h> -#include <vespa/document/bucket/bucketselector.h> +#include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vespalib/util/exceptions.h> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp index e2903097934..af1203926b5 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp @@ -2,6 +2,7 @@ #include "documentmessage.h" #include <vespa/documentapi/messagebus/documentprotocol.h> +#include <cassert> namespace documentapi { @@ -16,7 +17,7 @@ mbus::Reply::UP DocumentMessage::createReply() const { mbus::Reply::UP ret(doCreateReply().release()); - assert(ret.get() != NULL); + assert(ret.get() != nullptr); return ret; } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp index 641feb85f9d..24ac666eae9 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "emptybucketsmessage.h" -#include <vespa/documentapi/messagebus/documentprotocol.h> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp index e07566cf22f..f23ecdf3556 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "removelocationmessage.h" #include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/document/select/parser.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h index c56ef8b0ede..799fbe5bde8 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h @@ -3,8 +3,9 @@ #include "documentmessage.h" #include <vespa/document/bucket/bucketid.h> -#include <vespa/document/bucket/bucketselector.h> -#include <vespa/document/select/parser.h> + +namespace document::select { class Parser; } +namespace document { class BucketIdFactory; } namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp b/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp index 50cf7129016..ef791cc55f1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp @@ -6,6 +6,7 @@ #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> #include <vespa/documentapi/messagebus/messages/getdocumentreply.h> #include <vespa/messagebus/emptyreply.h> +#include <cassert> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp index b620611695e..d9c8cd765a1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -4,6 +4,8 @@ #include <vespa/documentapi/documentapi.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/select/parser.h> using vespalib::nbostream; using std::make_unique; diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp index 4572b2bbcc6..32956ff23bd 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp @@ -4,6 +4,8 @@ #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/document/util/stringutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".routablerepository"); @@ -11,9 +13,7 @@ namespace documentapi { RoutableRepository::VersionMap::VersionMap() : _factoryVersions() -{ - // empty -} +{ } bool RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version, diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp index 627a760b324..64e05fec793 100644 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "systemstate.h" +#include "nodestate.h" +#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/stringfmt.h> #include <boost/spirit/include/classic_core.hpp> #include <boost/spirit/include/classic_parse_tree.hpp> @@ -256,7 +258,9 @@ parseSystemState(SystemStateGrammar &grammar, boost::spirit::classic::tree_node< return ret; } -vespalib::Lock SystemState::_parseLock; +namespace { + vespalib::Lock _G_parseLock; +} SystemState::UP SystemState::newInstance(const string &state) @@ -265,7 +269,7 @@ SystemState::newInstance(const string &state) return SystemState::UP(new SystemState(NodeState::UP(new NodeState()))); } try { - vespalib::LockGuard guard(_parseLock); + vespalib::LockGuard guard(_G_parseLock); SystemStateGrammar grammar; boost::spirit::classic::tree_parse_info<> info = boost::spirit::classic::pt_parse(static_cast<const char *>(&*state.begin()), @@ -289,15 +293,14 @@ SystemState::newInstance(const string &state) } } catch(std::exception& e) { - std::cerr << "SystemState::parse() internal error: " - << e.what() << std::endl; + LOG(fatal, "SystemState::parse() internal error: %s", e.what()); } return SystemState::UP(); } SystemState::SystemState(NodeState::UP root) : _root(std::move(root)), - _lock() { - // empty -} + _lock(std::make_unique<vespalib::Lock>()) +{} +SystemState::~SystemState() {}
\ No newline at end of file diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h index 1d958c42b2d..169e1fbb9b6 100644 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h @@ -1,11 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "nodestate.h" -#include <vespa/vespalib/util/sync.h> +#include <vespa/documentapi/common.h> +namespace vespalib { class Lock; } namespace documentapi { +class NodeState; + /** * This class is a factory to create a tree of {@link NodeState} objects from a parseable node state * string. The naming of this class is intended to capture the fact that this annotated service tree actually @@ -13,10 +15,8 @@ namespace documentapi { */ class SystemState { private: - static vespalib::Lock _parseLock; - - NodeState::UP _root; - vespalib::Lock _lock; + std::unique_ptr<NodeState> _root; + std::unique_ptr<vespalib::Lock> _lock; friend class SystemStateHandle; @@ -26,9 +26,10 @@ private: * * @param root The root node state. */ - SystemState(NodeState::UP root); + SystemState(std::unique_ptr<NodeState> root); public: + ~SystemState(); SystemState(const SystemState &) = delete; SystemState & operator = (const SystemState &) = delete; /** @@ -47,4 +48,3 @@ public: }; } - diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp index 6d3d9a19568..ea43665facb 100644 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp @@ -11,14 +11,14 @@ SystemStateHandover::SystemStateHandover(SystemState *state, vespalib::LockGuard SystemStateHandle::SystemStateHandle(SystemState &state) : _state(&state), - _guard(state._lock) + _guard(*state._lock) {} SystemStateHandle::SystemStateHandle(SystemStateHandle &rhs) : _state(rhs._state), _guard(rhs._guard) { - rhs._state = NULL; + rhs._state = nullptr; } SystemStateHandle::SystemStateHandle(const SystemStateHandover &rhs) : @@ -31,7 +31,7 @@ SystemStateHandle::~SystemStateHandle() {} SystemStateHandle::operator SystemStateHandover() { SystemStateHandover ret(_state, _guard); - _state = NULL; + _state = nullptr; return ret; } diff --git a/fastos/src/vespa/fastos/time.h b/fastos/src/vespa/fastos/time.h index 97ebe21a3b5..1e37813a072 100644 --- a/fastos/src/vespa/fastos/time.h +++ b/fastos/src/vespa/fastos/time.h @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> - /** * Interface to OS time functions. */ @@ -12,7 +10,7 @@ protected: /** * Destructor. No cleanup needed for base class. */ - virtual ~FastOS_TimeInterface(void) { } + virtual ~FastOS_TimeInterface() { } public: /** @@ -49,30 +47,26 @@ public: * Note: Only millisecond accuracy is guaranteed. * @param microsecs Number of microseconds to add. */ - void AddMicroSecs(double microsecs) - { SetMicroSecs(MicroSecs() + microsecs); } + void AddMicroSecs(double microsecs) { SetMicroSecs(MicroSecs() + microsecs); } /** * Add a specified number of milliseconds to the time. * @param millisecs Number of milliseconds to add. */ - void AddMilliSecs(double millisecs) - { SetMilliSecs(MilliSecs() + millisecs); } + void AddMilliSecs(double millisecs) { SetMilliSecs(MilliSecs() + millisecs); } /** * Subtract a specified number of microseconds from the time. * Note: Only millisecond accuracy is guaranteed. * @param microsecs Number of microseconds to subtract. */ - void SubtractMicroSecs(double microsecs) - { SetMicroSecs(MicroSecs() - microsecs); } + void SubtractMicroSecs(double microsecs) { SetMicroSecs(MicroSecs() - microsecs); } /** * Subtract a specified number of milliseconds from the time. * @param millisecs Number of milliseconds to subtract. */ - void SubtractMilliSecs(double millisecs) - { SetMilliSecs(MilliSecs() - millisecs); } + void SubtractMilliSecs(double millisecs) { SetMilliSecs(MilliSecs() - millisecs); } /** * Return the time in microseconds. @@ -133,7 +127,6 @@ public: virtual long int GetMicroSeconds() const = 0; }; - #include <vespa/fastos/unix_time.h> -typedef FastOS_UNIX_Time FASTOS_PREFIX(Time); +using FastOS_Time = FastOS_UNIX_Time; diff --git a/fastos/src/vespa/fastos/unix_time.cpp b/fastos/src/vespa/fastos/unix_time.cpp index 202ffe6d82a..6b6a8e58ffb 100644 --- a/fastos/src/vespa/fastos/unix_time.cpp +++ b/fastos/src/vespa/fastos/unix_time.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/time.h> +#include "time.h" +#include "types.h" double FastOS_UNIX_Time::MicroSecs() const @@ -96,3 +97,8 @@ FastOS_UNIX_Time::SetSecs(double secs) _time.tv_usec = - static_cast<int>((- secs - (-_time.tv_sec)) * 1000000); } } + +void FastOS_UNIX_Time::SetNow() { + gettimeofday(&_time, NULL); +} + diff --git a/fastos/src/vespa/fastos/unix_time.h b/fastos/src/vespa/fastos/unix_time.h index dd6122bc463..5ae4916418a 100644 --- a/fastos/src/vespa/fastos/unix_time.h +++ b/fastos/src/vespa/fastos/unix_time.h @@ -89,9 +89,8 @@ public: void SetMilliSecs(double millisecs) override; void SetSecs(double secs) override; - void SetNow() override { gettimeofday(&_time, NULL); } + void SetNow() override; long int GetSeconds() const override { return _time.tv_sec; } long int GetMicroSeconds() const override { return _time.tv_usec; } }; - diff --git a/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp b/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp index 3cb5f4d008f..ddef7a2d175 100644 --- a/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp +++ b/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "componentsdeleter.h" +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".componentsdeleter"); diff --git a/fnet/src/vespa/fnet/scheduler.cpp b/fnet/src/vespa/fnet/scheduler.cpp index 71f02754a92..dd7a3246090 100644 --- a/fnet/src/vespa/fnet/scheduler.cpp +++ b/fnet/src/vespa/fnet/scheduler.cpp @@ -2,6 +2,7 @@ #include "scheduler.h" #include "task.h" +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".fnet.scheduler"); diff --git a/juniper/src/vespa/juniper/Matcher.cpp b/juniper/src/vespa/juniper/Matcher.cpp index 2eeac3417af..76460323f9b 100644 --- a/juniper/src/vespa/juniper/Matcher.cpp +++ b/juniper/src/vespa/juniper/Matcher.cpp @@ -6,14 +6,10 @@ #include "juniperdebug.h" #include "sumdesc.h" #include "Matcher.h" -#include "foreach_utils.h" -#include "SummaryConfig.h" -#include "querynode.h" -#include "mcand.h" -#include "matchobject.h" #include "result.h" #include "juniperparams.h" #include "config.h" +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".juniper.matcher"); diff --git a/juniper/src/vespa/juniper/querynode.cpp b/juniper/src/vespa/juniper/querynode.cpp index 8718a76bf61..47bd5146432 100644 --- a/juniper/src/vespa/juniper/querynode.cpp +++ b/juniper/src/vespa/juniper/querynode.cpp @@ -4,6 +4,7 @@ #include "queryvisitor.h" #include "juniperdebug.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".juniper.querynode"); @@ -145,12 +146,10 @@ void QueryNode::ComputeThreshold() else if (!(_options & X_AND)) no_threshold = true; - for (int i = 0; i < _nchild; i++) - { + for (int i = 0; i < _nchild; i++) { QueryExpr* qe = _children[i]; qe->ComputeThreshold(); - if (!no_threshold) - { + if (!no_threshold) { int w = qe->_weight; if (_options | X_AND) th += w; else @@ -232,8 +231,7 @@ bool QueryTerm::Complex() bool QueryNode::Complex() { - for (int i = 0; i < _nchild; i++) - { + for (int i = 0; i < _nchild; i++) { if (_children[i]->_arity > 1) return true; } return false; @@ -243,8 +241,7 @@ bool QueryNode::Complex() int QueryNode::MaxArity() { int max_arity = _arity; - for (int i = 0; i < _nchild; i++) - { + for (int i = 0; i < _nchild; i++) { int ma = _children[i]->MaxArity(); if (ma > max_arity) max_arity = ma; } @@ -273,8 +270,7 @@ void SimplifyStack(QueryExpr*& orig_stack) int compact = 0; int i; - if (!node->Complete()) - { + if (!node->Complete()) { LOG(warning, "juniper: query stack incomplete, got arity %d, expected %d", node->_nchild, node->_arity); delete node; @@ -282,10 +278,8 @@ void SimplifyStack(QueryExpr*& orig_stack) return; } - for (i = 0; i < node->_arity; i++) - { - if (i > 0 && (node->_options & X_ONLY_1)) - { + for (i = 0; i < node->_arity; i++) { + if (i > 0 && (node->_options & X_ONLY_1)) { // Get rid of children # >2 for RANK/ANDNOT delete node->_children[i]; node->_children[i] = NULL; @@ -296,15 +290,11 @@ void SimplifyStack(QueryExpr*& orig_stack) if (node->_children[i] == NULL) compact++; } - if (compact > 0) - { + if (compact > 0) { node->_nchild = 0; - for (i = 0; i < node->_arity; i++) - { - if (node->_children[i]) - { - if (i > node->_nchild) - { + for (i = 0; i < node->_arity; i++) { + if (node->_children[i]) { + if (i > node->_nchild) { // shift remaining nodes down - remember to update _childno for each node.. node->_children[node->_nchild] = node->_children[i]; node->_children[i]->_childno = node->_nchild; @@ -316,11 +306,9 @@ void SimplifyStack(QueryExpr*& orig_stack) node->_arity = node->_nchild; } - if (node->_arity <= 1) - { + if (node->_arity <= 1) { QueryExpr* ret = NULL; - if (node->_arity == 1) - { + if (node->_arity == 1) { ret = node->_children[0]; node->_children[0] = NULL; ret->_parent = node->_parent; diff --git a/juniper/src/vespa/juniper/queryparser.cpp b/juniper/src/vespa/juniper/queryparser.cpp index 2355e41d005..4acc70dc8ab 100644 --- a/juniper/src/vespa/juniper/queryparser.cpp +++ b/juniper/src/vespa/juniper/queryparser.cpp @@ -5,6 +5,7 @@ #include "queryparser.h" #include "juniperdebug.h" #include <vector> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".juniper.queryparser"); diff --git a/juniper/src/vespa/juniper/rpinterface.cpp b/juniper/src/vespa/juniper/rpinterface.cpp index 9f10972a448..890481b4b94 100644 --- a/juniper/src/vespa/juniper/rpinterface.cpp +++ b/juniper/src/vespa/juniper/rpinterface.cpp @@ -1,20 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpinterface.h" -#include <string> -#include <vector> #include "juniperparams.h" #include "foreach_utils.h" -#include "juniperdebug.h" -#include "SummaryConfig.h" #include "queryvisitor.h" -#include "querynode.h" #include "queryhandle.h" #include "propreader.h" #include "result.h" #include "config.h" -#include "querymodifier.h" -#include <vespa/fastlib/text/normwordfolder.h> +#include <vector> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".juniper.rpinterface"); diff --git a/logd/src/logd/forward.cpp b/logd/src/logd/forward.cpp index 6e980034c9b..560ccaf1063 100644 --- a/logd/src/logd/forward.cpp +++ b/logd/src/logd/forward.cpp @@ -3,10 +3,10 @@ #include "forward.h" #include "errhandle.h" #include <vespa/vespalib/component/vtag.h> +#include <unistd.h> #include <vespa/log/log.h> LOG_SETUP(""); -LOG_RCSID("$Id$"); namespace logdemon { diff --git a/logd/src/logd/service.h b/logd/src/logd/service.h index 700ab85fe27..3ff2e7fab92 100644 --- a/logd/src/logd/service.h +++ b/logd/src/logd/service.h @@ -1,9 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <assert.h> + #include <logd/config-logd.h> #include <vespa/vespalib/util/hashmap.h> #include <vespa/log/log.h> +#include <cassert> namespace logdemon { diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp index 651543219ec..63e0f35c95c 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "options.h" -#include <vespa/config-stor-memfilepersistence.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.slotfile.options"); -namespace storage { - -namespace memfile { +namespace storage::memfile { Options::Options(const vespa::config::storage::StorMemfilepersistenceConfig& newConfig, const vespa::config::content::PersistenceConfig& newPersistenceConfig) @@ -179,5 +177,3 @@ void Options::print(std::ostream& out, bool verbose, } } - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp index 2ae27f06671..ba249df2b73 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp @@ -2,6 +2,7 @@ #include "devicemanager.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> namespace storage::memfile { diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp index 8ddf215d616..fdecb38ccf0 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp @@ -5,13 +5,12 @@ #include <vespa/vespalib/util/exceptions.h> #include <fstream> #include <sstream> +#include <sys/stat.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.devicemapper"); -namespace storage { - -namespace memfile { +namespace storage::memfile { namespace { uint64_t getDevice(const std::string& path) { @@ -95,5 +94,3 @@ AdvancedDeviceMapper::getPartitionId(const std::string& fileOnFS) const } } - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp index 3ff816c55eb..614d399ca22 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/util/guard.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <fstream> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.mountpointlist"); diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp index 3bd855553c7..9ff402a89b5 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp @@ -3,13 +3,12 @@ #include "partition.h" #include "devicemanager.h" #include <vespa/vespalib/util/exceptions.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.device.partition"); -namespace storage { - -namespace memfile { +namespace storage::memfile { Partition::Partition(DeviceManager& manager, uint64_t id, @@ -59,6 +58,4 @@ Partition::print(std::ostream& out, bool verbose, Device::print(out, verbose, indent); } -} // memfile - -} // storage +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp index b5b94d29336..ebc33d40f42 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp @@ -3,13 +3,13 @@ #include "partitionmonitor.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/xmlstream.h> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".persistence.device.partition.monitor"); -namespace storage { - -namespace memfile { +namespace storage::memfile { namespace { @@ -387,5 +387,3 @@ PartitionMonitor::overrideRealStat(uint32_t blockSize, uint32_t totalBlocks, } } - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp b/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp index 3a83cee392a..e5466634283 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp @@ -5,11 +5,10 @@ #include <vespa/vespalib/util/exceptions.h> #include <iomanip> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.memfile.filescanner"); -namespace storage { -namespace memfile { +namespace storage::memfile { FileScanner::Metrics::Metrics(framework::Clock& clock) : metrics::MetricSet("dbinit.filescan", "", @@ -238,5 +237,4 @@ FileScanner::handleBadLocation(const document::BucketId& bucket, return true; } -} // memfile -} // storage +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp index 65b92e212f5..175c156dc9d 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp @@ -4,8 +4,9 @@ #include <vespa/vespalib/util/guard.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.memfile.bufferedfilewriter"); namespace storage { diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp index 2de4e062b5e..950acf09b28 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp @@ -9,6 +9,7 @@ #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/storageframework/generic/clock/timer.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfilev1"); diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp index 236afdf9a77..66f6cdff083 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp @@ -6,8 +6,9 @@ #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_set.hpp> -#include <vespa/log/log.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".persistence.memfilev1.verifier"); namespace storage::memfile { diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp index eaa20c66ca9..2d91e46c7b2 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp @@ -7,8 +7,9 @@ #include <vespa/memfilepersistence/common/exceptions.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/storageframework/generic/clock/timer.h> +#include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.memfile.mapper"); namespace storage::memfile { diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp index 6bda3d69113..ef5c865eb5a 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp @@ -6,6 +6,8 @@ #include <vespa/memfilepersistence/common/environment.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/util/bytebuffer.h> +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".memfile.simpleiobuffer"); diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp index 6b3f72d9187..73defc043af 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp @@ -12,7 +12,7 @@ #include <ext/algorithm> #include <iomanip> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.memfile.memfile"); namespace { diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h index e0b1a7b9a2a..42d3fb0b238 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/memfilepersistence/common/types.h> +#include <map> namespace storage { namespace memfile { diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp index a6b5b5bd782..6cb1c523325 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp @@ -4,12 +4,12 @@ #include "visitorslotmatcher.h" #include "cacheevictionguard.h" #include <vespa/document/select/bodyfielddetector.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfile.handler.iterator"); -namespace storage { -namespace memfile { +namespace storage::memfile { CachePrefetchRequirements CachePrefetchRequirements::createFromSelection(const document::DocumentTypeRepo& repo, @@ -435,4 +435,3 @@ IteratorHandler::iterate(spi::IteratorId id, uint64_t maxByteSize) } } -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp index d17dfdc77a4..9e580d4e234 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp @@ -6,6 +6,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".memfilepersistenceprovider"); diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp index da085ca9137..c9972dbd02b 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp @@ -3,12 +3,12 @@ #include "operationhandler.h" #include <vespa/memfilepersistence/common/exceptions.h> #include <vespa/document/select/parser.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfile.handler.operation"); -namespace storage { -namespace memfile { +namespace storage::memfile { OperationHandler::OperationHandler(Environment& env) : _env(env) @@ -283,5 +283,4 @@ OperationHandler::parseDocumentSelection( return ret; } -} // memfile -} // storage +} diff --git a/messagebus/src/vespa/messagebus/callstack.cpp b/messagebus/src/vespa/messagebus/callstack.cpp index 5311a75c3c5..44ae4190698 100644 --- a/messagebus/src/vespa/messagebus/callstack.cpp +++ b/messagebus/src/vespa/messagebus/callstack.cpp @@ -4,6 +4,7 @@ #include "message.h" #include "reply.h" #include "idiscardhandler.h" +#include <cassert> namespace mbus { diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 0ba6bdfbd2f..e4a2761cd73 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -9,6 +9,7 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> LOG_SETUP(".test.metricmanager"); diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index db05ea16488..c352f0ee5ab 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -13,8 +13,9 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/log/log.h> +#include <sstream> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".metrics.manager"); namespace metrics { diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp index 66f83cdd4bf..4a998b79294 100644 --- a/metrics/src/vespa/metrics/xmlwriter.cpp +++ b/metrics/src/vespa/metrics/xmlwriter.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/metrics/xmlwriter.h> - -#include <vespa/metrics/countmetric.h> -#include <vespa/metrics/metricset.h> -#include <vespa/metrics/metricsnapshot.h> -#include <vespa/metrics/valuemetric.h> +#include "xmlwriter.h" +#include "countmetric.h" +#include "metricset.h" +#include "metricsnapshot.h" +#include "valuemetric.h" +#include <vespa/vespalib/util/xmlstream.h> #include <sstream> namespace metrics { diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp index 70bc8724c96..841fef4620d 100644 --- a/persistence/src/vespa/persistence/spi/clusterstate.cpp +++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp @@ -5,9 +5,9 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> -namespace storage { -namespace spi { +namespace storage::spi { ClusterState::ClusterState(const lib::ClusterState& state, uint16_t nodeIndex, @@ -99,5 +99,4 @@ void ClusterState::serialize(vespalib::nbostream& o) const { o << _distribution->serialize(); } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h index 359f6af5164..93db642611f 100644 --- a/persistence/src/vespa/persistence/spi/result.h +++ b/persistence/src/vespa/persistence/spi/result.h @@ -5,6 +5,7 @@ #include "bucketinfo.h" #include "bucket.h" #include "docentry.h" +#include <vespa/document/bucket/bucketidlist.h> namespace storage::spi { @@ -196,7 +197,7 @@ private: class BucketIdListResult : public Result { public: - typedef document::BucketId::List List; + using List = document::bucket::BucketIdList; /** * Constructor used when there was an error listing the buckets. diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index 835c06db3cf..7966a705802 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -8,7 +8,7 @@ #include <vespa/fastos/app.h> #include <iostream> #include <algorithm> -#include <string> +#include <sstream> #include <openssl/sha.h> #include <vespa/log/log.h> diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 95c6d39c51a..0fdfb09ecef 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/util/programoptions.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/document/config/config-documenttypes.h> #include <iostream> #include <vespa/config/helper/configgetter.hpp> diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp index 29ef6ef016c..0a181de39ec 100644 --- a/searchcore/src/tests/proton/common/selectpruner_test.cpp +++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/common/selectpruner.h> #include <vespa/document/select/parser.h> #include <vespa/document/select/cloningvisitor.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/test/mock_attribute_manager.h> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp index eb462ebebe2..9b2fc71702b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attribute_directory"); diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index d15eddb3ac4..10c2adfd1f2 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -3,6 +3,8 @@ #include "attributefieldvaluenode.h" #include "selectcontext.h" #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchlib/attribute/attributevector.h> + namespace proton { @@ -20,11 +22,10 @@ using search::attribute::BasicType; using search::attribute::CollectionType; using search::attribute::IAttributeVector; - AttributeFieldValueNode:: AttributeFieldValueNode(const vespalib::string& doctype, const vespalib::string& field, - const search::AttributeVector::SP &attribute) + const std::shared_ptr<search::AttributeVector> &attribute) : FieldValueNode(doctype, field), _attribute(attribute) { @@ -94,5 +95,11 @@ AttributeFieldValueNode::traceValue(const Context &context, } +document::select::ValueNode::UP +AttributeFieldValueNode::clone() const +{ + return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attribute)); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h index 38e532053d4..c0ffbaea987 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h @@ -1,38 +1,24 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/select/valuenode.h> -#include <vespa/searchlib/attribute/attributevector.h> - -namespace proton -{ +#include <vespa/document/select/valuenodes.h> +namespace search { class AttributeVector; } +namespace proton { class AttributeFieldValueNode : public document::select::FieldValueNode { - search::AttributeVector::SP _attribute; + using Context = document::select::Context; + std::shared_ptr<search::AttributeVector> _attribute; public: AttributeFieldValueNode(const vespalib::string& doctype, const vespalib::string& field, - const search::AttributeVector::SP &attribute); - - virtual std::unique_ptr<document::select::Value> - getValue(const document::select::Context &context) const override; + const std::shared_ptr<search::AttributeVector> &attribute); - virtual std::unique_ptr<document::select::Value> - traceValue(const document::select::Context &context, - std::ostream& out) const override; - - document::select::ValueNode::UP - clone() const override - { - return wrapParens(new AttributeFieldValueNode(getDocType(), - getFieldName(), - _attribute)); - } + std::unique_ptr<document::select::Value> getValue(const Context &context) const override; + std::unique_ptr<document::select::Value> traceValue(const Context &context, std::ostream& out) const override; + document::select::ValueNode::UP clone() const override; }; } // namespace proton - - diff --git a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp index d54603fd898..51cb9a17b5b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp @@ -20,6 +20,7 @@ #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/changevector.hpp> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".attrupdate"); diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp index 861131642e4..9a15c257dcc 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "cachedselect.h" -#include <vespa/document/select/valuenode.h> -#include <vespa/document/select/cloningvisitor.h> #include "attributefieldvaluenode.h" +#include "selectpruner.h" #include <vespa/searchlib/attribute/iattributemanager.h> +#include <vespa/searchlib/attribute/attributevector.h> #include <vespa/document/select/parser.h> -#include "selectpruner.h" #include <vespa/log/log.h> LOG_SETUP(".proton.common.cachedselect"); @@ -85,7 +84,7 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) _valueNode = expr.clone(); return; } - AttributeVector::SP av(ag->getSP()); + std::shared_ptr<search::AttributeVector> av(ag->getSP()); if (av->getCollectionType() == CollectionType::SINGLE) { ++_svAttrs; AttrMap::iterator it(_amap.find(name)); diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp index 001eb7d16f3..7e92be7507e 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp @@ -9,6 +9,7 @@ #include <vespa/document/select/branch.h> #include <vespa/document/select/doctype.h> #include <vespa/document/select/invalidconstant.h> +#include <vespa/document/select/valuenodes.h> #include <vespa/searchlib/attribute/iattributemanager.h> using document::select::And; diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h index 5d4bbc038b9..7b6c9954124 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h @@ -9,8 +9,9 @@ namespace search { class IAttributeManager; } -namespace proton -{ +namespace document { class DocumentTypeRepo; } + +namespace proton { class SelectPrunerBase { @@ -50,100 +51,36 @@ public: bool hasFields); SelectPruner(const SelectPruner *rhs); - - virtual - ~SelectPruner(); - - uint32_t - getFieldNodes() const - { - return _fieldNodes; - } - - uint32_t - getAttrFieldNodes() const - { - return _attrFieldNodes; - } - - const document::select::ResultSet & - getResultSet() const - { - return _resultSet; - } - - bool - isFalse() const; - - bool - isTrue() const; - - bool - isInvalid() const; - - bool - isConst() const; - - void - trace(std::ostream &t); - - void - process(const document::select::Node &node); + virtual ~SelectPruner(); + + uint32_t getFieldNodes() const { return _fieldNodes; } + uint32_t getAttrFieldNodes() const { return _attrFieldNodes; } + const document::select::ResultSet & getResultSet() const { return _resultSet; } + bool isFalse() const; + bool isTrue() const; + bool isInvalid() const; + bool isConst() const; + void trace(std::ostream &t); + void process(const document::select::Node &node); private: - virtual void - visitAndBranch(const document::select::And &expr) override; - - virtual void - visitComparison(const document::select::Compare &expr) override; - - virtual void - visitDocumentType(const document::select::DocType &expr) override; - - virtual void - visitNotBranch(const document::select::Not &expr) override; - - virtual void - visitOrBranch(const document::select::Or &expr) override; - - virtual void - visitArithmeticValueNode(const document::select::ArithmeticValueNode &expr) override; - - virtual void - visitFunctionValueNode(const document::select::FunctionValueNode &expr) override; - - virtual void - visitFieldValueNode(const document::select::FieldValueNode &expr) override; - - void - invertNode(); - - const document::select::Operator & - getOperator(const document::select::Operator &op); - - void - addNodeCount(const SelectPruner &rhs); - - void - setInvalidVal(); - - void - setInvalidConst(); - - void - setTernaryConst(bool val); - - void - resolveTernaryConst(bool wantInverted); - - bool - isInvalidVal() const; - - bool - isNullVal() const; - - void - swap(SelectPruner &rhs); + void visitAndBranch(const document::select::And &expr) override; + void visitComparison(const document::select::Compare &expr) override; + void visitDocumentType(const document::select::DocType &expr) override; + void visitNotBranch(const document::select::Not &expr) override; + void visitOrBranch(const document::select::Or &expr) override; + void visitArithmeticValueNode(const document::select::ArithmeticValueNode &expr) override; + void visitFunctionValueNode(const document::select::FunctionValueNode &expr) override; + void visitFieldValueNode(const document::select::FieldValueNode &expr) override; + void invertNode(); + const document::select::Operator &getOperator(const document::select::Operator &op); + void addNodeCount(const SelectPruner &rhs); + void setInvalidVal(); + void setInvalidConst(); + void setTernaryConst(bool val); + void resolveTernaryConst(bool wantInverted); + bool isInvalidVal() const; + bool isNullVal() const; + void swap(SelectPruner &rhs); }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 67a02c5d3ee..0253e943883 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -10,6 +10,7 @@ #include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/searchsummary/docsummary/docsumconfig.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".proton.docsummary.summarymanager"); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp index aa56daddeb2..1a18fee265f 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp @@ -2,6 +2,7 @@ #include "pruneremoveddocumentsoperation.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".proton.feedoperation.pruneremoveddocumentsoperation"); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp index ed029751271..2244759e866 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp @@ -3,6 +3,7 @@ #include "prepare_restart_flush_strategy.h" #include "flush_target_candidates.h" #include "tls_stats_map.h" +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".proton.flushengine.prepare_restart_flush_strategy"); diff --git a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp index 0cc6823f300..60f35934831 100644 --- a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp @@ -3,7 +3,7 @@ #include "clusterstatehandler.h" #include "iclusterstatechangedhandler.h" #include <vespa/vespalib/util/closuretask.h> -#include <algorithm> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".proton.server.clusterstatehandler"); diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 2b8209d9e4e..b444cd99952 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -10,9 +10,10 @@ #include <vespa/config-summarymap.h> #include <vespa/config-rank-profiles.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <fstream> #include <vespa/fastos/file.h> #include <vespa/config/helper/configgetter.hpp> +#include <fstream> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".proton.server.fileconfigmanager"); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index bb45103a13f..3d655f3decf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "lid_space_compaction_job.h" -#include "ifrozenbuckethandler.h" #include "imaintenancejobrunner.h" #include "i_disk_mem_usage_notifier.h" #include <vespa/searchcore/proton/common/eventlogger.h> - -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.lid_space_compaction_job"); +#include <cassert> using search::DocumentMetaData; using search::LidUsageStats; @@ -76,9 +73,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); _opStorer.storeOperation(op); _handler.handleCompactLidSpace(op); - if (LOG_WOULD_LOG(event)) { - EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); - } + EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); _shouldCompactLidSpace = false; } diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h index 281c814fc77..4947c7500ed 100644 --- a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h +++ b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h @@ -2,11 +2,12 @@ #pragma once #include <vespa/searchcore/proton/server/ibucketstatecalculator.h> -#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/bucket/bucketidlist.h> namespace proton::test { -typedef document::BucketId::List BucketIdVector; +using BucketIdVector = document::bucket::BucketIdList; + typedef std::set<document::BucketId> BucketIdSet; class BucketStateCalculator : public IBucketStateCalculator diff --git a/searchcore/src/vespa/searchcore/proton/test/userdocuments.h b/searchcore/src/vespa/searchcore/proton/test/userdocuments.h index d1ea6f6ab4c..6b5a911352b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/userdocuments.h +++ b/searchcore/src/vespa/searchcore/proton/test/userdocuments.h @@ -2,10 +2,9 @@ #pragma once #include "bucketdocuments.h" +#include <map> -namespace proton { - -namespace test { +namespace proton::test { /** * Collection of documents for a set of users, @@ -48,7 +47,4 @@ public: void clear() { _docs.clear(); } }; -} // namespace test - -} // namespace proton - +} diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 6f374f1bea4..512252fd732 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -5,14 +5,13 @@ #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/vespalib/objects/objectdumper.h> #include <vespa/vespalib/testkit/testapp.h> -#include <stdexcept> #include <vespa/document/base/testdocman.h> #include <vespa/vespalib/util/md5.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/expression/getdocidnamespacespecificfunctionnode.h> #include <vespa/searchlib/expression/documentfieldnode.h> #include <cmath> #include <iostream> +#include <list> #define MU std::make_unique diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp index a7e2d5738ce..b96b2b4ff05 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp @@ -12,6 +12,7 @@ #include <vespa/searchlib/diskindex/pagedict4randread.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/fastos/app.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP("pagedict4test"); diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index 3719c5da1c7..b4285da447b 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -9,6 +9,7 @@ #include <vespa/fastos/app.h> #include <iostream> #include <stdexcept> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP("translogstress"); diff --git a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp index 4344895ad63..f92087f7352 100644 --- a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp +++ b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp @@ -3,6 +3,7 @@ #include "indexmetainfo.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/guard.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".indexmetainfo"); diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 5ebe6bdeefe..ea66a3435d1 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -3,18 +3,14 @@ #include "fusion.h" #include "fieldreader.h" -#include "fieldwriter.h" #include "dictionarywordreader.h" #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/fslimits.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/searchlib/common/documentsummary.h> -#include <vespa/searchlib/common/tunefileinfo.h> -#include <vespa/searchlib/index/postinglistparams.h> -#include <vespa/searchlib/util/postingpriorityqueue.h> #include <vespa/vespalib/util/error.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".diskindex.fusion"); diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h index a6c027beb22..2a202fe562e 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h @@ -4,6 +4,7 @@ #include "documentaccessornode.h" #include "resultnode.h" #include "resultvector.h" +#include <vespa/document/fieldvalue/iteratorhandler.h> namespace search { namespace expression { @@ -38,11 +39,9 @@ public: DocumentFieldNode & operator = (const DocumentFieldNode & rhs); const vespalib::string & getFieldName() const override { return _fieldName; } private: - class Handler : public document::FieldValue::IteratorHandler { + class Handler : public document::fieldvalue::IteratorHandler { public: virtual void reset() = 0; - protected: - typedef document::FieldValue::IteratorHandler::Content Content; private: void onCollectionStart(const Content & c) override; void onStructStart(const Content & c) override; diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp index 92d00e9f907..924318a4a5a 100644 --- a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp @@ -3,8 +3,8 @@ #include "nativerankfeature.h" #include "valuefeature.h" #include "utils.h" -#include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/properties.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".features.nativerankfeature"); diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index 127dd0feecf..44bbe0db6e3 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -21,7 +21,7 @@ RandomNormalExecutor::RandomNormalExecutor(uint64_t seed, double mean, double st _spare(0.0) { - LOG(debug, "RandomNormalExecutor: seed=%" PRIu64 ", mean=%f, stddev=%f", seed, mean, stddev); + LOG(debug, "RandomNormalExecutor: seed=%zu, mean=%f, stddev=%f", seed, mean, stddev); _rnd.srand48(seed); } diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h index c6f7928f4d0..9b9c8eaaa3e 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h @@ -3,10 +3,9 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/exceptions.h> +#include <climits> -namespace search { -namespace predicate { +namespace search::predicate { /** * Helper class for expanding a point in a predicate range query to @@ -92,7 +91,4 @@ void PredicateRangeTermExpander::expand(const vespalib::string &key, int64_t sig } } - -} // namespace search::predicate -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp index 9bf87ecac1a..b3b5f80a4a2 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp @@ -2,14 +2,12 @@ #include "fakewordset.h" #include "fakeword.h" -#include <vespa/searchlib/index/schemautil.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".fakewordset"); -namespace search { - -namespace fakedata { +namespace search::fakedata { using index::PostingListParams; using index::SchemaUtil; @@ -146,7 +144,6 @@ FakeWordSet::getNumWords() return ret; } - void FakeWordSet::addDocIdBias(uint32_t docIdBias) { @@ -154,7 +151,4 @@ FakeWordSet::addDocIdBias(uint32_t docIdBias) applyDocIdBiasToVector(_words[i], docIdBias); } - -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp index 342ccb0104e..4be1eb668c9 100644 --- a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp +++ b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp @@ -31,22 +31,19 @@ FileSizeCalculator::extractFileSize(const vespalib::GenericHeader &header, uint64_t fileByteSize = fileBitSize / 8; if (!byteAligned(fileBitSize)) { LOG(error, - "Bad header file size tag for %s, fileBitSize=%" PRIu64 - " which is not a multiple of 8", + "Bad header file size tag for %s, fileBitSize=%zu which is not a multiple of 8", fileName.c_str(), fileBitSize); return false; } if (fileByteSize < headerLen) { LOG(error, - "Bad header file size tag for %s, fileBitSize=%" PRIu64 - " but header is %" PRIu64 "bits", + "Bad header file size tag for %s, fileBitSize=%zu but header is %zu bits", fileName.c_str(), fileBitSize, headerLen * 8); return false; } if (fileByteSize > fileSize) { LOG(error, - "Bad header file size tag for %s, fileBitSize=%" PRIu64 - " but whole file size is %" PRIu64 "bits", + "Bad header file size tag for %s, fileBitSize=%zu but whole file size is %zu bits", fileName.c_str(), fileBitSize, fileSize * 8); return false; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 8760a12fb82..4e11bbcd979 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -2,17 +2,14 @@ #include "juniperdfw.h" #include "docsumwriter.h" -#include "docsumfieldwriter.h" #include "docsumstate.h" -#include "keywordextractor.h" -#include "docsumformat.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/queryeval/split_float.h> - #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/juniper/config.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.dynamicteaserdfw"); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index e2dda337e73..697e2ed6722 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -6,12 +6,12 @@ #include <vespa/vespalib/util/jsonwriter.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <climits> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.geoposdfw"); -namespace search { -namespace docsummary { +namespace search::docsummary { using attribute::IAttributeVector; using attribute::IAttributeContext; @@ -110,5 +110,3 @@ GeoPositionDFW::create(const char *attribute_name, } -} - diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 841840d8349..be9517eba03 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -2,16 +2,15 @@ #include "positionsdfw.h" #include "docsumstate.h" -#include "idocsumenvironment.h" #include <vespa/searchlib/common/location.h> #include <vespa/vespalib/stllike/asciistream.h> #include <cmath> +#include <climits> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.positionsdfw"); -namespace search { -namespace docsummary { +namespace search::docsummary { using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -223,5 +222,4 @@ AbsDistanceDFW::UP createAbsDistanceDFW(const char *attribute_name, return ret; } -} // namespace docsummary -} // namespace search +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp index 0d3a212fc1f..d0beb2bd315 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp @@ -3,12 +3,12 @@ #include "urlresult.h" #include "resultconfig.h" #include <zlib.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.urlresult"); -namespace search { -namespace docsummary { +namespace search::docsummary { urlresult::urlresult(uint32_t partition, uint32_t docid, HitRank metric) : _partition(partition), @@ -811,4 +811,3 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } } -} diff --git a/slobrok/src/tests/startsome/tstdst.cpp b/slobrok/src/tests/startsome/tstdst.cpp index 577bf6f59a1..b645f79efa4 100644 --- a/slobrok/src/tests/startsome/tstdst.cpp +++ b/slobrok/src/tests/startsome/tstdst.cpp @@ -6,6 +6,7 @@ #include <vespa/fnet/frt/invoker.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/frt/target.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP("testrpcserver"); diff --git a/staging_vespalib/src/tests/benchmark/testbase.cpp b/staging_vespalib/src/tests/benchmark/testbase.cpp index 93f5c9b975a..6de6f66f76b 100644 --- a/staging_vespalib/src/tests/benchmark/testbase.cpp +++ b/staging_vespalib/src/tests/benchmark/testbase.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "testbase.h" #include <vespa/fastos/timestamp.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".testbase"); diff --git a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp b/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp index 90fcdb8f94f..4af70f7e6fe 100644 --- a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp +++ b/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/xmlserializable.h> +#include <vespa/vespalib/util/xmlstream.h> namespace vespalib { diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt index ca440428e0a..3b36b863707 100644 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -18,5 +18,6 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT shutdownguard.cpp timer.cpp xmlserializable.cpp + xmlstream.cpp DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.h b/staging_vespalib/src/vespa/vespalib/util/clock.h index 4a5611bbe5a..e17ec962fc9 100644 --- a/staging_vespalib/src/vespa/vespalib/util/clock.h +++ b/staging_vespalib/src/vespa/vespalib/util/clock.h @@ -33,7 +33,7 @@ public: Clock(double timePeriod=0.100); ~Clock(); - fastos::TimeStamp getTimeNS(void) const { + fastos::TimeStamp getTimeNS() const { if (!_running) { setTime(); } @@ -41,7 +41,7 @@ public: } fastos::TimeStamp getTimeNSAssumeRunning() const { return _timeNS; } - void stop(void); + void stop(); }; } diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index 537b44c49eb..354d2e013ed 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/growablebytebuffer.h> +#include "growablebytebuffer.h" +#include <arpa/inet.h> using namespace vespalib; diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h index fd65f0134bc..fdf090d22f9 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/stllike/string.h> diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h new file mode 100644 index 00000000000..6c4d8e37311 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// +#pragma once + +#include "polymorphicarraybase.h" + +namespace vespalib { + +/** + * Describes an interface an array of polymorphic types. + * The intention is to allow efficient implementations when that is possible + * while still enjoying the flexibility of the polymorph interface. + * It is not a full feldged Array implementation as std::vector. It contains just + * the minimum required to allow for efficient implementations for document::ArrayFieldValue. + * + * You specify the base type the interface shall provide. This base type must define + * virtual void assign(const B & rhs); + * For use with ComplexArrayT your type also need + * virtual T * clone() const; + */ +template<typename B> +class IArrayT : public IArrayBase { +public: + class iterator { + public: + iterator(IArrayT &a, size_t i) : _a(&a), _i(i) {} + iterator operator+(size_t diff) const { return iterator(*_a, _i + diff); } + iterator &operator++() { + ++_i; + return *this; + } + iterator operator++(int) { + iterator other(*this); + ++_i; + return other; + } + bool operator==(const iterator &other) const { return (_a == other._a) && (_i == other._i); } + bool operator!=(const iterator &other) const { return (_i != other._i) || (_a != other._a); } + B &operator*() { return (*_a)[_i]; } + B *operator->() { return &(*_a)[_i]; } + friend ssize_t operator-(const iterator &a, const iterator &b) { return a._i - b._i; } + private: + IArrayT *_a; + size_t _i; + }; + + class const_iterator { + public: + const_iterator(const IArrayT &a, size_t i) : _a(&a), _i(i) {} + const_iterator operator+(size_t diff) const { return const_iterator(*_a, _i + diff); } + const_iterator &operator++() { + ++_i; + return *this; + } + const_iterator operator++(int) { + const_iterator other(*this); + ++_i; + return other; + } + bool operator==(const const_iterator &other) const { return (_a == other._a) && (_i == other._i); } + bool operator!=(const const_iterator &other) const { return (_i != other._i) || (_a != other._a); } + const B &operator*() const { return (*_a)[_i]; } + const B *operator->() const { return &(*_a)[_i]; } + size_t operator-(const const_iterator &b) const { return _i - b._i; } + private: + const IArrayT *_a; + size_t _i; + }; + + typedef std::unique_ptr<IArrayT> UP; + virtual const B &operator[](size_t i) const = 0; + virtual B &operator[](size_t i) = 0; + virtual IArrayT *clone() const override = 0; + virtual iterator erase(iterator it) = 0; + virtual const_iterator begin() const { return const_iterator(*this, 0); } + virtual const_iterator end() const { return const_iterator(*this, size()); } + virtual iterator begin() { return iterator(*this, 0); } + virtual iterator end() { return iterator(*this, size()); } + virtual void push_back(const B &v) = 0; +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h new file mode 100644 index 00000000000..0fccd6cabb7 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// +#pragma once + +namespace vespalib { + +class IArrayBase { +public: + virtual ~IArrayBase() {} + virtual void resize(size_t sz) = 0; + virtual void reserve(size_t sz) = 0; + virtual void clear() = 0; + virtual IArrayBase *clone() const = 0; + virtual size_t size() const = 0; + bool empty() const { return size() == 0; } +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h index 636586f839e..646ca522ba3 100644 --- a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h +++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h @@ -2,75 +2,12 @@ // #pragma once +#include "polymorphicarray.h" #include <vespa/vespalib/util/memory.h> +#include <vector> namespace vespalib { -/** - * Describes an interface an array of polymorphic types. - * The intention is to allow efficient implementations when that is possible - * while still enjoying the flexibility of the polymorph interface. - * It is not a full feldged Array implementation as std::vector. It contains just - * the minimum required to allow for efficient implementations for document::ArrayFieldValue. - * - * You specify the base type the interface shall provide. This base type must define - * virtual void assign(const B & rhs); - * For use with ComplexArrayT your type also need - * virtual T * clone() const; - */ -template<typename B> -class IArrayT -{ -public: - class iterator { - public: - iterator(IArrayT & a, size_t i) : _a(&a), _i(i) { } - iterator operator+(size_t diff) const { return iterator(*_a, _i + diff); } - iterator& operator++() { ++_i; return *this; } - iterator operator++(int) { iterator other(*this); ++_i; return other; } - bool operator==(const iterator & other) const { return (_a == other._a) && (_i == other._i); } - bool operator!=(const iterator & other) const { return (_i != other._i) || (_a != other._a); } - B & operator*() { return (*_a)[_i]; } - B * operator->() { return &(*_a)[_i]; } - friend ssize_t operator - (const iterator & a, const iterator & b) { return a._i - b._i; } - private: - IArrayT * _a; - size_t _i; - }; - class const_iterator { - public: - const_iterator(const IArrayT & a, size_t i) : _a(&a), _i(i) { } - const_iterator operator+(size_t diff) const { return const_iterator(*_a, _i + diff); } - const_iterator& operator++() { ++_i; return *this; } - const_iterator operator++(int) { const_iterator other(*this); ++_i; return other; } - bool operator==(const const_iterator & other) const { return (_a == other._a) && (_i == other._i); } - bool operator!=(const const_iterator & other) const { return (_i != other._i) || (_a != other._a); } - const B & operator*() const { return (*_a)[_i]; } - const B * operator->() const { return &(*_a)[_i]; } - size_t operator - (const const_iterator & b) const { return _i - b._i; } - private: - const IArrayT * _a; - size_t _i; - }; - typedef std::unique_ptr<IArrayT> UP; - - virtual ~IArrayT() { } - virtual const B & operator [] (size_t i) const = 0; - virtual B & operator [] (size_t i) = 0; - virtual void resize(size_t sz) = 0; - virtual void reserve(size_t sz) = 0; - virtual void clear() = 0; - virtual IArrayT * clone() const = 0; - virtual size_t size() const = 0; - virtual iterator erase(iterator it) = 0; - virtual const_iterator begin() const { return const_iterator(*this, 0); } - virtual const_iterator end() const { return const_iterator(*this, size()); } - virtual iterator begin() { return iterator(*this, 0); } - virtual iterator end() { return iterator(*this, size()); } - bool empty() const { return size() == 0; } - virtual void push_back(const B & v) = 0; -}; - template <typename T, typename B> class PrimitiveArrayT : public IArrayT<B> { diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp index 043cddc3259..357b3d94992 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp @@ -1,449 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "xmlserializable.hpp" -#include <vespa/vespalib/encoding/base64.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <cassert> +#include "xmlserializable.h" +#include "xmlstream.h" +#include <sstream> namespace vespalib { namespace xml { -namespace { - - std::vector<bool> getLegalIdentifierFirstCharacters() { - std::vector<bool> vec(256, false); - for (uint32_t i='a'; i<='z'; ++i) vec[i] = true; - for (uint32_t i='A'; i<='Z'; ++i) vec[i] = true; - vec[':'] = true; - vec['_'] = true; - return vec; - } - - std::vector<bool> getLegalIdentifierCharacters() { - std::vector<bool> vec(getLegalIdentifierFirstCharacters()); - vec['-'] = true; - vec['.'] = true; - for (uint32_t i='0'; i<='9'; ++i) { - vec[i] = true; - } - return vec; - } - - std::vector<bool> getBinaryCharacters() { - std::vector<bool> vec(256, false); - for (uint32_t i=0; i<32; ++i) { - vec[i] = true; - } - vec['\t'] = false; - vec['\n'] = false; - vec['\r'] = false; - vec['\f'] = false; - return vec; - } - - std::vector<bool> getEscapedXmlCharacters() { - std::vector<bool> vec(256, false); - for (uint32_t i=0; i<32; ++i) { - vec[i] = true; - } - vec['\n'] = false; - vec['<'] = true; - vec['>'] = true; - vec['&'] = true; - return vec; - } - - std::vector<bool> legalIdentifierFirstChar( - getLegalIdentifierFirstCharacters()); - std::vector<bool> legalIdentifierChars = getLegalIdentifierCharacters(); - std::vector<bool> binaryChars = getBinaryCharacters(); - std::vector<bool> escapedXmlChars = getEscapedXmlCharacters(); - - bool containsBinaryCharacters(const std::string& s) { - for (int i=0, n=s.size(); i<n; ++i) { - if (binaryChars[static_cast<uint8_t>(s[i])]) return true; - } - return false; - } - - const std::string xmlAttributeEscape(const std::string& s) { - vespalib::asciistream ost; - for (uint32_t i=0, n=s.size(); i<n; ++i) { - if (s[i] == '"' || s[i] == '\n' - || escapedXmlChars[static_cast<uint8_t>(s[i])]) - { - if (s[i] == '<') ost << "<"; - else if (s[i] == '>') ost << ">"; - else if (s[i] == '&') ost << "&"; - else if (s[i] == '"') ost << """; - else { - ost << "&#" << (int) s[i] << ";"; - } - } else { - ost << s[i]; - } - } - return ost.str(); - } - - void writeEscaped(std::ostream& out, const std::string& s) { - for (uint32_t i=0, n=s.size(); i<n; ++i) { - if (escapedXmlChars[static_cast<uint8_t>(s[i])]) { - if (s[i] == '<') out << "<"; - else if (s[i] == '>') out << ">"; - else if (s[i] == '&') out << "&"; - else { - out << "&#" << (int) s[i] << ";"; - } - } else { - out << s[i]; - } - } - } - - void writeBase64Encoded(std::ostream& out, const std::string& s) { - out << vespalib::Base64::encode(&s[0], s.size()); - } -} - -bool isLegalName(const std::string& name) { - if (name.size() == 0) return false; - if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) return false; - for (int i=1, n=name.size(); i<n; ++i) { - if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) return false; - } - return true; -} - -void convertToLegalName(std::string& name) { - if (name.size() == 0) { - name == "__no_name__"; - } else { - if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) { - name[0] = '_'; - } - for (int i=1, n=name.size(); i<n; ++i) { - if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) { - name[i] = '_'; - } - } - } -} - -XmlOutputStream::XmlOutputStream(std::ostream& ostream, - const std::string& indent) - : _indent(indent), - _wrappedStream(ostream), - _tagStack(), - _cachedTag(), - _cachedAttributes(), - _cachedContent() -{ -} - -XmlAttribute::~XmlAttribute() -{ -} - -XmlContent::~XmlContent() -{ -} - -XmlOutputStream::~XmlOutputStream() -{ -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlTag& tag) -{ - //std::cerr << "Trying to add tag " << tag.getName() << ". cached tag is " - // << (void*) _cachedTag.get() << "\n"; - if (_cachedTag.get() != 0) flush(false); - _cachedTag.reset(new XmlTag(tag)); - _cachedContentType = XmlContent::AUTO; - //std::cerr << "Added tag " << _cachedTag->getName() << "\n"; - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlAttribute& attribute) -{ - //std::cerr << "Adding attribute\n"; - if (_cachedTag.get() == 0) { - throw vespalib::IllegalStateException("Cannot add attribute " - + attribute.getName() + ", as no tag is open"); - } - _cachedAttributes.push_back(attribute); - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlEndTag&) -{ - //std::cerr << "Adding endtag\n"; - if (_cachedTag.get()) { - flush(true); - _cachedContentType = XmlContent::ESCAPED; - } else if (_tagStack.empty()) { - throw vespalib::IllegalStateException("No open tags left to end"); - } else { - for (uint32_t i=1; i<_tagStack.size(); ++i) { - _wrappedStream << _indent; - } - _wrappedStream << "</" << _tagStack.back() << ">"; - _tagStack.pop_back(); - if (!_tagStack.empty()) _wrappedStream << '\n'; - _cachedContentType = XmlContent::ESCAPED; - } - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlContent& content) -{ - //std::cerr << "Adding content\n"; - if (_cachedTag.get() == 0 && _tagStack.empty()) { - throw vespalib::IllegalStateException( - "No open tag to write content in"); - } - if (_cachedTag.get() != 0) { - //std::cerr << "Content is '" << content.getContent() << "'\n"; - if (content.getType() == XmlContent::AUTO) { // Do nothing.. Always ok - } else if (_cachedContentType == XmlContent::AUTO) { - _cachedContentType = content.getType(); - } else if (_cachedContentType != content.getType()) { - throw vespalib::IllegalStateException( - "Have already added content of different type"); - } - _cachedContent.push_back(content); - } else { - if (content.getType() == XmlContent::BASE64) { - throw vespalib::IllegalStateException( - "Cannot add Base64 encoded content after tag content"); - } - for (uint32_t i=0; i<_tagStack.size(); ++i) { - _wrappedStream << _indent; - } - _wrappedStream << content.getContent() << '\n'; - } - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlSerializable& serializable) -{ - //std::cerr << "Adding serializable\n"; - serializable.printXml(*this); - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const std::string& content) -{ - //std::cerr << "Adding content string\n"; - return *this << XmlContent(content); -} - -XmlOutputStream& -XmlOutputStream::operator<<(char c) -{ - return *this << XmlContent(std::string(&c, 1)); -} - -XmlOutputStream& -XmlOutputStream::operator<<(int32_t i) -{ - return *this << XmlContent(vespalib::make_string("%d", i)); -} - -XmlOutputStream& -XmlOutputStream::operator<<(int64_t i) -{ - return *this << XmlContent(vespalib::make_string("%" PRId64, i)); -} - -XmlOutputStream& -XmlOutputStream::operator<<(float f) -{ - return *this << XmlContent(vespalib::make_string("%g", f)); -} - -XmlOutputStream& -XmlOutputStream::operator<<(double d) -{ - return *this << XmlContent(vespalib::make_string("%g", d)); -} - -void -XmlOutputStream::flush(bool endTag) -{ - //std::cerr << "Flushing\n"; - if (_cachedTag.get() == 0) { - throw vespalib::IllegalStateException("Cannot write non-existing tag"); - } - for (uint32_t i=0; i<_tagStack.size(); ++i) { - _wrappedStream << _indent; - } - _wrappedStream << '<' << _cachedTag->getName(); - for (std::list<XmlAttribute>::const_iterator it = _cachedAttributes.begin(); - it != _cachedAttributes.end(); ++it) - { - _wrappedStream << ' ' << it->getName() << "=\"" - << xmlAttributeEscape(it->getValue()) << '"'; - } - _cachedAttributes.clear(); - if (_cachedContent.empty() && endTag) { - _wrappedStream << "/>\n"; - } else if (_cachedContent.empty()) { - _wrappedStream << ">\n"; - _tagStack.push_back(_cachedTag->getName()); - } else { - if (_cachedContentType == XmlContent::AUTO) { - _cachedContentType = XmlContent::ESCAPED; - for (std::list<XmlContent>::const_iterator it - = _cachedContent.begin(); it != _cachedContent.end(); ++it) - { - if (containsBinaryCharacters(it->getContent())) { - _cachedContentType = XmlContent::BASE64; - break; - } - } - } - if (_cachedContentType == XmlContent::BASE64) { - _wrappedStream << " binaryencoding=\"base64\""; - } - _wrappedStream << '>'; - for (std::list<XmlContent>::const_iterator it = _cachedContent.begin(); - it != _cachedContent.end(); ++it) - { - if (!endTag) { - _wrappedStream << '\n'; - for (uint32_t i=0; i<=_tagStack.size(); ++i) { - _wrappedStream << _indent; - } - } - switch (_cachedContentType) { - case XmlContent::ESCAPED: { - writeEscaped(_wrappedStream, it->getContent()); - break; - } - case XmlContent::BASE64: { - writeBase64Encoded(_wrappedStream, it->getContent()); - break; - } - default: assert(false); - } - } - _cachedContent.clear(); - if (endTag) { - _wrappedStream << "</" << _cachedTag->getName() << ">\n"; - } else { - _wrappedStream << '\n'; - _tagStack.push_back(_cachedTag->getName()); - } - } - _cachedTag.reset(0); -} - -XmlTag::XmlTag(const XmlTag& tag) - : _name(tag._name), - _attributes(), - _content(), - _flags(tag._flags) -{ -} - -XmlTag::~XmlTag() {} - -XmlTag::XmlTag(const std::string& name, XmlTagFlags flags) - : _name(name), - _attributes(), - _content(), - _flags(flags) -{ - if (_flags == CONVERT_ILLEGAL_CHARACTERS) { - convertToLegalName(_name); - } - if (!isLegalName(_name)) { - throw vespalib::IllegalArgumentException("Name '" + _name + "' contains " - "illegal XML characters and cannot be used as tag name"); - } -} - -XmlAttribute::XmlAttribute(const XmlAttribute& attribute) - : _name(attribute._name), - _value(attribute._value), - _next() -{ -} - -XmlAttribute::XmlAttribute(const std::string& name, const char * value, uint32_t flags) - : _name(name), - _value(), - _next() -{ - vespalib::asciistream ost; - if (flags & HEX) ost << vespalib::hex << "0x"; - ost << value; - _value = ost.str(); - if (!isLegalName(name)) { - throw vespalib::IllegalArgumentException("Name '" + name + "' contains " - "illegal XML characters and cannot be used as attribute name"); - } -} - -XmlEndTag::XmlEndTag() -{ -} - -XmlContent::XmlContent(Type type) - : _type(type), - _content(), - _nextContent(), - _nextTag() -{ -} - -XmlContent::XmlContent() - : _type(AUTO), - _content(), - _nextContent(), - _nextTag() -{ -} - -XmlContent::XmlContent(const XmlContent& content) - : _type(content._type), - _content(content._content), - _nextContent(), - _nextTag() -{ -} - -XmlContent::XmlContent(const std::string& value) - : _type(AUTO), - _content(value), - _nextContent(), - _nextTag() -{ -} - -XmlContentWrapper::XmlContentWrapper(const XmlContentWrapper& wrapper) - : XmlContent(wrapper) -{ -} - -XmlContentWrapper::XmlContentWrapper(const char* value) - : XmlContent(std::string(value)) -{ -} - -XmlContentWrapper::XmlContentWrapper(const char* value, uint32_t size) - : XmlContent(std::string(value, size)) -{ -} - std::string XmlSerializable::toXml(const std::string& indent) const { @@ -452,22 +15,6 @@ XmlSerializable::toXml(const std::string& indent) const printXml(xos); return ost.str(); } -using CharP = char *; -using ConstCharP = const char *; - -template XmlAttribute::XmlAttribute(const std::string &, const std::string &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const vespalib::string &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const vespalib::stringref &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const CharP &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const ConstCharP &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const bool &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const int16_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const int32_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const int64_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const uint16_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const uint32_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const uint64_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const double &, unsigned int); } // xml } // vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h index b688c699d76..a4ddb12ed8d 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h +++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h @@ -1,166 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file xmlserializable.h - * @ingroup util - * - * @brief Interfaces to be used for XML serialization. - * - * This file contains XML utility classes, to make XML serialization simple. - * Rather than users writing their own XML, these tools let you define a tree - * structure, and this library builds the XML for you. This ensures that you - * write legal XML and that stuff that needs to be escaped is. - * <p> - * It defines a superclass for XML serializable classes, called XmlSerializable. - * This is what classes that should be XML serializable will inherit. - * <p> - * When implementing the printXml() function in XmlSerializable, one will - * use the various XML helper classes defined here to build a tree structure - * creating the XML. These are: XmlTag, XmlEndTag, XmlAttribute, and XmlContent. - * Some subclasses exist of XmlContent to facilitate various types of content. - * <p> - * The XmlOutputStream wraps a regular std::ostream. You write XML objects to it - * and it is responsible for writing all the XML code. This way, the XML - * serialization is done without interfering with regular output operators. - * <p> - * For example usage, refer to the unit test: - * vespalib/tests/xmlserializable/xmlserializabletest.cpp - * - */ #pragma once -#include <iosfwd> -#include <list> -#include <memory> +#include <string> namespace vespalib { namespace xml { -class XmlAttribute; -class XmlContent; class XmlOutputStream; -bool isLegalName(const std::string& name); - -enum XmlTagFlags { NONE = 0, CONVERT_ILLEGAL_CHARACTERS = 1 }; - -/** - * @class document::XmlTag - * - * @brief Start a new tag with given name. - */ -class XmlTag { - std::string _name; - std::unique_ptr<XmlAttribute> _attributes; - std::unique_ptr<XmlContent> _content; - XmlTagFlags _flags; -public: - XmlTag(const XmlTag&); - XmlTag(const std::string& name, XmlTagFlags = NONE); - ~XmlTag(); - - const std::string& getName() const { return _name; } -}; - -/** - * @class document::XmlEndTag - * - * @brief Indicates that current tag is closed. - */ -class XmlEndTag { -public: - XmlEndTag(); -}; - -/** - * @class document::XmlAttribute - * - * @brief Defined a single attribute within an XML tag. - * - * When adding an XML to an XML stream, the attribute will be added to the last - * tag added. This can not be called after the last tag opened in the stream is - * closed, so add all attributes before starting to add new XML child tags. - */ -class XmlAttribute { - std::string _name; - std::string _value; - std::unique_ptr<XmlAttribute> _next; -public: - enum Flag { NONE = 0x0, HEX = 0x1 }; - XmlAttribute(const XmlAttribute&); - /** Add any value that can be written to an ostringstream. */ - template<typename T> - XmlAttribute(const std::string& name, const T& value, uint32_t flags = NONE); - XmlAttribute(const std::string& name, const char * value, uint32_t flags = NONE); - ~XmlAttribute(); - - const std::string& getName() const { return _name; } - const std::string& getValue() const { return _value; } -}; - - -/** - * @class document::XmlContent - * - * XML content to be written to stream. By default it will autodetect whether to - * escape or base64 encode content. XmlOutputStream functions taking primitives - * will generate XmlContent instances. - */ -class XmlContent { -public: - enum Type { AUTO, ESCAPED, BASE64 }; -protected: - XmlContent(Type type); -private: - Type _type; - std::string _content; - std::unique_ptr<XmlContent> _nextContent; - std::unique_ptr<XmlTag> _nextTag; - -public: - XmlContent(); - XmlContent(const XmlContent&); - XmlContent(const std::string& value); - ~XmlContent(); - - Type getType() const { return _type; } - const std::string& getContent() const { return _content; } -}; - -/** - * @class document::XmlEscapedContent - * - * Token used to tell that this content field should only be XML escaped. - */ -class XmlEscapedContent : public XmlContent { -public: - XmlEscapedContent() : XmlContent(ESCAPED) {} -}; - -/** - * @class document::XmlBase64Content - * - * Token used to tell that this content field should always be base64 encoded. - */ -class XmlBase64Content : public XmlContent { -public: - XmlBase64Content() : XmlContent(BASE64) {} -}; - -/** - * @class document::XmlContentWrapper - * - * A wrapper class for content that one doesn't want to copy or release - * ownership of. This wrapper merely takes pointer to data, and assumes it - * will stay alive as long as needed. - */ -class XmlContentWrapper : public XmlContent { -public: - XmlContentWrapper(const XmlContentWrapper&); - XmlContentWrapper(const char* value); - XmlContentWrapper(const char* value, uint32_t size); -}; - /** * @class document::XmlSerializable * @@ -178,51 +26,6 @@ public: virtual std::string toXml(const std::string& indent = "") const; }; -/** - * @class document::XmlOutputStream - * - * @brief std::ostream wrapper, only accepting data that will become XML. - * - * After XmlEndTag() has been sent to the stream, the tag is guarantueed to have - * been written. Call isFinalized() to ensure that you have closed all the tags - * that have been opened. Within a tag, the stream will cache some information, - * as more information might be required before knowing what to print. - */ -class XmlOutputStream { - const std::string _indent; - std::ostream& _wrappedStream; - std::list<std::string> _tagStack; - std::unique_ptr<XmlTag> _cachedTag; - std::list<XmlAttribute> _cachedAttributes; - std::list<XmlContent> _cachedContent; - XmlContent::Type _cachedContentType; - - void flush(bool endTag); - -public: - - XmlOutputStream(std::ostream& ostream, const std::string& indent = ""); - ~XmlOutputStream(); - - bool isFinalized() const - { return (_tagStack.empty() && _cachedTag.get() == 0); } - - std::ostream& getWrappedStream() { return _wrappedStream; } - - XmlOutputStream& operator<<(const XmlTag& tag); - XmlOutputStream& operator<<(const XmlAttribute& attribute); - XmlOutputStream& operator<<(const XmlEndTag& endtag); - XmlOutputStream& operator<<(const XmlContent& content); - XmlOutputStream& operator<<(const XmlSerializable& serializable); - - XmlOutputStream& operator<<(const std::string& content); - XmlOutputStream& operator<<(char c); - XmlOutputStream& operator<<(int32_t i); - XmlOutputStream& operator<<(int64_t i); - XmlOutputStream& operator<<(float f); - XmlOutputStream& operator<<(double d); -}; - } // xml // The XmlSerializable and XmlOutputStream is often used in header files @@ -230,8 +33,8 @@ public: // vespalib namespace with all the other classes, use // "using namespace vespalib::xml" within your printXml functions -typedef vespalib::xml::XmlOutputStream XmlOutputStream; -typedef vespalib::xml::XmlSerializable XmlSerializable; +using XmlSerializable = vespalib::xml::XmlSerializable; +using XmlOutputStream = vespalib::xml::XmlOutputStream; } // vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp new file mode 100644 index 00000000000..16fce61ddd1 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp @@ -0,0 +1,463 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "xmlstream.hpp" +#include <vespa/vespalib/encoding/base64.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> + +namespace vespalib::xml { + +namespace { + + std::vector<bool> getLegalIdentifierFirstCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i='a'; i<='z'; ++i) vec[i] = true; + for (uint32_t i='A'; i<='Z'; ++i) vec[i] = true; + vec[':'] = true; + vec['_'] = true; + return vec; + } + + std::vector<bool> getLegalIdentifierCharacters() { + std::vector<bool> vec(getLegalIdentifierFirstCharacters()); + vec['-'] = true; + vec['.'] = true; + for (uint32_t i='0'; i<='9'; ++i) { + vec[i] = true; + } + return vec; + } + + std::vector<bool> getBinaryCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i=0; i<32; ++i) { + vec[i] = true; + } + vec['\t'] = false; + vec['\n'] = false; + vec['\r'] = false; + vec['\f'] = false; + return vec; + } + + std::vector<bool> getEscapedXmlCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i=0; i<32; ++i) { + vec[i] = true; + } + vec['\n'] = false; + vec['<'] = true; + vec['>'] = true; + vec['&'] = true; + return vec; + } + + std::vector<bool> legalIdentifierFirstChar( + getLegalIdentifierFirstCharacters()); + std::vector<bool> legalIdentifierChars = getLegalIdentifierCharacters(); + std::vector<bool> binaryChars = getBinaryCharacters(); + std::vector<bool> escapedXmlChars = getEscapedXmlCharacters(); + + bool containsBinaryCharacters(const std::string& s) { + for (int i=0, n=s.size(); i<n; ++i) { + if (binaryChars[static_cast<uint8_t>(s[i])]) return true; + } + return false; + } + + const std::string xmlAttributeEscape(const std::string& s) { + vespalib::asciistream ost; + for (uint32_t i=0, n=s.size(); i<n; ++i) { + if (s[i] == '"' || s[i] == '\n' + || escapedXmlChars[static_cast<uint8_t>(s[i])]) + { + if (s[i] == '<') ost << "<"; + else if (s[i] == '>') ost << ">"; + else if (s[i] == '&') ost << "&"; + else if (s[i] == '"') ost << """; + else { + ost << "&#" << (int) s[i] << ";"; + } + } else { + ost << s[i]; + } + } + return ost.str(); + } + + void writeEscaped(std::ostream& out, const std::string& s) { + for (uint32_t i=0, n=s.size(); i<n; ++i) { + if (escapedXmlChars[static_cast<uint8_t>(s[i])]) { + if (s[i] == '<') out << "<"; + else if (s[i] == '>') out << ">"; + else if (s[i] == '&') out << "&"; + else { + out << "&#" << (int) s[i] << ";"; + } + } else { + out << s[i]; + } + } + } + + void writeBase64Encoded(std::ostream& out, const std::string& s) { + out << vespalib::Base64::encode(&s[0], s.size()); + } +} + +bool isLegalName(const std::string& name) { + if (name.size() == 0) return false; + if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) return false; + for (int i=1, n=name.size(); i<n; ++i) { + if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) return false; + } + return true; +} + +void convertToLegalName(std::string& name) { + if (name.size() == 0) { + name == "__no_name__"; + } else { + if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) { + name[0] = '_'; + } + for (int i=1, n=name.size(); i<n; ++i) { + if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) { + name[i] = '_'; + } + } + } +} + +XmlOutputStream::XmlOutputStream(std::ostream& ostream, + const std::string& indent) + : _indent(indent), + _wrappedStream(ostream), + _tagStack(), + _cachedTag(), + _cachedAttributes(), + _cachedContent() +{ +} + +XmlAttribute::~XmlAttribute() +{ +} + +XmlContent::~XmlContent() +{ +} + +XmlOutputStream::~XmlOutputStream() +{ +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlTag& tag) +{ + //std::cerr << "Trying to add tag " << tag.getName() << ". cached tag is " + // << (void*) _cachedTag.get() << "\n"; + if (_cachedTag.get() != 0) flush(false); + _cachedTag.reset(new XmlTag(tag)); + _cachedContentType = XmlContent::AUTO; + //std::cerr << "Added tag " << _cachedTag->getName() << "\n"; + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlAttribute& attribute) +{ + //std::cerr << "Adding attribute\n"; + if (_cachedTag.get() == 0) { + throw vespalib::IllegalStateException("Cannot add attribute " + + attribute.getName() + ", as no tag is open"); + } + _cachedAttributes.push_back(attribute); + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlEndTag&) +{ + //std::cerr << "Adding endtag\n"; + if (_cachedTag.get()) { + flush(true); + _cachedContentType = XmlContent::ESCAPED; + } else if (_tagStack.empty()) { + throw vespalib::IllegalStateException("No open tags left to end"); + } else { + for (uint32_t i=1; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << "</" << _tagStack.back() << ">"; + _tagStack.pop_back(); + if (!_tagStack.empty()) _wrappedStream << '\n'; + _cachedContentType = XmlContent::ESCAPED; + } + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlContent& content) +{ + //std::cerr << "Adding content\n"; + if (_cachedTag.get() == 0 && _tagStack.empty()) { + throw vespalib::IllegalStateException( + "No open tag to write content in"); + } + if (_cachedTag.get() != 0) { + //std::cerr << "Content is '" << content.getContent() << "'\n"; + if (content.getType() == XmlContent::AUTO) { // Do nothing.. Always ok + } else if (_cachedContentType == XmlContent::AUTO) { + _cachedContentType = content.getType(); + } else if (_cachedContentType != content.getType()) { + throw vespalib::IllegalStateException( + "Have already added content of different type"); + } + _cachedContent.push_back(content); + } else { + if (content.getType() == XmlContent::BASE64) { + throw vespalib::IllegalStateException( + "Cannot add Base64 encoded content after tag content"); + } + for (uint32_t i=0; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << content.getContent() << '\n'; + } + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlSerializable& serializable) +{ + //std::cerr << "Adding serializable\n"; + serializable.printXml(*this); + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const std::string& content) +{ + //std::cerr << "Adding content string\n"; + return *this << XmlContent(content); +} + +XmlOutputStream& +XmlOutputStream::operator<<(char c) +{ + return *this << XmlContent(std::string(&c, 1)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(int32_t i) +{ + return *this << XmlContent(vespalib::make_string("%d", i)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(int64_t i) +{ + return *this << XmlContent(vespalib::make_string("%" PRId64, i)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(float f) +{ + return *this << XmlContent(vespalib::make_string("%g", f)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(double d) +{ + return *this << XmlContent(vespalib::make_string("%g", d)); +} + +void +XmlOutputStream::flush(bool endTag) +{ + //std::cerr << "Flushing\n"; + if (_cachedTag.get() == 0) { + throw vespalib::IllegalStateException("Cannot write non-existing tag"); + } + for (uint32_t i=0; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << '<' << _cachedTag->getName(); + for (std::list<XmlAttribute>::const_iterator it = _cachedAttributes.begin(); + it != _cachedAttributes.end(); ++it) + { + _wrappedStream << ' ' << it->getName() << "=\"" + << xmlAttributeEscape(it->getValue()) << '"'; + } + _cachedAttributes.clear(); + if (_cachedContent.empty() && endTag) { + _wrappedStream << "/>\n"; + } else if (_cachedContent.empty()) { + _wrappedStream << ">\n"; + _tagStack.push_back(_cachedTag->getName()); + } else { + if (_cachedContentType == XmlContent::AUTO) { + _cachedContentType = XmlContent::ESCAPED; + for (std::list<XmlContent>::const_iterator it + = _cachedContent.begin(); it != _cachedContent.end(); ++it) + { + if (containsBinaryCharacters(it->getContent())) { + _cachedContentType = XmlContent::BASE64; + break; + } + } + } + if (_cachedContentType == XmlContent::BASE64) { + _wrappedStream << " binaryencoding=\"base64\""; + } + _wrappedStream << '>'; + for (std::list<XmlContent>::const_iterator it = _cachedContent.begin(); + it != _cachedContent.end(); ++it) + { + if (!endTag) { + _wrappedStream << '\n'; + for (uint32_t i=0; i<=_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + } + switch (_cachedContentType) { + case XmlContent::ESCAPED: { + writeEscaped(_wrappedStream, it->getContent()); + break; + } + case XmlContent::BASE64: { + writeBase64Encoded(_wrappedStream, it->getContent()); + break; + } + default: assert(false); + } + } + _cachedContent.clear(); + if (endTag) { + _wrappedStream << "</" << _cachedTag->getName() << ">\n"; + } else { + _wrappedStream << '\n'; + _tagStack.push_back(_cachedTag->getName()); + } + } + _cachedTag.reset(0); +} + +XmlTag::XmlTag(const XmlTag& tag) + : _name(tag._name), + _attributes(), + _content(), + _flags(tag._flags) +{ +} + +XmlTag::~XmlTag() {} + +XmlTag::XmlTag(const std::string& name, XmlTagFlags flags) + : _name(name), + _attributes(), + _content(), + _flags(flags) +{ + if (_flags == CONVERT_ILLEGAL_CHARACTERS) { + convertToLegalName(_name); + } + if (!isLegalName(_name)) { + throw vespalib::IllegalArgumentException("Name '" + _name + "' contains " + "illegal XML characters and cannot be used as tag name"); + } +} + +XmlAttribute::XmlAttribute(const XmlAttribute& attribute) + : _name(attribute._name), + _value(attribute._value), + _next() +{ +} + +XmlAttribute::XmlAttribute(const std::string& name, const char * value, uint32_t flags) + : _name(name), + _value(), + _next() +{ + vespalib::asciistream ost; + if (flags & HEX) ost << vespalib::hex << "0x"; + ost << value; + _value = ost.str(); + if (!isLegalName(name)) { + throw vespalib::IllegalArgumentException("Name '" + name + "' contains " + "illegal XML characters and cannot be used as attribute name"); + } +} + +XmlEndTag::XmlEndTag() +{ +} + +XmlContent::XmlContent(Type type) + : _type(type), + _content(), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent() + : _type(AUTO), + _content(), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent(const XmlContent& content) + : _type(content._type), + _content(content._content), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent(const std::string& value) + : _type(AUTO), + _content(value), + _nextContent(), + _nextTag() +{ +} + +XmlContentWrapper::XmlContentWrapper(const XmlContentWrapper& wrapper) + : XmlContent(wrapper) +{ +} + +XmlContentWrapper::XmlContentWrapper(const char* value) + : XmlContent(std::string(value)) +{ +} + +XmlContentWrapper::XmlContentWrapper(const char* value, uint32_t size) + : XmlContent(std::string(value, size)) +{ +} + +using CharP = char *; +using ConstCharP = const char *; + +template XmlAttribute::XmlAttribute(const std::string &, const std::string &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const vespalib::string &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const vespalib::stringref &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const CharP &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const ConstCharP &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const bool &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const int16_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const int32_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const int64_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const uint16_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const uint32_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const uint64_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const double &, unsigned int); + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.h b/staging_vespalib/src/vespa/vespalib/util/xmlstream.h new file mode 100644 index 00000000000..8ed781e57c3 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.h @@ -0,0 +1,210 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file xmlserializable.h + * @ingroup util + * + * @brief Interfaces to be used for XML serialization. + * + * This file contains XML utility classes, to make XML serialization simple. + * Rather than users writing their own XML, these tools let you define a tree + * structure, and this library builds the XML for you. This ensures that you + * write legal XML and that stuff that needs to be escaped is. + * <p> + * It defines a superclass for XML serializable classes, called XmlSerializable. + * This is what classes that should be XML serializable will inherit. + * <p> + * When implementing the printXml() function in XmlSerializable, one will + * use the various XML helper classes defined here to build a tree structure + * creating the XML. These are: XmlTag, XmlEndTag, XmlAttribute, and XmlContent. + * Some subclasses exist of XmlContent to facilitate various types of content. + * <p> + * The XmlOutputStream wraps a regular std::ostream. You write XML objects to it + * and it is responsible for writing all the XML code. This way, the XML + * serialization is done without interfering with regular output operators. + * <p> + * For example usage, refer to the unit test: + * vespalib/tests/xmlserializable/xmlserializabletest.cpp + * + */ + +#pragma once + +#include "xmlserializable.h" +#include <iosfwd> +#include <list> +#include <memory> + +namespace vespalib::xml { + +class XmlAttribute; +class XmlContent; +class XmlOutputStream; + +bool isLegalName(const std::string& name); + +enum XmlTagFlags { NONE = 0, CONVERT_ILLEGAL_CHARACTERS = 1 }; + +/** + * @class document::XmlTag + * + * @brief Start a new tag with given name. + */ +class XmlTag { + std::string _name; + std::unique_ptr<XmlAttribute> _attributes; + std::unique_ptr<XmlContent> _content; + XmlTagFlags _flags; +public: + XmlTag(const XmlTag&); + XmlTag(const std::string& name, XmlTagFlags = NONE); + ~XmlTag(); + + const std::string& getName() const { return _name; } +}; + +/** + * @class document::XmlEndTag + * + * @brief Indicates that current tag is closed. + */ +class XmlEndTag { +public: + XmlEndTag(); +}; + +/** + * @class document::XmlAttribute + * + * @brief Defined a single attribute within an XML tag. + * + * When adding an XML to an XML stream, the attribute will be added to the last + * tag added. This can not be called after the last tag opened in the stream is + * closed, so add all attributes before starting to add new XML child tags. + */ +class XmlAttribute { + std::string _name; + std::string _value; + std::unique_ptr<XmlAttribute> _next; +public: + enum Flag { NONE = 0x0, HEX = 0x1 }; + XmlAttribute(const XmlAttribute&); + /** Add any value that can be written to an ostringstream. */ + template<typename T> + XmlAttribute(const std::string& name, const T& value, uint32_t flags = NONE); + XmlAttribute(const std::string& name, const char * value, uint32_t flags = NONE); + ~XmlAttribute(); + + const std::string& getName() const { return _name; } + const std::string& getValue() const { return _value; } +}; + + +/** + * @class document::XmlContent + * + * XML content to be written to stream. By default it will autodetect whether to + * escape or base64 encode content. XmlOutputStream functions taking primitives + * will generate XmlContent instances. + */ +class XmlContent { +public: + enum Type { AUTO, ESCAPED, BASE64 }; +protected: + XmlContent(Type type); +private: + Type _type; + std::string _content; + std::unique_ptr<XmlContent> _nextContent; + std::unique_ptr<XmlTag> _nextTag; + +public: + XmlContent(); + XmlContent(const XmlContent&); + XmlContent(const std::string& value); + ~XmlContent(); + + Type getType() const { return _type; } + const std::string& getContent() const { return _content; } +}; + +/** + * @class document::XmlEscapedContent + * + * Token used to tell that this content field should only be XML escaped. + */ +class XmlEscapedContent : public XmlContent { +public: + XmlEscapedContent() : XmlContent(ESCAPED) {} +}; + +/** + * @class document::XmlBase64Content + * + * Token used to tell that this content field should always be base64 encoded. + */ +class XmlBase64Content : public XmlContent { +public: + XmlBase64Content() : XmlContent(BASE64) {} +}; + +/** + * @class document::XmlContentWrapper + * + * A wrapper class for content that one doesn't want to copy or release + * ownership of. This wrapper merely takes pointer to data, and assumes it + * will stay alive as long as needed. + */ +class XmlContentWrapper : public XmlContent { +public: + XmlContentWrapper(const XmlContentWrapper&); + XmlContentWrapper(const char* value); + XmlContentWrapper(const char* value, uint32_t size); +}; + +/** + * @class document::XmlOutputStream + * + * @brief std::ostream wrapper, only accepting data that will become XML. + * + * After XmlEndTag() has been sent to the stream, the tag is guarantueed to have + * been written. Call isFinalized() to ensure that you have closed all the tags + * that have been opened. Within a tag, the stream will cache some information, + * as more information might be required before knowing what to print. + */ +class XmlOutputStream { + const std::string _indent; + std::ostream& _wrappedStream; + std::list<std::string> _tagStack; + std::unique_ptr<XmlTag> _cachedTag; + std::list<XmlAttribute> _cachedAttributes; + std::list<XmlContent> _cachedContent; + XmlContent::Type _cachedContentType; + + void flush(bool endTag); + +public: + + XmlOutputStream(std::ostream& ostream, const std::string& indent = ""); + ~XmlOutputStream(); + + bool isFinalized() const + { return (_tagStack.empty() && _cachedTag.get() == 0); } + + std::ostream& getWrappedStream() { return _wrappedStream; } + + XmlOutputStream& operator<<(const XmlTag& tag); + XmlOutputStream& operator<<(const XmlAttribute& attribute); + XmlOutputStream& operator<<(const XmlEndTag& endtag); + XmlOutputStream& operator<<(const XmlContent& content); + XmlOutputStream& operator<<(const XmlSerializable& serializable); + + XmlOutputStream& operator<<(const std::string& content); + XmlOutputStream& operator<<(char c); + XmlOutputStream& operator<<(int32_t i); + XmlOutputStream& operator<<(int64_t i); + XmlOutputStream& operator<<(float f); + XmlOutputStream& operator<<(double d); +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp b/staging_vespalib/src/vespa/vespalib/util/xmlstream.hpp index 0da684d6b28..243efe9e94b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp +++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.hpp @@ -2,12 +2,11 @@ #pragma once -#include "xmlserializable.h" +#include "xmlstream.h" #include <vespa/vespalib/util/exceptions.h> #include <sstream> -namespace vespalib { -namespace xml { +namespace vespalib::xml { template<typename T> XmlAttribute::XmlAttribute(const std::string& name, const T& value, uint32_t flags) @@ -20,10 +19,9 @@ XmlAttribute::XmlAttribute(const std::string& name, const T& value, uint32_t fla ost << value; _value = ost.str(); if (!isLegalName(name)) { - throw vespalib::IllegalArgumentException("Name '" + name + "' contains " + throw IllegalArgumentException("Name '" + name + "' contains " "illegal XML characters and cannot be used as attribute name"); } } } -} diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index b7d3554e506..26317465b5a 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -2,15 +2,15 @@ #include <vespa/document/base/testdocrepo.h> #include <cppunit/extensions/HelperMacros.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/documentapi/documentapi.h> -#include <vespa/messagebus/emptyreply.h> #include <vespa/storage/storageserver/documentapiconverter.h> #include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/storageapi/message/persistence.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vespalib/testkit/test_kit.h> using document::DataType; diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 18fc5efb4b5..79150c160d5 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -19,8 +19,8 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/config/config.h> #include <unordered_map> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.bucketdb.manager"); namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index d3a939d90c5..09cbc60b58e 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -25,6 +25,7 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/config/subscription/configuri.h> #include <unordered_set> +#include <list> namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp index 80f82e65a9a..39e45e1b541 100644 --- a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp +++ b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "distribution_hash_normalizer.h" -#include <vespa/log/log.h> #include <vespa/vespalib/stllike/asciistream.h> #include <boost/spirit/include/qi.hpp> #include <boost/spirit/include/phoenix_core.hpp> @@ -13,6 +12,7 @@ #include <iterator> #include <functional> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.bucketdb.distribution_hash_normalizer"); // TODO diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp index 178731469ba..49022db523b 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp @@ -13,7 +13,7 @@ #include <vespa/config/helper/configgetter.hpp> #include <iomanip> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.bucketdb.initializer"); namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h index f56b45c71b3..92c00c4420a 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h @@ -50,6 +50,7 @@ #include <vespa/vespalib/util/sync.h> #include <vespa/vdslib/state/nodestate.h> #include <vespa/config/subscription/configuri.h> +#include <list> namespace storage { diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.cpp b/storage/src/vespa/storage/bucketmover/bucketmover.cpp index bda0a394802..b38c061de44 100644 --- a/storage/src/vespa/storage/bucketmover/bucketmover.cpp +++ b/storage/src/vespa/storage/bucketmover/bucketmover.cpp @@ -8,10 +8,10 @@ #include <vespa/storage/storageutil/log.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".bucketmover"); -namespace storage { -namespace bucketmover { +namespace storage::bucketmover { BucketMover::BucketMover(const config::ConfigUri & configUri, ServiceLayerComponentRegister& reg) @@ -526,5 +526,4 @@ BucketMover::printRunStatisticsHtml(std::ostream& out, rs.print(out, true, ""); } -} // bucketmover -} // storage +} diff --git a/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp b/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp index 66eb891840e..76a38e3e730 100644 --- a/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp +++ b/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "cpureporter.h" #include "kernelmetrictool.h" - #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/text/stringtokenizer.h> -#include <array> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".cpureporter"); namespace storage { diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp index 3111508fddc..8cb5c9018f4 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp +++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp @@ -1,20 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statusmetricconsumer.h" - +#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <boost/assign.hpp> #include <boost/lexical_cast.hpp> -#include <vespa/log/log.h> #include <vespa/metrics/printutils.h> #include <vespa/metrics/jsonwriter.h> #include <vespa/metrics/textwriter.h> #include <vespa/metrics/xmlwriter.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> -#include <vespa/storageframework/generic/status/htmlstatusreporter.h> - +#include <vespa/vespalib/util/xmlstream.h> +#include <vespa/log/log.h> LOG_SETUP(".status.metricreporter"); namespace storage { diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp index d3b6370347b..50097b132f6 100644 --- a/storage/src/vespa/storage/common/storagelink.cpp +++ b/storage/src/vespa/storage/common/storagelink.cpp @@ -5,7 +5,7 @@ #include <vespa/vespalib/util/backtrace.h> #include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".application.link"); using std::shared_ptr; diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 10d135ccf2d..57e8c55c23b 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -3,6 +3,7 @@ #include <vespa/document/select/parser.h> #include <vespa/document/select/traversingvisitor.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".distributorconfiguration"); diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index 068f42f8b31..9051bbac0f0 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -7,8 +7,9 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/multioperation.h> +#include <vespa/vespalib/util/xmlstream.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.bucketdb.updater"); using storage::lib::Node; diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index 3f0989e9b4f..3bd86fc3db6 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -17,6 +17,7 @@ #include <vespa/storageapi/messageapi/messagehandler.h> #include <set> #include <deque> +#include <list> namespace storage::distributor { diff --git a/storage/src/vespa/storage/distributor/messageguard.h b/storage/src/vespa/storage/distributor/messageguard.h index 8d549965136..2944cc20c70 100644 --- a/storage/src/vespa/storage/distributor/messageguard.h +++ b/storage/src/vespa/storage/distributor/messageguard.h @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "pendingclusterstate.h" #include <vespa/storage/common/messagesender.h> -#include <vespa/storage/distributor/pendingclusterstate.h> +#include <vespa/vespalib/util/sync.h> namespace storage { diff --git a/storage/src/vespa/storage/distributor/messagetracker.cpp b/storage/src/vespa/storage/distributor/messagetracker.cpp index aeca85e821d..0b213a85dd3 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.cpp +++ b/storage/src/vespa/storage/distributor/messagetracker.cpp @@ -33,7 +33,7 @@ MessageTracker::handleReply(api::BucketReply& reply) { std::map<uint64_t, uint16_t>::iterator found = _sentMessages.find(reply.getMsgId()); if (found == _sentMessages.end()) { - LOG(warning, "Received reply %" PRIu64 " for callback which we have no recollection of", reply.getMsgId()); + LOG(warning, "Received reply %zu for callback which we have no recollection of", reply.getMsgId()); return (uint16_t)-1; } else { uint16_t node = found->second; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp index 29e65bfcc00..e2417ea1585 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp @@ -1,14 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "joinoperation.h" -#include <vespa/storage/distributor/idealstatemanager.h> -#include <vespa/storage/distributor/pendingmessagetracker.h> -#include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/vdslib/state/clusterstate.h> - -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.join"); using namespace storage::distributor; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 0821408560b..80a79a8cde9 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -2,7 +2,7 @@ #include "mergeoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.merge"); namespace storage::distributor { diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp index 7601d7faa6d..a6986c62715 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp @@ -5,7 +5,7 @@ #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.split"); diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp index fba6b131e09..6f331e23e2c 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp @@ -4,9 +4,9 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/printable.hpp> #include <sstream> +#include <cassert> -namespace storage { -namespace distributor { +namespace storage::distributor { BucketInstance::BucketInstance( const document::BucketId& id, const api::BucketInfo& info, @@ -197,5 +197,4 @@ OperationTargetResolverImpl::getAllInstances(OperationType type, return instances; } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index 9497bf05f17..5a2f0d35e8c 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -4,7 +4,7 @@ #include "bucketdbupdater.h" #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storage/common/bucketoperationlogger.h> -#include <vespa/vespalib/util/xmlserializable.hpp> +#include <vespa/vespalib/util/xmlstream.hpp> #include <vespa/log/log.h> LOG_SETUP(".pendingclusterstate"); diff --git a/storage/src/vespa/storage/distributor/sentmessagemap.cpp b/storage/src/vespa/storage/distributor/sentmessagemap.cpp index 7efbd3f55d2..02a6da98780 100644 --- a/storage/src/vespa/storage/distributor/sentmessagemap.cpp +++ b/storage/src/vespa/storage/distributor/sentmessagemap.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "sentmessagemap.h" - #include <vespa/storage/distributor/operations/operation.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.map"); diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp index 883dc13cdb3..53967c1cae2 100644 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp +++ b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp @@ -6,9 +6,9 @@ #include <vespa/storage/storageutil/piechart.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/messageapi/storagemessage.h> +#include <sstream> -#include <vespa/log/log.h> - +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".memory.status.viewer"); using storage::framework::defaultimplementation::MemoryState; diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 00dce18e084..045a7514928 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/component/vtag.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".status"); diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index 2861ab904ff..3d9586ea398 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -5,7 +5,7 @@ #include <vespa/storage/bucketmover/htmltable.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".deadlock.detector"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp index b26c6ec893f..0a173b2945c 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp @@ -7,7 +7,7 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/backtrace.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.bucketownershipnotifier"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/fieldvisitor.cpp b/storage/src/vespa/storage/persistence/fieldvisitor.cpp index e67d15d4e91..6657a73543e 100644 --- a/storage/src/vespa/storage/persistence/fieldvisitor.cpp +++ b/storage/src/vespa/storage/persistence/fieldvisitor.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // @author Vegard Sjonfjell -#include <vespa/storage/persistence/fieldvisitor.h> + +#include "fieldvisitor.h" +#include <vespa/document/select/valuenodes.h> namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 43d5de4a5a0..128eff61436 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -19,6 +19,7 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.filestor.manager"); using std::shared_ptr; diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h index 5e45e846337..9938d6522e7 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <memory> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/persistence/messages.h> #include <vespa/storage/persistence/types.h> +#include <vespa/document/bucket/bucketidlist.h> #include <vespa/vespalib/util/sync.h> #include <vespa/config/config.h> @@ -51,8 +51,7 @@ private: } bool requestModifiedBucketsFromProvider(); void nextRecheckChunk(std::vector<RecheckBucketInfoCommand::SP>&); - void dispatchAllToPersistenceQueues( - const std::vector<RecheckBucketInfoCommand::SP>&); + void dispatchAllToPersistenceQueues(const std::vector<RecheckBucketInfoCommand::SP>&); spi::PersistenceProvider& _provider; ServiceLayerComponent::UP _component; @@ -60,7 +59,7 @@ private: config::ConfigFetcher _configFetcher; vespalib::Monitor _monitor; vespalib::Lock _stateLock; - document::BucketId::List _rechecksNotStarted; + document::bucket::BucketIdList _rechecksNotStarted; size_t _pendingRequests; size_t _maxPendingChunkSize; bool _singleThreadMode; // For unit testing only diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 10a5a7505dd..011c6ecc1b0 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -11,7 +11,8 @@ #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> + +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.thread"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 2d8a8561ba6..202a7c5cac5 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -4,7 +4,7 @@ #include <vespa/config/config.h> #include <vespa/config/helper/configgetter.hpp> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.util"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.cpp b/storage/src/vespa/storage/persistence/splitbitdetector.cpp index 62039f36133..cf3fde77b2b 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.cpp +++ b/storage/src/vespa/storage/persistence/splitbitdetector.cpp @@ -1,11 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/persistence/splitbitdetector.h> -#include <vespa/storage/persistence/bucketprocessor.h> +#include "splitbitdetector.h" +#include "bucketprocessor.h" #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/base/documentid.h> +#include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.split.bitdetector"); namespace storage { diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index 6ea898c344c..c285645309a 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bouncer.h" - #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/persistence.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".bouncer"); diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp b/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp index 14b6adcb9a4..05b581d839a 100644 --- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp +++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp @@ -7,10 +7,10 @@ #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/config/config.h> #include <vespa/storage/bucketdb/lockablemap.hpp> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".bucketintegritychecker"); using std::shared_ptr; diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h index 3add8c56e4b..f55cb18366c 100644 --- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h +++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h @@ -16,6 +16,7 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/config/config.h> +#include <list> namespace storage { diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp index 27261424894..3390afcb9ec 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp @@ -7,7 +7,7 @@ #include <vespa/storage/common/nodestateupdater.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".bucketownershiphandler"); namespace storage { @@ -56,8 +56,7 @@ void ChangedBucketOwnershipHandler::reloadClusterState() { vespalib::LockGuard guard(_stateLock); - lib::ClusterState::CSP newState(_component.getStateUpdater() - .getSystemState()); + lib::ClusterState::CSP newState(_component.getStateUpdater().getSystemState()); setCurrentOwnershipWithStateNoLock(*newState); } diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 1c93cefefbc..f32b1c242cf 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -10,8 +10,8 @@ #include <vespa/storage/common/nodestateupdater.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_map.hpp> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".communication.manager"); namespace storage { diff --git a/storage/src/vespa/storage/storageserver/fnetlistener.cpp b/storage/src/vespa/storage/storageserver/fnetlistener.cpp index 5bbeba4aa16..7daf2fb4777 100644 --- a/storage/src/vespa/storage/storageserver/fnetlistener.cpp +++ b/storage/src/vespa/storage/storageserver/fnetlistener.cpp @@ -5,6 +5,7 @@ #include <vespa/storageapi/message/state.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/host_name.h> +#include <sstream> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp index 2a4465c62bc..f770fc0af51 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.cpp +++ b/storage/src/vespa/storage/storageserver/opslogger.cpp @@ -2,6 +2,7 @@ #include "opslogger.h" #include <vespa/storageapi/message/persistence.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".operationslogger"); diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 6940706e211..aff3cb0725a 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -12,6 +12,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> #include <unordered_map> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance"); diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index 0df82dad80f..5e1f8b4df79 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -11,6 +11,7 @@ #include <vespa/storage/common/statusmessages.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/vespalib/util/stringfmt.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".visitor.manager"); diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index e57bcebf372..aeea3c7ba8e 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -14,10 +14,11 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/base/exceptions.h> -#include <locale> #include <vespa/vespalib/stllike/hash_map.hpp> -#include <vespa/log/log.h> +#include <locale> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".visitor.thread"); using storage::api::ReturnCode; diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp index 07ba2773e91..f9924c953c2 100644 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp +++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bucketinfo.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/xmlstream.h> -namespace storage { -namespace api { +namespace storage::api { BucketInfo::BucketInfo() : _lastModified(0), @@ -124,5 +124,4 @@ BucketInfo::printXml(vespalib::XmlOutputStream& xos) const << XmlAttribute("lastmodified", _lastModified); } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp index 1ad07387ba4..9d4d25d9d8e 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp @@ -13,14 +13,13 @@ #include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/vespalib/util/exceptions.h> - #include <vespa/vespalib/util/growablebytebuffer.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".storage.api.mbusprot.serialization.base"); -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { ProtocolSerialization::ProtocolSerialization( const document::DocumentTypeRepo::SP& repo) @@ -299,5 +298,4 @@ ProtocolSerialization::decodeReply(mbus::BlobRef data, return StorageReply::UP(new StorageReply(SRep::SP(reply.release()))); } -} // mbusprot -} // storage +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 62a85e0f45e..87dcacb0fb9 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -5,12 +5,12 @@ #include "storagereply.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/util/stringutil.h> +#include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.api.mbusprot.protocol"); -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { mbus::string StorageProtocol::NAME = "StorageProtocol"; @@ -169,5 +169,4 @@ StorageProtocol::decode(const vespalib::Version & version, return mbus::Routable::UP(); } -} // mbusprot -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index 536135c48e6..93c65e6c745 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -2,13 +2,20 @@ #include "storagemessage.h" -#include <vespa/vespalib/util/exceptions.h> #include <vespa/messagebus/routing/verbatimdirective.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/asciistream.h> #include <sstream> +#include <cassert> + +namespace storage::api { + +namespace { -namespace storage { -namespace api { +vespalib::Lock _G_msgIdLock; + +} static const vespalib::string STORAGEADDRESS_PREFIX = "storage/cluster."; @@ -177,6 +184,20 @@ MessageType::MessageType::get(Id id) } return *it->second; } +MessageType::MessageType(const vespalib::stringref & name, Id id, + const MessageType* replyOf) + : _name(name), _id(id), _reply(NULL), _replyOf(replyOf) +{ + _codes[id] = this; + if (_replyOf != 0) { + assert(_replyOf->_reply == 0); + // Ugly cast to let initialization work + MessageType& type = const_cast<MessageType&>(*_replyOf); + type._reply = this; + } +} + +MessageType::~MessageType() {} void MessageType::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -305,17 +326,14 @@ StorageMessageAddress::print(vespalib::asciistream & out) const } } -TransportContext::~TransportContext() -{ -} +TransportContext::~TransportContext() { } -vespalib::Lock StorageMessage::_msgIdLock; StorageMessage::Id StorageMessage::_lastMsgId = 1000; StorageMessage::Id StorageMessage::generateMsgId() { - vespalib::LockGuard sync(_msgIdLock); + vespalib::LockGuard sync(_G_msgIdLock); Id msgId = _lastMsgId++; _lastMsgId &= ((Id(-1) << 8) >> 8); return msgId; @@ -343,7 +361,7 @@ StorageMessage::~StorageMessage() { } void StorageMessage::setNewMsgId() { - vespalib::LockGuard sync(_msgIdLock); + vespalib::LockGuard sync(_G_msgIdLock); _msgId = _lastMsgId++; _lastMsgId &= ((Id(-1) << 8) >> 8); } @@ -353,5 +371,5 @@ StorageMessage::getSummary() const { return toString(); } -} // api -} // storage +} + diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h index 59ae462432b..442ee0c9899 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -19,7 +19,6 @@ #include <vespa/vdslib/state/nodetype.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/printable.h> -#include <vespa/vespalib/util/sync.h> #include <map> namespace vespalib { @@ -82,10 +81,6 @@ namespace api { * This is used to be able to deserialize messages of various classes. */ class MessageType : public vespalib::Printable { -private: - MessageType(const MessageType &); - MessageType& operator=(const MessageType &); - public: enum Id { GET_ID = 4, @@ -172,18 +167,7 @@ private: MessageType *_reply; const MessageType *_replyOf; - MessageType(const vespalib::stringref & name, Id id, - const MessageType* replyOf = 0) - : _name(name), _id(id), _reply(NULL), _replyOf(replyOf) - { - _codes[id] = this; - if (_replyOf != 0) { - assert(_replyOf->_reply == 0); - // Ugly cast to let initialization work - MessageType& type = const_cast<MessageType&>(*_replyOf); - type._reply = this; - } - } + MessageType(const vespalib::stringref & name, Id id, const MessageType* replyOf = 0); public: static const MessageType DOCBLOCK; static const MessageType DOCBLOCK_REPLY; @@ -264,16 +248,17 @@ public: static const MessageType& get(Id id); + MessageType(const MessageType &) = delete; + MessageType& operator=(const MessageType &) = delete; + ~MessageType(); Id getId() const { return _id; } static Id getMaxId() { return MESSAGETYPE_MAX_ID; } const vespalib::string& getName() const { return _name; } bool isReply() const { return (_replyOf != 0); } - /** Only valid to call on replies. */ - const MessageType& getCommandType() const - { assert(_replyOf); return *_replyOf; } - /** Only valid to call on commands. */ - const MessageType& getReplyType() const - { assert(_reply); return *_reply; } + /** Only valid to call on replies. */ + const MessageType& getCommandType() const { return *_replyOf; } + /** Only valid to call on commands. */ + const MessageType& getReplyType() const { return *_reply; } bool operator==(const MessageType& type) const { return (_id == type._id); } bool operator!=(const MessageType& type) const { return (_id != type._id); } @@ -294,7 +279,7 @@ private: mbus::Route _route; bool _retryEnabled; Protocol _protocol; - // Used for internal VDS addresses only + // Used for internal VDS addresses only vespalib::string _cluster; const lib::NodeType* _type; uint16_t _index; @@ -346,7 +331,6 @@ public: static const char* getPriorityString(Priority); private: - static vespalib::Lock _msgIdLock; static Id _lastMsgId; StorageMessage& operator=(const StorageMessage&); diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp index 125e01db179..a19851ae01f 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "memorystate.h" +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".memory.state"); -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { MemoryState::Entry::Entry() : _currentUsedSize(0), @@ -223,6 +222,4 @@ MemoryState::print(std::ostream& out, bool verbose, out << "\n" << indent << "}"; } -} // defaultimplementation -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp index 55723eec558..5d5d0b97541 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simplememorylogic.h" - #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".memory.logic.simple"); -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { SimpleMemoryLogic::SimpleMemoryLogic(Clock& c, uint64_t maxMemory) : _cacheThreshold(0.98), @@ -232,6 +229,4 @@ SimpleMemoryLogic::print(std::ostream& out, bool verbose, _state.print(out, verbose, indent + " "); } -} // defaultimplementation -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp index 3e3785f67b1..eaa66fedc88 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp @@ -4,7 +4,7 @@ #include "threadpoolimpl.h" #include <vespa/storageframework/generic/clock/clock.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".framework.thread.impl"); namespace storage::framework::defaultimplementation { diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h index f7229a1d297..2704f90d5f1 100644 --- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h +++ b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h @@ -9,6 +9,7 @@ #include <vespa/vespalib/util/printable.h> #include <vespa/vespalib/stllike/string.h> #include <map> +#include <sstream> namespace storage::framework { diff --git a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h b/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h index 522a6480a75..2f38b89fa15 100644 --- a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h +++ b/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h @@ -18,7 +18,7 @@ #pragma once #include "statusreporter.h" -#include <vespa/vespalib/util/xmlserializable.h> +#include <vespa/vespalib/util/xmlstream.h> namespace storage::framework { diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 56c95e7fbc7..7b5e5f17671 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -368,7 +368,7 @@ SearchVisitorFactory::makeVisitor(StorageComponent& component, } void -SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const IteratorContent & c) +SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const Content & c) { const document::FieldValue & value = c.getValue(); LOG(debug, "AttributeInserter: Adding value '%s'(%d) to attribute '%s' for docid '%d'", @@ -402,13 +402,13 @@ SearchVisitor::PositionInserter::PositionInserter(search::AttributeVector & attr SearchVisitor::PositionInserter::~PositionInserter() {} void -SearchVisitor::PositionInserter::onPrimitive(uint32_t, const IteratorContent & c) +SearchVisitor::PositionInserter::onPrimitive(uint32_t, const Content & c) { (void) c; } void -SearchVisitor::PositionInserter::onStructStart(const IteratorContent & c) +SearchVisitor::PositionInserter::onStructStart(const Content & c) { const document::StructuredFieldValue & value = static_cast<const document::StructuredFieldValue &>(c.getValue()); LOG(debug, "PositionInserter: Adding value '%s'(%d) to attribute '%s' for docid '%d'", diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 7767a601ea4..47d6157525c 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -27,6 +27,7 @@ #include <vespa/storage/visiting/visitor.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/documentapi/messagebus/messages/queryresultmessage.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> using namespace search::aggregation; @@ -86,13 +87,12 @@ private: * This class gets callbacks when iterating through a field value and * inserts the values into a given attribute vector. **/ - class AttributeInserter : public document::FieldValue::IteratorHandler { + class AttributeInserter : public document::fieldvalue::IteratorHandler { protected: - typedef document::FieldValue::IteratorHandler::Content IteratorContent; search::AttributeVector & _attribute; search::AttributeVector::DocId _docId; - void onPrimitive(uint32_t fid, const IteratorContent & c) override; + void onPrimitive(uint32_t fid, const Content & c) override; public: AttributeInserter(search::AttributeVector & attribute, search::AttributeVector::DocId docId); @@ -103,7 +103,7 @@ private: PositionInserter(search::AttributeVector & attribute, search::AttributeVector::DocId docId); ~PositionInserter(); private: - void onPrimitive(uint32_t fid, const IteratorContent & c) override; + void onPrimitive(uint32_t fid, const Content & c) override; void onStructStart(const Content & fv) override; document::Field _fieldX; document::Field _fieldY; @@ -466,4 +466,3 @@ public: }; } - diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index d82f89da29d..c685a9b884f 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/xmlstream.h> using namespace vdslib; @@ -60,7 +61,7 @@ void Parameters::onDeserialize(const document::DocumentTypeRepo &repo, document: void Parameters::printXml(document::XmlOutputStream& xos) const { - using namespace document; + using namespace vespalib::xml; xos << XmlTag("parameters"); for (const auto & entry : _parameters) { xos << XmlTag("item") diff --git a/vdslib/src/vespa/vdslib/container/visitorordering.h b/vdslib/src/vespa/vdslib/container/visitorordering.h index 3d31a6ac078..e5a61d6332d 100644 --- a/vdslib/src/vespa/vdslib/container/visitorordering.h +++ b/vdslib/src/vespa/vdslib/container/visitorordering.h @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> #include <string> +#include <cstdint> namespace vdslib { diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index a75ba591af5..cc099164306 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -13,11 +13,10 @@ #include <vespa/config-stor-distribution.h> #include <list> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".vdslib.distribution"); -namespace storage { -namespace lib { +namespace storage::lib { namespace { std::vector<uint32_t> getDistributionBitMasks() { @@ -711,5 +710,4 @@ Distribution::splitNodesIntoLeafGroups(IndexList nodeList) const return result; } -} // lib -} // storage +} diff --git a/vdslib/src/vespa/vdslib/distribution/group.cpp b/vdslib/src/vespa/vdslib/distribution/group.cpp index 8ad3dcb2f80..2fce519298f 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.cpp +++ b/vdslib/src/vespa/vdslib/distribution/group.cpp @@ -6,9 +6,9 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> #include <ostream> +#include <cassert> -namespace storage { -namespace lib { +namespace storage::lib { Group::Group(uint16_t index, vespalib::stringref name) : _name(name), @@ -204,4 +204,3 @@ Group::getDistributionConfigHash() const { } } -} diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp index ca7ab742278..0ab4bc6b21a 100644 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp +++ b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp @@ -3,9 +3,9 @@ #include "idealnodecalculatorimpl.h" #include <vespa/vespalib/util/exceptions.h> #include <ostream> +#include <cassert> -namespace storage { -namespace lib { +namespace storage::lib { IdealNodeList::IdealNodeList() : _idealNodes() @@ -71,5 +71,4 @@ IdealNodeCalculatorImpl::initUpStateMapping() { } } -} // lib -} // storage +} diff --git a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp index 7ea270eb306..b1be394b65f 100644 --- a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp +++ b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - #include <vespa/document/util/stringutil.h> #include <vespa/fnet/frt/frt.h> #include <vespa/slobrok/sbmirror.h> -#include <iostream> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/util/programoptions.h> @@ -14,6 +11,8 @@ #include <vespa/config-stor-distribution.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/fastos/app.h> +#include <sstream> +#include <iostream> #include <vespa/log/log.h> LOG_SETUP("vdsstatetool"); diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp index 303d929f992..ce23fcd93cb 100644 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp @@ -1,9 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "application.h" #include <boost/program_options.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/config/common/exceptions.h> -#include "application.h" +#include <vespa/document/base/idstringexception.h> #include <iostream> + #include <vespa/log/log.h> LOG_SETUP("vespadoclocator"); diff --git a/vespalib/src/tests/random/friendfinder.cpp b/vespalib/src/tests/random/friendfinder.cpp index 5f6e55815e5..7bf390ef083 100644 --- a/vespalib/src/tests/random/friendfinder.cpp +++ b/vespalib/src/tests/random/friendfinder.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/random.h> -#include <stdio.h> #include <vespa/vespalib/stllike/string.h> #include <cmath> +#include <vector> int main(int argc, char **argv) { diff --git a/vespalib/src/vespa/vespalib/net/server_socket.cpp b/vespalib/src/vespa/vespalib/net/server_socket.cpp index 9359f5824c0..0a5310bbeb3 100644 --- a/vespalib/src/vespa/vespalib/net/server_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/server_socket.cpp @@ -2,6 +2,8 @@ #include "server_socket.h" #include "socket_spec.h" +#include <sys/stat.h> +#include <dirent.h> #include <vespa/log/log.h> LOG_SETUP(".vespalib.net.server_socket"); diff --git a/vespalib/src/vespa/vespalib/util/random.cpp b/vespalib/src/vespa/vespalib/util/random.cpp index 06bfc1f9c25..2abf67605a8 100644 --- a/vespalib/src/vespa/vespalib/util/random.cpp +++ b/vespalib/src/vespa/vespalib/util/random.cpp @@ -2,6 +2,9 @@ #include "random.h" #include <cmath> +#include <cstring> +#include <ctime> +#include <unistd.h> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/random.h b/vespalib/src/vespa/vespalib/util/random.h index c4cce3b42d8..f9d2587272d 100644 --- a/vespalib/src/vespa/vespalib/util/random.h +++ b/vespalib/src/vespa/vespalib/util/random.h @@ -1,14 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <stdlib.h> -#include <stdio.h> -#include <inttypes.h> -#include <vespa/fastos/types.h> -#include <vector> -#include <sys/types.h> -#include <unistd.h> -#include <time.h> +#include <cstdint> namespace vespalib { @@ -22,9 +15,9 @@ private: /** * step the random generator once **/ - void iterate(void) { - _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & - UINT64_C(0xFFFFFFFFFFFF); + void iterate() { + _state = (0x5DEECE66Dul * _state + 0xb) & + 0xFFFFFFFFFFFFul; } /** @@ -73,7 +66,7 @@ public: * @brief reset the seed **/ void setSeed(int64_t seed) { - _state = (seed ^ UINT64_C(0x5DEECE66D)) & ((1L << 48) -1); + _state = (seed ^ 0x5DEECE66Dul) & ((1L << 48) -1); }; /** diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp index db3c84bd42f..2bae7c03589 100644 --- a/vespalog/src/logctl/logctl.cpp +++ b/vespalog/src/logctl/logctl.cpp @@ -1,20 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <sys/types.h> -#include <cstdio> -#include <cstdlib> -#include <unistd.h> -#include <cstring> -#include <limits.h> - -#include <memory> -#include <vector> -#include <string> #include <vespa/defaults.h> #include <vespa/log/control-file.h> #include <vespa/log/internal.h> #include <vespa/log/component.h> -LOG_SETUP("vespa-logctl", "$Id$"); + +#include <unistd.h> +#include <dirent.h> +#include <sys/stat.h> + +LOG_SETUP("vespa-logctl"); + using namespace ns_log; diff --git a/vespalog/src/test/bufferedlogskiptest.cpp b/vespalog/src/test/bufferedlogskiptest.cpp index 5b386d18e90..cb16b73fa24 100644 --- a/vespalog/src/test/bufferedlogskiptest.cpp +++ b/vespalog/src/test/bufferedlogskiptest.cpp @@ -1,18 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include <fstream> #include <iostream> -#include <sstream> - -#include <sys/types.h> -#include <stdlib.h> #include <unistd.h> -#include <signal.h> -LOG_SETUP("bufferedlogskiptest", - "$Id$"); +LOG_SETUP("bufferedlogskiptest"); std::string readFile(const std::string& file) { std::ostringstream ost; diff --git a/vespalog/src/test/bufferedlogtest.cpp b/vespalog/src/test/bufferedlogtest.cpp index 0e27bd65988..97f70ebe004 100644 --- a/vespalog/src/test/bufferedlogtest.cpp +++ b/vespalog/src/test/bufferedlogtest.cpp @@ -1,21 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include "bufferedlogtest.logger1.h" #include "bufferedlogtest.logger2.h" #include <fstream> #include <iostream> -#include <sstream> - -#include <sys/types.h> -#include <stdlib.h> #include <unistd.h> -#include <signal.h> -LOG_SETUP("bufferedlogtest", - "$Id$"); +LOG_SETUP("bufferedlogtest"); std::string readFile(const std::string& file) { std::ostringstream ost; diff --git a/vespalog/src/test/bufferedlogtest.logger1.cpp b/vespalog/src/test/bufferedlogtest.logger1.cpp index 992de19fea5..5af4b9489a4 100644 --- a/vespalog/src/test/bufferedlogtest.logger1.cpp +++ b/vespalog/src/test/bufferedlogtest.logger1.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include "bufferedlogtest.logger1.h" LOG_SETUP(".logger1"); diff --git a/vespalog/src/test/bufferedlogtest.logger2.cpp b/vespalog/src/test/bufferedlogtest.logger2.cpp index 688ecd153aa..ac6935c2c68 100644 --- a/vespalog/src/test/bufferedlogtest.logger2.cpp +++ b/vespalog/src/test/bufferedlogtest.logger2.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include "bufferedlogtest.logger2.h" LOG_SETUP(".logger2"); diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp index 056f5aad2a2..90b8c13e497 100644 --- a/vespalog/src/test/threads/testthreads.cpp +++ b/vespalog/src/test/threads/testthreads.cpp @@ -2,7 +2,7 @@ #include <vespa/fastos/app.h> #include <vespa/fastos/time.h> #include <vespa/fastos/thread.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include <iostream> using std::string; diff --git a/vespalog/src/vespa/log/bufferedlogger.cpp b/vespalog/src/vespa/log/bufferedlogger.cpp index 30f3dcb648d..db2021324c3 100644 --- a/vespalog/src/vespa/log/bufferedlogger.cpp +++ b/vespalog/src/vespa/log/bufferedlogger.cpp @@ -1,14 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/bufferedlogger.h> - -#include <iomanip> -#include <iostream> -#include <sstream> -#include <vector> -#include <stdarg.h> -#include <sys/time.h> - +#include "bufferedlogger.h" +#include <vespa/fastos/mutex.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/identity.hpp> #include <boost/multi_index/member.hpp> @@ -16,6 +9,13 @@ #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> +#include <iomanip> +#include <iostream> +#include <sstream> +#include <vector> +#include <cstdarg> +#include <ctime> + namespace ns_log { // implementation details for BufferedLogger diff --git a/vespalog/src/vespa/log/bufferedlogger.h b/vespalog/src/vespa/log/bufferedlogger.h index 4f20f46af4c..9ab02f7842f 100644 --- a/vespalog/src/vespa/log/bufferedlogger.h +++ b/vespalog/src/vespa/log/bufferedlogger.h @@ -82,7 +82,6 @@ #define VESPA_LOG_LOGENTRYMAXAGE 300 // Max seconds an entry can be cached #define VESPA_LOG_COUNTAGEFACTOR 5 // How many seconds each count counts for -#include <vespa/fastos/mutex.h> #include <vespa/log/log.h> #include <sstream> #include <string> @@ -199,7 +198,6 @@ public: /** Trim the buffer. Removing old messages if wanted. */ void trimCache(); - }; } // ns_log diff --git a/vespalog/src/vespa/log/log-target-file.cpp b/vespalog/src/vespa/log/log-target-file.cpp index 07b638d7f84..e88f1018675 100644 --- a/vespalog/src/vespa/log/log-target-file.cpp +++ b/vespalog/src/vespa/log/log-target-file.cpp @@ -1,16 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <sys/types.h> #include <unistd.h> #include <cstring> -#include <sys/stat.h> #include <fcntl.h> -#include <cstdlib> -#include <errno.h> -#include <cstdio> +#include <cerrno> +#include <cassert> #include "log.h" LOG_SETUP(".log"); -LOG_RCSID("$Id$"); #include "log-target-file.h" #include "internal.h" diff --git a/vespalog/src/vespa/log/log-target.cpp b/vespalog/src/vespa/log/log-target.cpp index d34e0067b04..b6692caa308 100644 --- a/vespalog/src/vespa/log/log-target.cpp +++ b/vespalog/src/vespa/log/log-target.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <sys/types.h> #include <cstring> -#include <cstdlib> +#include <cassert> #include "log.h" LOG_SETUP(".log"); -LOG_RCSID("$Id$"); #include "log-target-fd.h" #include "log-target-file.h" diff --git a/vespalog/src/vespa/log/log.cpp b/vespalog/src/vespa/log/log.cpp index 7e92008504e..c31d110d241 100644 --- a/vespalog/src/vespa/log/log.cpp +++ b/vespalog/src/vespa/log/log.cpp @@ -10,6 +10,7 @@ LOG_SETUP_INDIRECT(".log", "$Id$"); #include "log-target.h" #include "internal.h" #include "control-file.h" +#include "bufferedlogger.h" #include <vespa/fastos/thread.h> @@ -222,6 +223,7 @@ Logger::doLog(LogLevel level, const char *file, int line, const char *fmt, ...) actualSize = tryLog(sizeofPayload, level, file, line, fmt, args); va_end(args); } while (sizeofPayload < actualSize); + ns_log::BufferedLogger::logger.trimCache(); } void Logger::doLogCore(uint64_t timestamp, LogLevel level, diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h index 6ecd69de792..53a0eec140c 100644 --- a/vespalog/src/vespa/log/log.h +++ b/vespalog/src/vespa/log/log.h @@ -38,14 +38,12 @@ static int log_dummmy __attribute__((unused)) = logger.setRcsId(x) do { \ if (logger.wants(ns_log::Logger::level)) { \ logger.doLog(ns_log::Logger::level, __FILE__, __LINE__, __VA_ARGS__); \ - ns_log::BufferedLogger::logger.trimCache(); \ } \ } while (0) #define VLOG(level, ...) \ do { \ if (logger.wants(level)) { \ logger.doLog(level, __FILE__, __LINE__, __VA_ARGS__); \ - ns_log::BufferedLogger::logger.trimCache(); \ } \ } while (0) #endif @@ -302,6 +300,3 @@ extern void log_abort(const char *message, #else #define LOG_ASSERT(expr) #endif // #ifndef NDEBUG - -#include <vespa/log/bufferedlogger.h> - diff --git a/vsm/src/vespa/vsm/common/document.h b/vsm/src/vespa/vsm/common/document.h index a0341aaf7c0..ea0f9216ab9 100644 --- a/vsm/src/vespa/vsm/common/document.h +++ b/vsm/src/vespa/vsm/common/document.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/query/base.h> #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/vespalib/stllike/hash_map.h> +#include <map> namespace vespalib { class asciistream; diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.h b/vsm/src/vespa/vsm/searcher/fieldsearcher.h index 3b82f68072b..e28275536f3 100644 --- a/vsm/src/vespa/vsm/searcher/fieldsearcher.h +++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/query/query.h> #include <vespa/vsm/common/document.h> #include <vespa/vsm/common/storagedocument.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> namespace vsm { @@ -79,9 +80,8 @@ public: size_t maxFieldLength() const { return _maxFieldLength; } private: - class IteratorHandler : public document::FieldValue::IteratorHandler { + class IteratorHandler : public document::fieldvalue::IteratorHandler { private: - typedef document::FieldValue::IteratorHandler::Content Content; FieldSearcher & _searcher; void onPrimitive(uint32_t fid, const Content & c) override; diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp index 961add6d7d6..b7ed6c0b7da 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp @@ -4,6 +4,7 @@ #include "slimefieldwriter.h" #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> #include <vespa/document/base/exceptions.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.docsumfilter"); @@ -13,9 +14,8 @@ using namespace search::docsummary; namespace { -class Handler : public document::FieldValue::IteratorHandler { +class Handler : public document::fieldvalue::IteratorHandler { public: - typedef document::FieldValue::IteratorHandler::Content Content; }; struct IntResultHandler : public Handler { diff --git a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h index 9c6ba8dae9c..aa58a6ae6df 100644 --- a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h +++ b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/vsm/common/charbuffer.h> namespace vsm { @@ -10,10 +11,8 @@ namespace vsm { * This class is used to flatten out and write a complex field value. * A separator string is inserted between primitive field values. **/ -class FlattenDocsumWriter : public document::FieldValue::IteratorHandler { +class FlattenDocsumWriter : public document::fieldvalue::IteratorHandler { private: - typedef document::FieldValue::IteratorHandler::Content Content; - CharBuffer _output; vespalib::string _separator; bool _useSeparator; diff --git a/vsm/src/vespa/vsm/vsm/snippetmodifier.h b/vsm/src/vespa/vsm/vsm/snippetmodifier.h index 17d6bb44d4c..182f09d5d55 100644 --- a/vsm/src/vespa/vsm/vsm/snippetmodifier.h +++ b/vsm/src/vespa/vsm/vsm/snippetmodifier.h @@ -7,6 +7,7 @@ #include <vespa/vsm/common/fieldmodifier.h> #include <vespa/vsm/searcher/utf8substringsnippetmodifier.h> #include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> namespace vsm { @@ -19,11 +20,9 @@ namespace vsm { * responsible for modifying the field value by inserting unit separators before and after matches. * A group separator is inserted between primitive field values the same way as done by FlattenDocsumWriter. **/ -class SnippetModifier : public FieldModifier, public document::FieldValue::IteratorHandler +class SnippetModifier : public FieldModifier, public document::fieldvalue::IteratorHandler { private: - typedef document::FieldValue::IteratorHandler::Content Content; - UTF8SubstringSnippetModifier::SP _searcher; CharBuffer::SP _valueBuf; // buffer to store the final modified field value char _groupSep; |