diff options
13 files changed, 166 insertions, 105 deletions
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 55b5f63c92b..a5cac711d39 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("docsummary_test"); #include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/common/bucketfactory.h> @@ -27,6 +24,9 @@ LOG_SETUP("docsummary_test"); #include <vespa/vespalib/tensor/default_tensor.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/log/log.h> +LOG_SETUP("docsummary_test"); using namespace document; using namespace search; diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index bc7deea304f..018849cc9ad 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("documentdb_test"); #include <vespa/document/repo/documenttyperepo.h> #include <vespa/messagebus/emptyreply.h> @@ -23,6 +20,7 @@ LOG_SETUP("documentdb_test"); #include <tests/proton/common/dummydbowner.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/vespalib/data/slime/slime.h> using document::DocumentType; using document::DocumentTypeRepo; diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 5e6a7acbc2c..bffedd30a9f 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("documentmetastore_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> @@ -16,8 +13,11 @@ LOG_SETUP("documentmetastore_test"); #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/searchcore/proton/server/itlssyncer.h> #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/log/log.h> +LOG_SETUP("documentmetastore_test"); using namespace document; using search::AttributeVector; diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 6582df2f9d3..01bc87761fc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -60,6 +60,7 @@ vespa_add_library(searchcore_server STATIC matchers.cpp matchhandlerproxy.cpp matchview.cpp + memoryconfigstore.cpp memory_flush_config_updater.cpp memoryflush.cpp minimal_document_retriever.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp new file mode 100644 index 00000000000..b070e2d8fc2 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "memoryconfigstore.h" + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.memoryconfigstore"); + +namespace proton { + +MemoryConfigStore::MemoryConfigStore() : _maps(new ConfigMaps) {} +MemoryConfigStore::MemoryConfigStore(ConfigMaps::SP maps) : _maps(maps) {} +MemoryConfigStore::~MemoryConfigStore() { } + +ConfigStore::SerialNum +MemoryConfigStore::getBestSerialNum() const { + return _maps->_valid.empty() ? 0 : *_maps->_valid.rbegin(); +} +ConfigStore::SerialNum +MemoryConfigStore::getOldestSerialNum() const { + return _maps->_valid.empty() ? 0 : *_maps->_valid.begin(); +} +bool +MemoryConfigStore::hasValidSerial(SerialNum serial) const { + return _maps->_valid.find(serial) != _maps->_valid.end(); +} +ConfigStore::SerialNum +MemoryConfigStore::getPrevValidSerial(SerialNum serial) const { + if (_maps->_valid.empty() || + *_maps->_valid.begin() >= serial) { + return 0; + } + return *(--(_maps->_valid.lower_bound(serial))); +} +void +MemoryConfigStore::saveConfig(const DocumentDBConfig &config, + const Schema &history, + SerialNum serial) +{ + _maps->configs[serial].reset(new DocumentDBConfig(config)); + _maps->histories[serial].reset(new Schema(history)); + _maps->_valid.insert(serial); +} +void +MemoryConfigStore::loadConfig(const DocumentDBConfig &, SerialNum serial, + DocumentDBConfig::SP &loaded_config, + Schema::SP &history_schema) +{ + assert(hasValidSerial(serial)); + loaded_config = _maps->configs[serial]; + history_schema = _maps->histories[serial]; +} +void +MemoryConfigStore::removeInvalid() +{ + // Note: Depends on C++11 semantics for erase + for (auto it = _maps->configs.begin(); it != _maps->configs.end();) { + if (!hasValidSerial(it->first)) { + it = _maps->configs.erase(it); + continue; + } + ++it; + } + for (auto it = _maps->histories.begin(); + it != _maps->histories.end();) { + if (!hasValidSerial(it->first)) { + it = _maps->histories.erase(it); + continue; + } + ++it; + } +} +void +MemoryConfigStore::prune(SerialNum serial) { + _maps->configs.erase(_maps->configs.begin(), + _maps->configs.upper_bound(serial)); + _maps->histories.erase(_maps->histories.begin(), + _maps->histories.upper_bound(serial)); + _maps->_valid.erase(_maps->_valid.begin(), + _maps->_valid.upper_bound(serial)); +} +void +MemoryConfigStore::saveWipeHistoryConfig(SerialNum serial, + fastos::TimeStamp wipeTimeLimit) +{ + if (hasValidSerial(serial)) { + return; + } + SerialNum prev = getPrevValidSerial(serial); + Schema::SP schema(new Schema); + if (wipeTimeLimit != 0) { + Schema::SP oldHistorySchema(_maps->histories[prev]); + Schema::UP wipeSchema; + wipeSchema = oldHistorySchema->getOldFields(wipeTimeLimit); + schema.reset(Schema::set_difference(*oldHistorySchema, *wipeSchema).release()); + + } + _maps->histories[serial] = schema; + _maps->configs[serial] = _maps->configs[prev]; + _maps->_valid.insert(serial); +} +void +MemoryConfigStore::serializeConfig(SerialNum, vespalib::nbostream &) { + LOG(info, "Serialization of config not implemented."); +} +void +MemoryConfigStore::deserializeConfig(SerialNum, vespalib::nbostream &) { + assert(!"Not implemented"); +} +void +MemoryConfigStore::setProtonConfig(const ProtonConfigSP &) { } + +MemoryConfigStores::MemoryConfigStores() { } +MemoryConfigStores::~MemoryConfigStores() { } + +ConfigStore::UP +MemoryConfigStores::getConfigStore(const std::string &type) { + if (!_config_maps[type].get()) { + _config_maps[type].reset(new ConfigMaps); + } + return ConfigStore::UP(new MemoryConfigStore(_config_maps[type])); +} + +} // namespace proton + diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h index c0c0f5aa6ca..27bf81a0f8a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h +++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h @@ -12,103 +12,37 @@ namespace proton { struct ConfigMaps { typedef std::shared_ptr<ConfigMaps> SP; std::map<search::SerialNum, DocumentDBConfig::SP> configs; - std::map<search::SerialNum, - search::index::Schema::SP> histories; + std::map<search::SerialNum, search::index::Schema::SP> histories; std::set<search::SerialNum> _valid; }; class MemoryConfigStore : public ConfigStore { + using Schema = search::index::Schema; ConfigMaps::SP _maps; public: - MemoryConfigStore() : _maps(new ConfigMaps) {} - MemoryConfigStore(ConfigMaps::SP maps) : _maps(maps) {} - - virtual SerialNum getBestSerialNum() const { - return _maps->_valid.empty() ? 0 : *_maps->_valid.rbegin(); - } - virtual SerialNum getOldestSerialNum() const { - return _maps->_valid.empty() ? 0 : *_maps->_valid.begin(); - } - virtual bool hasValidSerial(SerialNum serial) const { - return _maps->_valid.find(serial) != _maps->_valid.end(); - } - virtual SerialNum getPrevValidSerial(SerialNum serial) const { - if (_maps->_valid.empty() || - *_maps->_valid.begin() >= serial) { - return 0; - } - return *(--(_maps->_valid.lower_bound(serial))); - } + MemoryConfigStore(const MemoryConfigStore &) = delete; + MemoryConfigStore & operator = (const MemoryConfigStore &) = delete; + MemoryConfigStore(); + MemoryConfigStore(ConfigMaps::SP maps); + ~MemoryConfigStore(); + + virtual SerialNum getBestSerialNum() const; + virtual SerialNum getOldestSerialNum() const; + virtual bool hasValidSerial(SerialNum serial) const; + virtual SerialNum getPrevValidSerial(SerialNum serial) const; virtual void saveConfig(const DocumentDBConfig &config, - const search::index::Schema &history, - SerialNum serial) { - _maps->configs[serial].reset(new DocumentDBConfig(config)); - _maps->histories[serial].reset(new search::index::Schema(history)); - _maps->_valid.insert(serial); - } + const Schema &history, + SerialNum serial); virtual void loadConfig(const DocumentDBConfig &, SerialNum serial, DocumentDBConfig::SP &loaded_config, - search::index::Schema::SP &history_schema) { - assert(hasValidSerial(serial)); - loaded_config = _maps->configs[serial]; - history_schema = _maps->histories[serial]; - } - virtual void removeInvalid() - { - // Note: Depends on C++11 semantics for erase - for (auto it = _maps->configs.begin(); it != _maps->configs.end();) { - if (!hasValidSerial(it->first)) { - it = _maps->configs.erase(it); - continue; - } - ++it; - } - for (auto it = _maps->histories.begin(); - it != _maps->histories.end();) { - if (!hasValidSerial(it->first)) { - it = _maps->histories.erase(it); - continue; - } - ++it; - } - } - void prune(SerialNum serial) { - _maps->configs.erase(_maps->configs.begin(), - _maps->configs.upper_bound(serial)); - _maps->histories.erase(_maps->histories.begin(), - _maps->histories.upper_bound(serial)); - _maps->_valid.erase(_maps->_valid.begin(), - _maps->_valid.upper_bound(serial)); - } - virtual void saveWipeHistoryConfig(SerialNum serial, - fastos::TimeStamp wipeTimeLimit) - { - if (hasValidSerial(serial)) { - return; - } - SerialNum prev = getPrevValidSerial(serial); - search::index::Schema::SP schema(new search::index::Schema); - if (wipeTimeLimit != 0) { - search::index::Schema::SP oldHistorySchema(_maps->histories[prev]); - search::index::Schema::UP wipeSchema; - wipeSchema = oldHistorySchema->getOldFields(wipeTimeLimit); - schema.reset(search::index::Schema:: - set_difference(*oldHistorySchema, - *wipeSchema).release()); - - } - _maps->histories[serial] = schema; - _maps->configs[serial] = _maps->configs[prev]; - _maps->_valid.insert(serial); - } - virtual void serializeConfig(SerialNum, vespalib::nbostream &) { - LOG(info, "Serialization of config not implemented."); - } - virtual void deserializeConfig(SerialNum, vespalib::nbostream &) { - assert(!"Not implemented"); - } - virtual void setProtonConfig(const ProtonConfigSP &) override { } + Schema::SP &history_schema); + virtual void removeInvalid(); + void prune(SerialNum serial); + virtual void saveWipeHistoryConfig(SerialNum serial, fastos::TimeStamp wipeTimeLimit); + virtual void serializeConfig(SerialNum, vespalib::nbostream &); + virtual void deserializeConfig(SerialNum, vespalib::nbostream &); + virtual void setProtonConfig(const ProtonConfigSP &) override; }; // Holds the state of a set of MemoryConfigStore objects, making stored @@ -117,12 +51,11 @@ class MemoryConfigStores { std::map<std::string, ConfigMaps::SP> _config_maps; public: - ConfigStore::UP getConfigStore(const std::string &type) { - if (!_config_maps[type].get()) { - _config_maps[type].reset(new ConfigMaps); - } - return ConfigStore::UP(new MemoryConfigStore(_config_maps[type])); - } + MemoryConfigStores(const MemoryConfigStores &) = delete; + MemoryConfigStores & operator = (const MemoryConfigStores &) = delete; + MemoryConfigStores(); + ~MemoryConfigStores(); + ConfigStore::UP getConfigStore(const std::string &type); }; } // namespace proton diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index b8ce7d3cab5..dcd19468ac7 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -11,6 +11,7 @@ #include <vespa/config-fleetcontroller.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/config/config.h> #include <vespa/log/log.h> LOG_SETUP(".test.servicelayerapp"); diff --git a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp b/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp index dc0fcc5592a..7a03d1c017a 100644 --- a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp +++ b/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp @@ -6,6 +6,7 @@ #include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> #include <tests/common/teststorageapp.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/util/exceptions.h> #include <boost/lexical_cast.hpp> namespace storage { diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index 03d73193086..d1a5100b7de 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -6,6 +6,7 @@ #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/exceptions.h> using document::DocumentType; using storage::framework::defaultimplementation::AllocationLogic; diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index e705db80788..a5e637146fc 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <cppunit/extensions/HelperMacros.h> #include <memory> #include <iterator> @@ -16,6 +15,7 @@ #include <vespa/storage/persistence/messages.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> +#include <vespa/vespalib/util/exceptions.h> using namespace document; using namespace storage::api; diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index d782abf7d54..f83ac825aa4 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -22,6 +22,7 @@ #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/vespalib/util/exceptions.h> LOG_SETUP(".visitormanagertest"); diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index aed08a676b8..90a1c51fe1f 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/rawfieldvalue.h> -#include <vespa/log/log.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/visitor.h> @@ -20,10 +18,12 @@ #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/vespalib/util/exceptions.h> #include <vector> #include <thread> #include <chrono> +#include <vespa/log/log.h> LOG_SETUP(".visitortest"); using namespace std::chrono_literals; diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index 7cd337a861b..62a96ea8111 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -23,6 +23,7 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/storageserver/app/distributorprocess.h> #include <vespa/storageserver/app/memfileservicelayerprocess.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> LOG_SETUP(".storageservertest"); |